From 41514c8241b0e4463ee95e1d40740ad003549148 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Fri, 21 Jan 2011 01:02:20 +0900 Subject: [PATCH 01/11] recipe: update: Endgadget japanese, add tags removal --- resources/recipes/endgadget_ja.recipe | 3 +++ 1 file changed, 3 insertions(+) diff --git a/resources/recipes/endgadget_ja.recipe b/resources/recipes/endgadget_ja.recipe index 891e6720a5..3c20380e9b 100644 --- a/resources/recipes/endgadget_ja.recipe +++ b/resources/recipes/endgadget_ja.recipe @@ -18,3 +18,6 @@ class EndgadgetJapan(BasicNewsRecipe): language = 'ja' encoding = 'utf-8' feeds = [(u'engadget', u'http://japanese.engadget.com/rss.xml')] + + remove_tags_before = dict(name="div", attrs={'id':"content_wrap"}) + remove_tags_after = dict(name='h3', attrs={'id':'addcomments'}) From 00f0e37bffe5f3795b3ddb79b5ace2bb18fed88e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 29 Jan 2011 09:26:51 +0000 Subject: [PATCH 02/11] Fix exception in restore_database caused by comma in a chained message --- src/calibre/gui2/dialogs/restore_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/restore_library.py b/src/calibre/gui2/dialogs/restore_library.py index dd1befc11b..b5af0fcadd 100644 --- a/src/calibre/gui2/dialogs/restore_library.py +++ b/src/calibre/gui2/dialogs/restore_library.py @@ -104,7 +104,7 @@ def restore_database(db, parent=None): else: if r.errors_occurred: warning_dialog(parent, _('Success'), - _('Restoring the database succeeded with some warnings', + _('Restoring the database succeeded with some warnings' ' click Show details to see the details.'), det_msg=r.report, show=True) else: From 2dafa291845e380c7a1b00bb098b82d944b90961 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 29 Jan 2011 12:30:12 +0000 Subject: [PATCH 03/11] 1) restore: provide OPF with the path to the folder containing the OPF file. Make the cover path be correct. 2) check_library.py: change the wording of the cover messages. --- src/calibre/library/check_library.py | 4 ++-- src/calibre/library/restore.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/check_library.py b/src/calibre/library/check_library.py index b49330db3e..19ecb97308 100644 --- a/src/calibre/library/check_library.py +++ b/src/calibre/library/check_library.py @@ -30,8 +30,8 @@ CHECKS = [('invalid_titles', _('Invalid titles'), True, False), ('missing_formats', _('Missing book formats'), False, False), ('extra_formats', _('Extra book formats'), True, False), ('extra_files', _('Unknown files in books'), True, False), - ('missing_covers', _('Missing covers in books'), False, True), - ('extra_covers', _('Extra covers in books'), True, True), + ('missing_covers', _('Missing covers files'), False, True), + ('extra_covers', _('Cover files not in database'), True, True), ('failed_folders', _('Folders raising exception'), False, False) ] diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index bc2c740279..76f3c0333d 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -141,7 +141,7 @@ class Restore(Thread): sizes = [os.path.getsize(os.path.join(dirpath, x)) for x in formats] names = [os.path.splitext(x)[0] for x in formats] opf = os.path.join(dirpath, 'metadata.opf') - mi = OPF(opf).to_book_metadata() + mi = OPF(opf, basedir=dirpath).to_book_metadata() timestamp = os.path.getmtime(opf) path = os.path.relpath(dirpath, self.src_library_path).replace(os.sep, '/') From 60687f9ba60cb4f20aefffe991e63d0d4ed8ea61 Mon Sep 17 00:00:00 2001 From: GRiker Date: Sat, 29 Jan 2011 06:10:08 -0700 Subject: [PATCH 04/11] GwR patch for EB600 driver, add gui_name --- src/calibre/devices/eb600/driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/devices/eb600/driver.py b/src/calibre/devices/eb600/driver.py index 95f6dc6ab0..1f723ce46a 100644 --- a/src/calibre/devices/eb600/driver.py +++ b/src/calibre/devices/eb600/driver.py @@ -21,6 +21,7 @@ from calibre.devices.usbms.driver import USBMS class EB600(USBMS): name = 'Netronix EB600 Device Interface' + gui_name = 'Netronix EB600' description = _('Communicate with the EB600 eBook reader.') author = 'Kovid Goyal' supported_platforms = ['windows', 'osx', 'linux'] From 6f8cdbe41ae1e3d51605707dc060961cf037e177 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jan 2011 08:30:54 -0700 Subject: [PATCH 05/11] ... --- src/calibre/gui2/dialogs/check_library.py | 34 +++++++++++++---------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/dialogs/check_library.py b/src/calibre/gui2/dialogs/check_library.py index bd665a7e2e..b6b15d8be8 100644 --- a/src/calibre/gui2/dialogs/check_library.py +++ b/src/calibre/gui2/dialogs/check_library.py @@ -74,21 +74,27 @@ class DBCheck(QDialog): self.reject() def start_load(self): - self.conn.close() - self.pb.setMaximum(self.count) - self.pb.setValue(0) - self.msg.setText(_('Loading database from SQL')) - self.db.conn.close() - self.ndbpath = PersistentTemporaryFile('.db') - self.ndbpath.close() - self.ndbpath = self.ndbpath.name - t = DBThread(self.ndbpath, False) - t.connect() - self.conn = t.conn - self.conn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)') - self.conn.commit() + try: + self.conn.close() + self.pb.setMaximum(self.count) + self.pb.setValue(0) + self.msg.setText(_('Loading database from SQL')) + self.db.conn.close() + self.ndbpath = PersistentTemporaryFile('.db') + self.ndbpath.close() + self.ndbpath = self.ndbpath.name + t = DBThread(self.ndbpath, False) + t.connect() + self.conn = t.conn + self.conn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)') + self.conn.commit() + + QTimer.singleShot(0, self.do_one_load) + except Exception, e: + import traceback + self.error = (as_unicode(e), traceback.format_exc()) + self.reject() - QTimer.singleShot(0, self.do_one_load) def do_one_load(self): if self.rejected: From af6ce250b3fc3e7c636f0f8b6bf8b3b52ecae92b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jan 2011 09:17:12 -0700 Subject: [PATCH 06/11] Fix device job error dialog not showing actual error --- src/calibre/gui2/device.py | 2 +- src/calibre/gui2/dialogs/message_box.py | 29 +++++++++++++++++-------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 1cf0fa5d67..020ba7fb32 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -687,7 +687,7 @@ class DeviceMixin(object): # {{{ except: pass if not self.device_error_dialog.isVisible(): - self.device_error_dialog.setDetailedText(job.details) + self.device_error_dialog.set_details(job.details) self.device_error_dialog.show() # Device connected {{{ diff --git a/src/calibre/gui2/dialogs/message_box.py b/src/calibre/gui2/dialogs/message_box.py index 565fb147fc..9d586ce28d 100644 --- a/src/calibre/gui2/dialogs/message_box.py +++ b/src/calibre/gui2/dialogs/message_box.py @@ -45,14 +45,12 @@ class MessageBox(QDialog, Ui_Dialog): self.ctc_button.clicked.connect(self.copy_to_clipboard) - if det_msg: - self.show_det_msg = _('Show &details') - self.hide_det_msg = _('Hide &details') - self.det_msg_toggle = self.bb.addButton(self.show_det_msg, self.bb.ActionRole) - self.det_msg_toggle.clicked.connect(self.toggle_det_msg) - self.det_msg_toggle.setToolTip( - _('Show detailed information about this error')) - + self.show_det_msg = _('Show &details') + self.hide_det_msg = _('Hide &details') + self.det_msg_toggle = self.bb.addButton(self.show_det_msg, self.bb.ActionRole) + self.det_msg_toggle.clicked.connect(self.toggle_det_msg) + self.det_msg_toggle.setToolTip( + _('Show detailed information about this error')) self.copy_action = QAction(self) self.addAction(self.copy_action) @@ -66,10 +64,14 @@ class MessageBox(QDialog, Ui_Dialog): else: self.bb.button(self.bb.Ok).setDefault(True) + if not det_msg: + self.det_msg_toggle.setVisible(False) + self.do_resize() + def toggle_det_msg(self, *args): - vis = self.det_msg.isVisible() + vis = unicode(self.det_msg_toggle.text()) == self.hide_det_msg self.det_msg_toggle.setText(self.show_det_msg if vis else self.hide_det_msg) self.det_msg.setVisible(not vis) @@ -100,6 +102,15 @@ class MessageBox(QDialog, Ui_Dialog): self.bb.button(self.bb.Ok).setFocus(Qt.OtherFocusReason) return ret + def set_details(self, msg): + if not msg: + msg = '' + self.det_msg.setPlainText(msg) + self.det_msg_toggle.setText(self.show_det_msg) + self.det_msg_toggle.setVisible(bool(msg)) + self.det_msg.setVisible(False) + self.do_resize() + if __name__ == '__main__': app = QApplication([]) from calibre.gui2 import question_dialog From ac8ea5b16babcb9ef55025ad005e3cd86fc6343d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jan 2011 09:28:13 -0700 Subject: [PATCH 07/11] Fix #8646 (conversion of kindle format to epub fails ValueError: All strings must be XML compatible: Unicode or ASCII, no NULL bytes) --- src/calibre/ebooks/mobi/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 2f397006a1..0ae3c9ac9d 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -488,7 +488,7 @@ class MobiReader(object): def remove_random_bytes(self, html): - return re.sub('\x14|\x15|\x19|\x1c|\x1d|\xef|\x12|\x13|\xec|\x08', + return re.sub('\x14|\x15|\x19|\x1c|\x1d|\xef|\x12|\x13|\xec|\x08|\x01|\x02|\x03|\x04|\x05|\x06|\x07', '', html) def ensure_unit(self, raw, unit='px'): From 7239c3af0134a9f1ea8150f0a8493e9a65acfe8d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jan 2011 09:49:34 -0700 Subject: [PATCH 08/11] Fix #8656 (Attempting to merge two books, getting ValueError) --- src/calibre/library/database2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 3c6d4016f2..3fc16e99b4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1549,7 +1549,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): elif mi.cover is not None: if os.access(mi.cover, os.R_OK): with lopen(mi.cover, 'rb') as f: - doit(self.set_cover, id, f, commit=False) + raw = f.read() + if raw: + doit(self.set_cover, id, raw, commit=False) if mi.tags: doit(self.set_tags, id, mi.tags, notify=False, commit=False) if mi.comments: From 0cd6535a0396943f1722b7261232294346c79fc7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jan 2011 09:57:21 -0700 Subject: [PATCH 09/11] Fix #8662 (Updated recipe for Vijesti) --- resources/recipes/vijesti.recipe | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/resources/recipes/vijesti.recipe b/resources/recipes/vijesti.recipe index c901755b78..d650bdec06 100644 --- a/resources/recipes/vijesti.recipe +++ b/resources/recipes/vijesti.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2009-2010, Darko Miletic ' +__copyright__ = '2009-2011, Darko Miletic ' ''' vijesti.me @@ -18,12 +18,16 @@ class Vijesti(BasicNewsRecipe): oldest_article = 2 max_articles_per_feed = 150 no_stylesheets = True - encoding = 'cp1250' + encoding = 'utf8' use_embedded_content = False language = 'sr' publication_type = 'newspaper' - masthead_url = 'http://www.vijesti.me/img/logo.gif' - extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}' + extra_css = """ + @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} + @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} + body{font-family: Georgia,"Times New Roman",Times,serif1,serif} + .articledescription,.article,.chapter{font-family: sans1, sans-serif} + """ conversion_options = { 'comment' : description @@ -34,11 +38,11 @@ class Vijesti(BasicNewsRecipe): preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')] - keep_only_tags = [dict(name='div', attrs={'id':'mainnews'})] + keep_only_tags = [dict(name='div', attrs={'id':['article_intro_text','article_text']})] remove_tags = [dict(name=['object','link','embed','form'])] - feeds = [(u'Sve vijesti', u'http://www.vijesti.me/rss.php' )] + feeds = [(u'Sve vijesti', u'http://www.vijesti.me/rss/' )] def preprocess_html(self, soup): return self.adeify_images(soup) From 7764b0b0074f24faec1711e090787471a5a0cd8d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Jan 2011 10:16:51 -0700 Subject: [PATCH 10/11] LIT Input: Handle LIT files that have all their text in a single 
 tag and use entities

---
 src/calibre/ebooks/lit/input.py             | 5 ++++-
 src/calibre/gui2/dialogs/restore_library.py | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/lit/input.py b/src/calibre/ebooks/lit/input.py
index ff901c3715..9ccbba543f 100644
--- a/src/calibre/ebooks/lit/input.py
+++ b/src/calibre/ebooks/lit/input.py
@@ -38,13 +38,16 @@ class LITInput(InputFormatPlugin):
                 if len(body) == 1 and body[0].tag == XHTML('pre'):
                     pre = body[0]
                     from calibre.ebooks.txt.processor import convert_basic, preserve_spaces, \
-                    separate_paragraphs_single_line
+                        separate_paragraphs_single_line
+                    from calibre.ebooks.chardet import xml_to_unicode
                     from lxml import etree
                     import copy
                     html = separate_paragraphs_single_line(pre.text)
                     html = preserve_spaces(html)
                     html = convert_basic(html).replace('',
                             ''%XHTML_NS)
+                    html = xml_to_unicode(html, strip_encoding_pats=True,
+                            resolve_entities=True)[0]
                     root = etree.fromstring(html)
                     body = XPath('//h:body')(root)
                     pre.tag = XHTML('div')
diff --git a/src/calibre/gui2/dialogs/restore_library.py b/src/calibre/gui2/dialogs/restore_library.py
index b5af0fcadd..a57d6c86c1 100644
--- a/src/calibre/gui2/dialogs/restore_library.py
+++ b/src/calibre/gui2/dialogs/restore_library.py
@@ -21,7 +21,7 @@ class DBRestore(QDialog):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
         self.l1 = QLabel(''+_('Restoring database from backups, do not'
-            ' interrupt, this will happen in two stages')+'...')
+            ' interrupt, this will happen in three stages')+'...')
         self.setWindowTitle(_('Restoring database'))
         self.l.addWidget(self.l1)
         self.pb = QProgressBar(self)

From f3ecd4f9ec4b43c72a41cfd30494f89adbf39846 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 29 Jan 2011 11:17:19 -0700
Subject: [PATCH 11/11] Improve Ledevoir

---
 resources/recipes/ledevoir.recipe | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/resources/recipes/ledevoir.recipe b/resources/recipes/ledevoir.recipe
index c54f21c7ec..bc473be181 100644
--- a/resources/recipes/ledevoir.recipe
+++ b/resources/recipes/ledevoir.recipe
@@ -9,6 +9,8 @@ __description__   = 'Canadian Paper '
 http://www.ledevoir.com/
 '''
 
+import re
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class ledevoir(BasicNewsRecipe):
@@ -32,6 +34,8 @@ class ledevoir(BasicNewsRecipe):
     remove_javascript     = True
     no_stylesheets        = True
 
+    preprocess_regexps = [(re.compile(r'(title|alt)=".*?>.*?"', re.DOTALL), lambda m: '')]
+
     keep_only_tags  = [
                         dict(name='div', attrs={'id':'article'}),
                         dict(name='ul', attrs={'id':'ariane'})