From e995160f2880aeda5d979ce1b026aabf3b6cba07 Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 7 May 2010 22:41:08 -0400 Subject: [PATCH 01/13] Fix bug #5441: PDB created in Dropbook not convertable by Calibre. --- src/calibre/ebooks/metadata/ereader.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/metadata/ereader.py b/src/calibre/ebooks/metadata/ereader.py index 036baff2aa..dd8b97b46f 100644 --- a/src/calibre/ebooks/metadata/ereader.py +++ b/src/calibre/ebooks/metadata/ereader.py @@ -8,6 +8,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' +import re import struct from calibre.ebooks.metadata import MetaInformation @@ -44,12 +45,12 @@ def get_metadata(stream, extract_cover=True): if hr.compression in (2, 10) and hr.has_metadata == 1: try: mdata = pheader.section_data(hr.metadata_offset) - + mdata = mdata.split('\x00') - mi.title = mdata[0] - mi.authors = [mdata[1]] - mi.publisher = mdata[3] - mi.isbn = mdata[4] + mi.title = re.sub(r'[^a-zA-Z0-9 \._=\+\-!\?,\'\"]', '', mdata[0]) + mi.authors = [re.sub(r'[^a-zA-Z0-9 \._=\+\-!\?,\'\"]', '', mdata[1])] + mi.publisher = re.sub(r'[^a-zA-Z0-9 \._=\+\-!\?,\'\"]', '', mdata[3]) + mi.isbn = re.sub(r'[^a-zA-Z0-9 \._=\+\-!\?,\'\"]', '', mdata[4]) except: pass From 6b79a732b192668bb2ae71969a7ea8581d1f759c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 7 May 2010 22:30:13 -0600 Subject: [PATCH 02/13] Make proxy detection more robust on windows and OS X. calibre now queries OS X Network Settigns if no environment variables are set. Also handle proxies with a trailing slash correctly. --- src/calibre/__init__.py | 98 +++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 57 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 97a3842f1b..5039331660 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -4,6 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' __docformat__ = 'restructuredtext en' import sys, os, re, logging, time, mimetypes, \ __builtin__, warnings, multiprocessing +from urllib import getproxies __builtin__.__dict__['dynamic_property'] = lambda(func): func(None) from htmlentitydefs import name2codepoint from math import floor @@ -199,71 +200,54 @@ def extract(path, dir): extractor(path, dir) def get_proxies(debug=True): - proxies = {} + proxies = getproxies() + for key, proxy in list(proxies.items()): + if not proxy: + del proxies[key] + continue + if proxy.startswith(key+'://'): + proxy = proxy[len(key)+3:] + if proxy.endswith('/'): + proxy = proxy[:-1] + if len(proxy) > 4: + proxies[key] = proxy + else: + prints('Removing invalid', key, 'proxy:', proxy) + del proxies[key] - for q in ('http', 'ftp'): - proxy = os.environ.get(q+'_proxy', None) - if not proxy: continue - if proxy.startswith(q+'://'): - proxy = proxy[7:] - proxies[q] = proxy - - if iswindows: - try: - winreg = __import__('_winreg') - settings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, - 'Software\\Microsoft\\Windows' - '\\CurrentVersion\\Internet Settings') - proxy = winreg.QueryValueEx(settings, "ProxyEnable")[0] - if proxy: - server = str(winreg.QueryValueEx(settings, 'ProxyServer')[0]) - if ';' in server: - for p in server.split(';'): - protocol, address = p.split('=') - proxies[protocol] = address - else: - proxies['http'] = server - proxies['ftp'] = server - settings.Close() - except Exception, e: - prints('Unable to detect proxy settings: %s' % str(e)) - for x in list(proxies): - if len(proxies[x]) < 5: - prints('Removing invalid', x, 'proxy:', proxies[x]) - del proxies[x] if proxies and debug: prints('Using proxies:', proxies) return proxies def get_parsed_proxy(typ='http', debug=True): proxies = get_proxies(debug) - if typ not in proxies: - return - pattern = re.compile(( - '(?:ptype://)?' \ - '(?:(?P\w+):(?P.*)@)?' \ - '(?P[\w\-\.]+)' \ - '(?::(?P\d+))?').replace('ptype', typ) - ) + proxy = proxies.get(typ, None) + if proxy: + pattern = re.compile(( + '(?:ptype://)?' \ + '(?:(?P\w+):(?P.*)@)?' \ + '(?P[\w\-\.]+)' \ + '(?::(?P\d+))?').replace('ptype', typ) + ) - match = pattern.match(proxies['typ']) - if match: - try: - ans = { - 'host' : match.group('host'), - 'port' : match.group('port'), - 'user' : match.group('user'), - 'pass' : match.group('pass') - } - if ans['port']: - ans['port'] = int(ans['port']) - except: - if debug: - traceback.print_exc() - return - if debug: - prints('Using http proxy', ans) - return ans + match = pattern.match(proxies[typ]) + if match: + try: + ans = { + 'host' : match.group('host'), + 'port' : match.group('port'), + 'user' : match.group('user'), + 'pass' : match.group('pass') + } + if ans['port']: + ans['port'] = int(ans['port']) + except: + if debug: + traceback.print_exc() + else: + if debug: + prints('Using http proxy', str(ans)) + return ans def browser(honor_time=True, max_time=2, mobile_browser=False): From e520f31ed674d1ebbfa750ef6f92950d49ce4fbf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 00:24:37 -0600 Subject: [PATCH 03/13] CBC Input: Handle comics.txt encoded in UTF-16 with a BOM --- src/calibre/ebooks/comic/input.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/comic/input.py b/src/calibre/ebooks/comic/input.py index 3a3cb7d83e..0a05bd2cca 100755 --- a/src/calibre/ebooks/comic/input.py +++ b/src/calibre/ebooks/comic/input.py @@ -341,8 +341,15 @@ class ComicInput(InputFormatPlugin): if not os.path.exists('comics.txt'): raise ValueError('%s is not a valid comic collection' %stream.name) - raw = open('comics.txt', 'rb').read().decode('utf-8') - raw.lstrip(unicode(codecs.BOM_UTF8, "utf8" )) + raw = open('comics.txt', 'rb').read() + if raw.startswith(codecs.BOM_UTF16_BE): + raw = raw.decode('utf-16-be')[1:] + elif raw.startswith(codecs.BOM_UTF16_LE): + raw = raw.decode('utf-16-le')[1:] + elif raw.startswith(codecs.BOM_UTF8): + raw = raw.decode('utf-8')[1:] + else: + raw = raw.decode('utf-8') for line in raw.splitlines(): line = line.strip() if not line: From 655fcb828a823844a288e948aa46bce8594acb80 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 06:51:25 -0600 Subject: [PATCH 04/13] Regression that broke CHM conversion of OS X. Fixes #5483 (Error when converting CHM to PDF) --- src/calibre/ebooks/html/input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index 35c0acc097..326afc00c0 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -292,7 +292,7 @@ class HTMLInput(InputFormatPlugin): encoding=opts.input_encoding) def is_case_sensitive(self, path): - if self._is_case_sensitive is not None: + if getattr(self, '_is_case_sensitive', None) is not None: return self._is_case_sensitive if not path or not os.path.exists(path): return islinux or isfreebsd From e94a44b1a3413e0bfe615a321d4fc4ad2befea6d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 09:36:31 -0600 Subject: [PATCH 05/13] Fix #5487 (Error after sending books to PRS-600) --- src/calibre/__init__.py | 2 +- src/calibre/devices/prs505/books.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 5039331660..c237e15dff 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -202,7 +202,7 @@ def extract(path, dir): def get_proxies(debug=True): proxies = getproxies() for key, proxy in list(proxies.items()): - if not proxy: + if not proxy or '..' in proxy: del proxies[key] continue if proxy.startswith(key+'://'): diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index ee5209c563..406a139fd1 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -284,7 +284,12 @@ class BookList(_BookList): plitems = [] for pl in self.playlists(): for c in pl.childNodes: - if hasattr(c, 'tagName') and c.tagName.endswith('item'): + if hasattr(c, 'tagName') and c.tagName.endswith('item') and \ + hasattr(c, 'getAttribute'): + try: + c.getAttribute('id') + except: # Unlinked node + continue plitems.append(c) return plitems From 67b0c32f64d45936d458eb28adff18e037f290a9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 10:34:20 -0600 Subject: [PATCH 06/13] SONY drivers: Fix regression that broke collection ordering by series when sending to device --- Changelog.yaml | 2 +- src/calibre/devices/prs505/books.py | 6 +++--- src/calibre/devices/prs505/driver.py | 8 ++++++++ src/calibre/devices/usbms/device.py | 5 ++++- src/calibre/library/__init__.py | 5 +++++ 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Changelog.yaml b/Changelog.yaml index 603ac97a40..ec083af809 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -5,7 +5,7 @@ # Also, each release can have new and improved recipes. - version: 0.6.52 - date: 2010-07-30 + date: 2010-05-07 new features: - title: "Support for the Kobo Reader and the HTC Desire" diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index 406a139fd1..cb6f4df7c5 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -390,9 +390,9 @@ class BookList(_BookList): continue db_ids = [i.getAttribute('id') for i in pl.childNodes if hasattr(i, 'getAttribute')] pl_book_ids = [getattr(self.book_by_id(i), 'db_id', None) for i in db_ids] - map = {} + imap = {} for i, j in zip(pl_book_ids, db_ids): - map[i] = j + imap[i] = j pl_book_ids = [i for i in pl_book_ids if i is not None] ordered_ids = [i for i in self.tag_order[title] if i in pl_book_ids] @@ -404,7 +404,7 @@ class BookList(_BookList): child.unlink() for id in ordered_ids: item = self.document.createElement(self.prefix+'item') - item.setAttribute('id', str(map[id])) + item.setAttribute('id', str(imap[id])) pl.appendChild(item) def fix_ids(main, carda, cardb): diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index e73a341909..f4fc4b0d29 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -121,6 +121,14 @@ class PRS505(CLI, Device): self.report_progress(1.0, _('Getting list of books on device...')) return bl + def filename_callback(self, fname, mi): + if getattr(mi, 'application_id', None) is not None: + base = fname.rpartition('.')[0] + suffix = '_%s'%mi.application_id + if not base.endswith(suffix): + fname = base + suffix + '.' + fname.rpartition('.')[-1] + return fname + def upload_books(self, files, names, on_card=None, end_session=True, metadata=None): diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index c84028d699..897baf82ca 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -825,7 +825,10 @@ class Device(DeviceConfig, DevicePlugin): from calibre.library.save_to_disk import get_components if not isinstance(template, unicode): template = template.decode('utf-8') - extra_components = get_components(template, mdata, fname) + app_id = str(getattr(mdata, 'application_id', '')) + # The SONY readers need to have the db id in the created filename + extra_components = get_components(template, mdata, fname, + length=250-len(app_id)-1) if not extra_components: extra_components.append(sanitize(self.filename_callback(fname, mdata))) diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py index 49a1107222..3c98db5e8a 100644 --- a/src/calibre/library/__init__.py +++ b/src/calibre/library/__init__.py @@ -26,3 +26,8 @@ def server_config(defaults=None): help=_('The maximum number of matches to return per OPDS query. ' 'This affects Stanza, WordPlayer, etc. integration.')) return c + +def db(): + from calibre.library.database2 import LibraryDatabase2 + from calibre.utils.config import prefs + return LibraryDatabase2(prefs['library_path']) From 8d4a5026203d53728c20f61e752d09c49e6e1c2c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 10:45:22 -0600 Subject: [PATCH 07/13] Improved recipe for Discover Magazine --- resources/recipes/discover_magazine.recipe | 55 +++++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/resources/recipes/discover_magazine.recipe b/resources/recipes/discover_magazine.recipe index cd4a078231..93a344bd2d 100644 --- a/resources/recipes/discover_magazine.recipe +++ b/resources/recipes/discover_magazine.recipe @@ -7,13 +7,15 @@ __docformat__ = 'restructuredtext en' discovermagazine.com ''' +import re from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class DiscoverMagazine(BasicNewsRecipe): title = u'Discover Magazine' description = u'Science, Technology and the Future' - __author__ = 'Mike Diaz' + __author__ = 'Starson17' language = 'en' oldest_article = 33 @@ -24,10 +26,49 @@ class DiscoverMagazine(BasicNewsRecipe): encoding = 'utf-8' extra_css = '.headline {font-size: x-large;} \n .fact {padding-top: 10pt}' - remove_tags = [dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}), + remove_tags = [ + dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}), + dict(name='div', attrs={'id':['footer','teaser','already-subscriber','teaser-suite','related-articles']}), + dict(name='div', attrs={'class':['column']}), dict(name='img', attrs={'src':'http://discovermagazine.com/onebyone.gif'})] - remove_tags_after = [dict(name='div', attrs={'class':'articlebody'})] + remove_tags_after = [dict(name='div', attrs={'class':'listingBar'})] + + def append_page(self, soup, appendtag, position): + pager = soup.find('span',attrs={'class':'next'}) + if pager: + nexturl = pager.a['href'] + print 'nexturl is: ', nexturl + soup2 = self.index_to_soup(nexturl) + texttag = soup2.find('div', attrs={'class':'articlebody'}) + newpos = len(texttag.contents) + self.append_page(soup2,texttag,newpos) + texttag.extract() + appendtag.insert(position,texttag) + + def preprocess_html(self, soup): + mtag = '\n' + print 'soup1 is: ', soup + soup.head.insert(0,mtag) + self.append_page(soup, soup.body, 3) + pager = soup.find('div',attrs={'class':'listingBar'}) + if pager: + pager.extract() + return soup + + def postprocess_html(self, soup, first_fetch): + for tag in soup.findAll(text=re.compile('^This article is a sample')): + tag.parent.extract() + for tag in soup.findAll(['table', 'tr', 'td']): + tag.name = 'div' + for tag in soup.findAll('div', attrs={'class':'discreet advert'}): + tag.extract() + for tag in soup.findAll('hr', attrs={'size':'1'}): + tag.extract() + for tag in soup.findAll('br'): + tag.extract() + print 'soup2 is: ', soup + return soup feeds = [ (u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'), @@ -38,10 +79,12 @@ class DiscoverMagazine(BasicNewsRecipe): (u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'), (u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'), (u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'), - (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'), (u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'), (u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'), (u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'), - (u'Stupid Science Word of the Month', u'http://discovermagazine.com/columns/stupid-science-word-of-the-month/rss.xml'), - (u'Science Not Fiction', u'http://blogs.discovermagazine.com/sciencenotfiction/wp-rss.php') + (u'What is This', u'http://discovermagazine.com/columns/what-is-this/rss.xml'), + (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'), + (u'Think Tech', u'http://discovermagazine.com/columns/think-tech/rss.xml'), + (u'Future Tech', u'http://discovermagazine.com/columns/future-tech/rss.xml'), + (u'Discover Interview', u'http://discovermagazine.com/columns/discover-interview/rss.xml'), ] \ No newline at end of file From 062f943b4c3df7a3e2bcea08a24470005c4d3326 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 11:00:24 -0600 Subject: [PATCH 08/13] ... --- resources/recipes/discover_magazine.recipe | 52 ++++++++++------------ 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/resources/recipes/discover_magazine.recipe b/resources/recipes/discover_magazine.recipe index 93a344bd2d..02cdb952b5 100644 --- a/resources/recipes/discover_magazine.recipe +++ b/resources/recipes/discover_magazine.recipe @@ -9,13 +9,12 @@ discovermagazine.com import re from calibre.web.feeds.news import BasicNewsRecipe -from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag class DiscoverMagazine(BasicNewsRecipe): title = u'Discover Magazine' - description = u'Science, Technology and the Future' - __author__ = 'Starson17' + description = u'Science, Technology and the Future' + __author__ = 'Starson17' language = 'en' oldest_article = 33 @@ -25,7 +24,7 @@ class DiscoverMagazine(BasicNewsRecipe): use_embedded_content = False encoding = 'utf-8' extra_css = '.headline {font-size: x-large;} \n .fact {padding-top: 10pt}' - + remove_tags = [ dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}), dict(name='div', attrs={'id':['footer','teaser','already-subscriber','teaser-suite','related-articles']}), @@ -33,29 +32,27 @@ class DiscoverMagazine(BasicNewsRecipe): dict(name='img', attrs={'src':'http://discovermagazine.com/onebyone.gif'})] remove_tags_after = [dict(name='div', attrs={'class':'listingBar'})] - + def append_page(self, soup, appendtag, position): pager = soup.find('span',attrs={'class':'next'}) if pager: nexturl = pager.a['href'] - print 'nexturl is: ', nexturl soup2 = self.index_to_soup(nexturl) texttag = soup2.find('div', attrs={'class':'articlebody'}) - newpos = len(texttag.contents) + newpos = len(texttag.contents) self.append_page(soup2,texttag,newpos) texttag.extract() appendtag.insert(position,texttag) - + def preprocess_html(self, soup): mtag = '\n' - print 'soup1 is: ', soup - soup.head.insert(0,mtag) + soup.head.insert(0,mtag) self.append_page(soup, soup.body, 3) pager = soup.find('div',attrs={'class':'listingBar'}) if pager: - pager.extract() + pager.extract() return soup - + def postprocess_html(self, soup, first_fetch): for tag in soup.findAll(text=re.compile('^This article is a sample')): tag.parent.extract() @@ -67,24 +64,23 @@ class DiscoverMagazine(BasicNewsRecipe): tag.extract() for tag in soup.findAll('br'): tag.extract() - print 'soup2 is: ', soup - return soup - + return soup + feeds = [ - (u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'), - (u'Health - Medicine', u'http://discovermagazine.com/topics/health-medicine/rss.xml'), - (u'Mind Brain', u'http://discovermagazine.com/topics/mind-brain/rss.xml'), - (u'Space', u'http://discovermagazine.com/topics/space/rss.xml'), - (u'Human Origins', u'http://discovermagazine.com/topics/human-origins/rss.xml'), - (u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'), - (u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'), - (u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'), - (u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'), - (u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'), - (u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'), + (u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'), + (u'Health - Medicine', u'http://discovermagazine.com/topics/health-medicine/rss.xml'), + (u'Mind Brain', u'http://discovermagazine.com/topics/mind-brain/rss.xml'), + (u'Space', u'http://discovermagazine.com/topics/space/rss.xml'), + (u'Human Origins', u'http://discovermagazine.com/topics/human-origins/rss.xml'), + (u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'), + (u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'), + (u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'), + (u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'), + (u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'), + (u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'), (u'What is This', u'http://discovermagazine.com/columns/what-is-this/rss.xml'), - (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'), + (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'), (u'Think Tech', u'http://discovermagazine.com/columns/think-tech/rss.xml'), (u'Future Tech', u'http://discovermagazine.com/columns/future-tech/rss.xml'), (u'Discover Interview', u'http://discovermagazine.com/columns/discover-interview/rss.xml'), - ] \ No newline at end of file + ] From 7eecfb06680bf44783d9c9d5f7b759dfac9125fa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 11:41:26 -0600 Subject: [PATCH 09/13] More robust creation of dynamic id filters --- src/calibre/library/database2.py | 4 +++- src/calibre/library/sqlite.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 724c1bd41a..a7d68896cf 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -106,6 +106,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn = connect(self.dbpath, self.row_factory) if self.user_version == 0: self.initialize_database() + # remember to add any filter to the connect method in sqlite.py as well + # so that various code taht connects directly will not complain about + # missing functions self.books_list_filter = self.conn.create_dynamic_filter('books_list_filter') def __init__(self, library_path, row_factory=False): @@ -1469,7 +1472,6 @@ books_series_link feeds conn = ndb.conn conn.execute('create table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)') conn.commit() - conn.create_function(self.books_list_filter.name, 1, lambda x: 1) conn.executescript(sql) conn.commit() conn.execute('pragma user_version=%d'%user_version) diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 755d8e64b4..236f81da2d 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -117,6 +117,8 @@ class DBThread(Thread): self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate) self.conn.create_function('title_sort', 1, title_sort) self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4())) + # Dummy functions for dynamically created filters + self.conn.create_function('books_list_filter', 1, lambda x: 1) def run(self): try: From 68a32562589ca664abd7fb824bbb9c79e1b90581 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 12:44:24 -0600 Subject: [PATCH 10/13] Add --reinitialize-db option to calibre-debug --- src/calibre/debug.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 58385f9dc6..9945add7c8 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -34,9 +34,40 @@ Run an embedded python interpreter. help='Add a simple plugin (i.e. a plugin that consists of only a ' '.py file), by specifying the path to the py file containing the ' 'plugin code.') + parser.add_option('--reinitialize-db', default=None, + help='Re-initialize the sqlite calibre database at the ' + 'specified path. Useful to recover from db corruption.') return parser +def reinit_db(dbpath): + if not os.path.exists(dbpath): + raise ValueError(dbpath + ' does not exist') + from calibre.library.sqlite import connect + from contextlib import closing + import shutil + conn = connect(dbpath, False) + uv = conn.get('PRAGMA user_version;', all=False) + conn.execute('PRAGMA writable_schema=ON') + conn.commit() + sql = conn.dump() + conn.close() + dest = dbpath + '.tmp' + try: + with closing(connect(dest, False)) as nconn: + nconn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)') + nconn.commit() + nconn.executescript(sql) + nconn.commit() + nconn.execute('pragma user_version=%d'%int(uv)) + nconn.commit() + os.remove(dbpath) + shutil.copyfile(dest, dbpath) + finally: + if os.path.exists(dest): + os.remove(dest) + prints('Database successfully re-initialized') + def migrate(old, new): from calibre.utils.config import prefs from calibre.library.database import LibraryDatabase @@ -122,6 +153,8 @@ def main(args=sys.argv): prints('CALIBRE_RESOURCES_PATH='+sys.resources_location) prints('CALIBRE_EXTENSIONS_PATH='+sys.extensions_location) prints('CALIBRE_PYTHON_PATH='+os.pathsep.join(sys.path)) + elif opts.reinitialize_db is not None: + reinit_db(opts.reinitialize_db) else: from calibre import ipython ipython() From 14adce229b026bafa9725536aa45906f51d513af Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 20:04:54 -0600 Subject: [PATCH 11/13] Clean up GUI initialization and add support for restoring corrupted databases automatically --- src/calibre/debug.py | 19 ++- src/calibre/gui2/__init__.py | 13 ++- src/calibre/gui2/main.py | 194 ++++++++++++++++++++++++++++--- src/calibre/gui2/ui.py | 63 +--------- src/calibre/library/database2.py | 2 +- src/calibre/library/sqlite.py | 2 +- 6 files changed, 208 insertions(+), 85 deletions(-) diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 9945add7c8..c84ce3dfcc 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -40,7 +40,7 @@ Run an embedded python interpreter. return parser -def reinit_db(dbpath): +def reinit_db(dbpath, callback=None): if not os.path.exists(dbpath): raise ValueError(dbpath + ' does not exist') from calibre.library.sqlite import connect @@ -50,15 +50,26 @@ def reinit_db(dbpath): uv = conn.get('PRAGMA user_version;', all=False) conn.execute('PRAGMA writable_schema=ON') conn.commit() - sql = conn.dump() + sql_lines = conn.dump() conn.close() dest = dbpath + '.tmp' try: with closing(connect(dest, False)) as nconn: nconn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)') nconn.commit() - nconn.executescript(sql) - nconn.commit() + if callable(callback): + callback(len(sql_lines), True) + for i, line in enumerate(sql_lines): + try: + nconn.execute(line) + except: + import traceback + prints('SQL line %r failed with error:'%line) + prints(traceback.format_exc()) + continue + finally: + if callable(callback): + callback(i, False) nconn.execute('pragma user_version=%d'%int(uv)) nconn.commit() os.remove(dbpath) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 5aad257711..94f969d7e9 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -410,6 +410,7 @@ class FileDialog(QObject): modal = True, name = '', mode = QFileDialog.ExistingFiles, + default_dir='~' ): QObject.__init__(self) ftext = '' @@ -428,9 +429,10 @@ class FileDialog(QObject): self.selected_files = None self.fd = None - initial_dir = dynamic.get(self.dialog_name, os.path.expanduser('~')) + initial_dir = dynamic.get(self.dialog_name, + os.path.expanduser(default_dir)) if not isinstance(initial_dir, basestring): - initial_dir = os.path.expanduser('~') + initial_dir = os.path.expanduser(default_dir) self.selected_files = [] if mode == QFileDialog.AnyFile: f = unicode(QFileDialog.getSaveFileName(parent, title, initial_dir, ftext, "")) @@ -465,9 +467,10 @@ class FileDialog(QObject): return tuple(self.selected_files) -def choose_dir(window, name, title): - fd = FileDialog(title, [], False, window, name=name, - mode=QFileDialog.DirectoryOnly) +def choose_dir(window, name, title, default_dir='~'): + fd = FileDialog(title=title, filters=[], add_all_files_filter=False, + parent=window, name=name, mode=QFileDialog.DirectoryOnly, + default_dir=default_dir) dir = fd.get_files() if dir: return dir[0] diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index b71f6e6922..c261c38dcf 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -4,15 +4,18 @@ __copyright__ = '2008, Kovid Goyal ' import sys, os, time, socket, traceback from functools import partial -from PyQt4.Qt import QCoreApplication, QIcon, QMessageBox +from PyQt4.Qt import QCoreApplication, QIcon, QMessageBox, QObject, QTimer, \ + QThread, pyqtSignal, Qt, QProgressDialog, QString -from calibre import prints -from calibre.constants import iswindows, __appname__, isosx +from calibre import prints, plugins +from calibre.constants import iswindows, __appname__, isosx, filesystem_encoding from calibre.utils.ipc import ADDRESS, RC from calibre.gui2 import ORG_NAME, APP_UID, initialize_file_icon_provider, \ - Application + Application, choose_dir, error_dialog, question_dialog from calibre.gui2.main_window import option_parser as _option_parser from calibre.utils.config import prefs, dynamic +from calibre.library.database2 import LibraryDatabase2 +from calibre.library.sqlite import sqlite, DatabaseException def option_parser(): parser = _option_parser('''\ @@ -48,25 +51,186 @@ def init_qt(args): app.setWindowIcon(QIcon(I('library.png'))) return app, opts, args, actions +def get_library_path(): + library_path = prefs['library_path'] + if library_path is None: # Need to migrate to new database layout + base = os.path.expanduser('~') + if iswindows: + base = plugins['winutil'][0].special_folder_path( + plugins['winutil'][0].CSIDL_PERSONAL) + if not base or not os.path.exists(base): + from PyQt4.Qt import QDir + base = unicode(QDir.homePath()).replace('/', os.sep) + candidate = choose_dir(None, 'choose calibre library', + _('Choose a location for your calibre e-book library'), + default_dir=base) + if not candidate: + candidate = os.path.join(base, 'Calibre Library') + library_path = os.path.abspath(candidate) + if not os.path.exists(library_path): + try: + os.makedirs(library_path) + except: + error_dialog(None, _('Failed to create library'), + _('Failed to create calibre library at: %r. Aborting.')%library_path, + det_msg = traceback.print_exc(), show=True) + library_path = None + return library_path + +class DBRepair(QThread): + + repair_done = pyqtSignal(object, object) + progress = pyqtSignal(object, object) + + def __init__(self, library_path, parent, pd): + QThread.__init__(self, parent) + self.library_path = library_path + self.pd = pd + self.progress.connect(self._callback, type=Qt.QueuedConnection) + + def _callback(self, num, is_length): + if is_length: + self.pd.setRange(0, num-1) + num = 0 + self.pd.setValue(num) + + def callback(self, num, is_length): + self.progress.emit(num, is_length) + + def run(self): + from calibre.debug import reinit_db + try: + reinit_db(os.path.join(self.library_path, 'metadata.db'), + self.callback) + db = LibraryDatabase2(self.library_path) + tb = None + except: + db, tb = None, traceback.format_exc() + self.repair_done.emit(db, tb) + +class GuiRunner(QObject): + '''Make sure an event loop is running before starting the main work of + initialization''' + + def __init__(self, opts, args, actions, listener, app): + self.opts, self.args, self.listener, self.app = opts, args, listener, app + self.actions = actions + self.main = None + QObject.__init__(self) + self.timer = QTimer.singleShot(1, self.initialize) + + def start_gui(self): + from calibre.gui2.ui import Main + main = Main(self.library_path, self.db, self.listener, self.opts, self.actions) + add_filesystem_book = partial(main.add_filesystem_book, allow_device=False) + sys.excepthook = main.unhandled_exception + if len(self.args) > 1: + p = os.path.abspath(self.args[1]) + add_filesystem_book(p) + self.app.file_event_hook = add_filesystem_book + self.main = main + + def initialization_failed(self): + print 'Catastrophic failure initializing GUI, bailing out...' + QCoreApplication.exit(1) + raise SystemExit(1) + + def initialize_db_stage2(self, db, tb): + repair_pd = getattr(self, 'repair_pd', None) + if repair_pd is not None: + repair_pd.cancel() + + if db is None and tb is not None: + # DB Repair failed + error_dialog(None, _('Repairing failed'), + _('The database repair failed. Starting with ' + 'a new empty library.'), + det_msg=tb, show=True) + if db is None: + fname = _('Calibre Library') + if isinstance(fname, unicode): + try: + fname = fname.encode(filesystem_encoding) + except: + fname = 'Calibre Library' + x = os.path.expanduser('~'+os.sep+fname) + if not os.path.exists(x): + try: + os.makedirs(x) + except: + x = os.path.expanduser('~') + candidate = choose_dir(None, 'choose calibre library', + _('Choose a location for your new calibre e-book library'), + default_dir=x) + + if not candidate: + self.initialization_failed() + + try: + self.library_path = candidate + db = LibraryDatabase2(candidate) + except: + error_dialog(None, _('Bad database location'), + _('Bad database location %r. calibre will now quit.' + )%self.library_path, + det_msg=traceback.format_exc(), show=True) + self.initialization_failed() + + self.db = db + self.start_gui() + + def initialize_db(self): + db = None + try: + db = LibraryDatabase2(self.library_path) + except (sqlite.Error, DatabaseException): + repair = question_dialog(None, _('Corrupted database'), + _('Your calibre database appears to be corrupted. Do ' + 'you want calibre to try and repair it automatically? ' + 'If you say No, a new empty calibre library will be created.'), + det_msg=traceback.format_exc() + ) + if repair: + self.repair_pd = QProgressDialog(_('Repairing database. This ' + 'can take a very long time for a large collection'), QString(), + 0, 0) + self.repair_pd.setWindowModality(Qt.WindowModal) + self.repair_pd.show() + + self.repair = DBRepair(self.library_path, self, self.repair_pd) + self.repair.repair_done.connect(self.initialize_db_stage2, + type=Qt.QueuedConnection) + self.repair.start() + return + except: + error_dialog(None, _('Bad database location'), + _('Bad database location %r. Will start with ' + ' a new, empty calibre library')%self.library_path, + det_msg=traceback.format_exc(), show=True) + + self.initialize_db_stage2(db, None) + + def initialize(self, *args): + self.library_path = get_library_path() + if self.library_path is None: + self.initialization_failed() + + self.initialize_db() + + + def run_gui(opts, args, actions, listener, app): - from calibre.gui2.ui import Main initialize_file_icon_provider() if not dynamic.get('welcome_wizard_was_run', False): from calibre.gui2.wizard import wizard wizard().exec_() dynamic.set('welcome_wizard_was_run', True) - main = Main(listener, opts, actions) - add_filesystem_book = partial(main.add_filesystem_book, allow_device=False) - sys.excepthook = main.unhandled_exception - if len(args) > 1: - args[1] = os.path.abspath(args[1]) - add_filesystem_book(args[1]) - app.file_event_hook = add_filesystem_book + runner = GuiRunner(opts, args, actions, listener, app) ret = app.exec_() - if getattr(main, 'run_wizard_b4_shutdown', False): + if getattr(runner.main, 'run_wizard_b4_shutdown', False): from calibre.gui2.wizard import wizard wizard().exec_() - if getattr(main, 'restart_after_quit', False): + if getattr(runner.main, 'restart_after_quit', False): e = sys.executable if getattr(sys, 'frozen', False) else sys.argv[0] print 'Restarting with:', e, sys.argv if hasattr(sys, 'frameworks_dir'): @@ -78,7 +242,7 @@ def run_gui(opts, args, actions, listener, app): else: if iswindows: try: - main.system_tray_icon.hide() + runner.main.system_tray_icon.hide() except: pass return ret diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 62178c4fc4..bf520f62e7 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -14,9 +14,9 @@ from xml.parsers.expat import ExpatError from Queue import Queue, Empty from threading import Thread from functools import partial -from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \ +from PyQt4.Qt import Qt, SIGNAL, QObject, QUrl, QTimer, \ QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \ - QToolButton, QDialog, QDesktopServices, QFileDialog, \ + QToolButton, QDialog, QDesktopServices, \ QSystemTrayIcon, QApplication, QKeySequence, QAction, \ QMessageBox, QStackedLayout, QHelpEvent, QInputDialog,\ QThread, pyqtSignal @@ -125,8 +125,9 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.default_thumbnail = (pixmap.width(), pixmap.height(), pixmap_to_data(pixmap)) - def __init__(self, listener, opts, actions, parent=None): + def __init__(self, library_path, db, listener, opts, actions, parent=None): self.preferences_action, self.quit_action = actions + self.library_path = library_path self.spare_servers = [] MainWindow.__init__(self, opts, parent) # Initialize fontconfig in a separate thread as this can be a lengthy @@ -513,31 +514,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): if self.system_tray_icon.isVisible() and opts.start_in_tray: self.hide_windows() self.stack.setCurrentIndex(0) - try: - db = LibraryDatabase2(self.library_path) - except Exception: - import traceback - error_dialog(self, _('Bad database location'), - _('Bad database location')+':'+self.library_path, - det_msg=traceback.format_exc()).exec_() - fname = _('Calibre Library') - if isinstance(fname, unicode): - try: - fname = fname.encode(filesystem_encoding) - except: - fname = 'Calibre Library' - x = os.path.expanduser('~'+os.sep+fname) - if not os.path.exists(x): - os.makedirs(x) - dir = unicode(QFileDialog.getExistingDirectory(self, - _('Choose a location for your ebook library.'), - x)) - if not dir: - QCoreApplication.exit(1) - raise SystemExit(1) - else: - self.library_path = dir - db = LibraryDatabase2(self.library_path) self.library_view.set_database(db) prefs['library_path'] = self.library_path self.library_view.sortByColumn(*dynamic.get('sort_column', @@ -2330,38 +2306,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): d.show() self._modeless_dialogs.append(d) - - def initialize_database(self): - self.library_path = prefs['library_path'] - if self.library_path is None: # Need to migrate to new database layout - base = os.path.expanduser('~') - if iswindows: - from calibre import plugins - from PyQt4.Qt import QDir - base = plugins['winutil'][0].special_folder_path( - plugins['winutil'][0].CSIDL_PERSONAL) - if not base or not os.path.exists(base): - base = unicode(QDir.homePath()).replace('/', os.sep) - dir = unicode(QFileDialog.getExistingDirectory(self, - _('Choose a location for your ebook library.'), base)) - if not dir: - dir = os.path.expanduser('~/Library') - self.library_path = os.path.abspath(dir) - if not os.path.exists(self.library_path): - try: - os.makedirs(self.library_path) - except: - self.library_path = os.path.expanduser('~/CalibreLibrary') - error_dialog(self, _('Invalid library location'), - _('Could not access %s. Using %s as the library.')% - (repr(self.library_path), repr(self.library_path)) - ).exec_() - if not os.path.exists(self.library_path): - os.makedirs(self.library_path) - - def read_settings(self): - self.initialize_database() geometry = config['main_window_geometry'] if geometry is not None: self.restoreGeometry(geometry) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index f375b3345a..8606a91cca 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1399,7 +1399,7 @@ books_series_link feeds def check_integrity(self, callback): callback(0., _('Checking SQL integrity...')) user_version = self.user_version - sql = self.conn.dump() + sql = '\n'.join(self.conn.dump()) self.conn.close() dest = self.dbpath+'.tmp' if os.path.exists(dest): diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 9718cab872..1e937499fb 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -116,7 +116,7 @@ class DBThread(Thread): break if func == 'dump': try: - ok, res = True, '\n'.join(self.conn.iterdump()) + ok, res = True, tuple(self.conn.iterdump()) except Exception, err: ok, res = False, (err, traceback.format_exc()) else: From c97464fd22e5be12c5ef61beea1d5932c0c466fa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 20:07:19 -0600 Subject: [PATCH 12/13] Kobo driver: Show all sideloaded content on the device. Fixes #5492 (Updated kobo driver) --- src/calibre/devices/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 534e944415..6cbe8aadec 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -47,5 +47,5 @@ class KOBO(USBMS): VENDOR_NAME = 'KOBO_INC' WINDOWS_MAIN_MEM = '.KOBOEREADER' - EBOOK_DIR_MAIN = 'e-books' + EBOOK_DIR_MAIN = '' From 2660fcae0fd9f9a2b6d2c50d8c57e142d866ca3f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 May 2010 20:18:31 -0600 Subject: [PATCH 13/13] Interpret a Keyboard interrupt (Ctrl+C) as a request to quit the main GUI --- src/calibre/gui2/main_window.py | 7 ++++++- src/calibre/gui2/ui.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/main_window.py b/src/calibre/gui2/main_window.py index 3b0e995308..2779a18733 100644 --- a/src/calibre/gui2/main_window.py +++ b/src/calibre/gui2/main_window.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' import StringIO, traceback, sys from PyQt4.Qt import QMainWindow, QString, Qt, QFont, QCoreApplication, SIGNAL,\ - QAction, QMenu, QMenuBar, QIcon + QAction, QMenu, QMenuBar, QIcon, pyqtSignal from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog from calibre.utils.config import OptionParser from calibre.gui2 import error_dialog @@ -41,6 +41,8 @@ class MainWindow(QMainWindow): ___menu = None __actions = [] + keyboard_interrupt = pyqtSignal() + @classmethod def create_application_menubar(cls): mb = QMenuBar(None) @@ -76,6 +78,9 @@ class MainWindow(QMainWindow): print 'Received signal:', repr(signal) def unhandled_exception(self, type, value, tb): + if type == KeyboardInterrupt: + self.keyboard_interrupt.emit() + return try: sio = StringIO.StringIO() traceback.print_exception(type, value, tb, file=sio) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index bf520f62e7..99ab5470dc 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -634,6 +634,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): height = v.rowHeight(0) self.library_view.verticalHeader().setDefaultSectionSize(height) + self.keyboard_interrupt.connect(self.quit, type=Qt.QueuedConnection) + def resizeEvent(self, ev): MainWindow.resizeEvent(self, ev)