Merge from custcol trunk

Changelog.yaml

@@ -5,7 +5,7 @@
 # Also, each release can have new and improved recipes.
 
 - version: 0.6.52
-  date: 2010-07-30
+  date: 2010-05-07
 
   new features:
     - title: "Support for the Kobo Reader and the HTC Desire"

resources/recipes/discover_magazine.recipe

@@ -7,13 +7,14 @@ __docformat__ = 'restructuredtext en'
 discovermagazine.com
 '''
 
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class DiscoverMagazine(BasicNewsRecipe):
 
     title = u'Discover Magazine'
-    description = u'Science, Technology and the Future' 
+    description = u'Science, Technology and the Future'
-    __author__ = 'Mike Diaz' 
+    __author__ = 'Starson17'
     language = 'en'
 
     oldest_article = 33
@@ -23,25 +24,63 @@ class DiscoverMagazine(BasicNewsRecipe):
     use_embedded_content  = False
     encoding = 'utf-8'
     extra_css = '.headline {font-size: x-large;} \n .fact {padding-top: 10pt}'
-    
+
-    remove_tags = [dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}),
+    remove_tags = [
+                   dict(name='div', attrs={'id':['searchModule', 'mainMenu', 'tool-box']}),
+                   dict(name='div', attrs={'id':['footer','teaser','already-subscriber','teaser-suite','related-articles']}),
+                   dict(name='div', attrs={'class':['column']}),
                    dict(name='img', attrs={'src':'http://discovermagazine.com/onebyone.gif'})]
 
-    remove_tags_after = [dict(name='div', attrs={'class':'articlebody'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'listingBar'})]
- 
+
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('span',attrs={'class':'next'})
+        if pager:
+           nexturl = pager.a['href']
+           soup2 = self.index_to_soup(nexturl)
+           texttag = soup2.find('div', attrs={'class':'articlebody'})
+           newpos = len(texttag.contents)
+           self.append_page(soup2,texttag,newpos)
+           texttag.extract()
+           appendtag.insert(position,texttag)
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Language" content="en-US"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
+        soup.head.insert(0,mtag)
+        self.append_page(soup, soup.body, 3)
+        pager = soup.find('div',attrs={'class':'listingBar'})
+        if pager:
+           pager.extract()
+        return soup
+
+    def postprocess_html(self, soup, first_fetch):
+        for tag in soup.findAll(text=re.compile('^This article is a sample')):
+            tag.parent.extract()
+        for tag in soup.findAll(['table', 'tr', 'td']):
+            tag.name = 'div'
+        for tag in soup.findAll('div', attrs={'class':'discreet advert'}):
+            tag.extract()
+        for tag in soup.findAll('hr', attrs={'size':'1'}):
+            tag.extract()
+        for tag in soup.findAll('br'):
+            tag.extract()
+        return soup
+
     feeds = [
-             (u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'), 
+             (u'Technology', u'http://discovermagazine.com/topics/technology/rss.xml'),
-             (u'Health - Medicine', u'http://discovermagazine.com/topics/health-medicine/rss.xml'), 
+             (u'Health - Medicine', u'http://discovermagazine.com/topics/health-medicine/rss.xml'),
-             (u'Mind Brain', u'http://discovermagazine.com/topics/mind-brain/rss.xml'), 
+             (u'Mind Brain', u'http://discovermagazine.com/topics/mind-brain/rss.xml'),
-             (u'Space', u'http://discovermagazine.com/topics/space/rss.xml'), 
+             (u'Space', u'http://discovermagazine.com/topics/space/rss.xml'),
-             (u'Human Origins', u'http://discovermagazine.com/topics/human-origins/rss.xml'), 
+             (u'Human Origins', u'http://discovermagazine.com/topics/human-origins/rss.xml'),
-             (u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'), 
+             (u'Living World', u'http://discovermagazine.com/topics/living-world/rss.xml'),
-             (u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'), 
+             (u'Environment', u'http://discovermagazine.com/topics/environment/rss.xml'),
-             (u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'), 
+             (u'Physics & Math', u'http://discovermagazine.com/topics/physics-math/rss.xml'),
-             (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'), 
+             (u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'),
-             (u"20 Things you didn't know about...", u'http://discovermagazine.com/columns/20-things-you-didnt-know/rss.xml'), 
+             (u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'),
-             (u'Fuzzy Math', u'http://discovermagazine.com/columns/fuzzy-math/rss.xml'), 
+             (u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'),
-             (u'The Brain', u'http://discovermagazine.com/columns/the-brain/rss.xml'), 
+             (u'What is This', u'http://discovermagazine.com/columns/what-is-this/rss.xml'),
-             (u'Stupid Science Word of the Month', u'http://discovermagazine.com/columns/stupid-science-word-of-the-month/rss.xml'), 
+             (u'Vital Signs', u'http://discovermagazine.com/columns/vital-signs/rss.xml'),
-             (u'Science Not Fiction', u'http://blogs.discovermagazine.com/sciencenotfiction/wp-rss.php')
+             (u'Think Tech', u'http://discovermagazine.com/columns/think-tech/rss.xml'),
-            ]
+             (u'Future Tech', u'http://discovermagazine.com/columns/future-tech/rss.xml'),
+             (u'Discover Interview', u'http://discovermagazine.com/columns/discover-interview/rss.xml'),
+            ]

src/calibre/__init__.py

@@ -4,6 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 import sys, os, re, logging, time, mimetypes, \
        __builtin__, warnings, multiprocessing
+from urllib import getproxies
 __builtin__.__dict__['dynamic_property'] = lambda(func): func(None)
 from htmlentitydefs import name2codepoint
 from math import floor
@@ -199,71 +200,54 @@ def extract(path, dir):
     extractor(path, dir)
 
 def get_proxies(debug=True):
-    proxies = {}
+    proxies = getproxies()
+    for key, proxy in list(proxies.items()):
+        if not proxy or '..' in proxy:
+            del proxies[key]
+            continue
+        if proxy.startswith(key+'://'):
+            proxy = proxy[len(key)+3:]
+        if proxy.endswith('/'):
+            proxy = proxy[:-1]
+        if len(proxy) > 4:
+            proxies[key] = proxy
+        else:
+            prints('Removing invalid', key, 'proxy:', proxy)
+            del proxies[key]
 
-    for q in ('http', 'ftp'):
-        proxy =  os.environ.get(q+'_proxy', None)
-        if not proxy: continue
-        if proxy.startswith(q+'://'):
-            proxy = proxy[7:]
-        proxies[q] = proxy
-
-    if iswindows:
-        try:
-            winreg = __import__('_winreg')
-            settings = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
-                                      'Software\\Microsoft\\Windows'
-                                      '\\CurrentVersion\\Internet Settings')
-            proxy = winreg.QueryValueEx(settings, "ProxyEnable")[0]
-            if proxy:
-                server = str(winreg.QueryValueEx(settings, 'ProxyServer')[0])
-                if ';' in server:
-                    for p in server.split(';'):
-                        protocol, address = p.split('=')
-                        proxies[protocol] = address
-                else:
-                    proxies['http'] = server
-                    proxies['ftp'] =  server
-            settings.Close()
-        except Exception, e:
-            prints('Unable to detect proxy settings: %s' % str(e))
-    for x in list(proxies):
-        if len(proxies[x]) < 5:
-            prints('Removing invalid', x, 'proxy:', proxies[x])
-            del proxies[x]
     if proxies and debug:
         prints('Using proxies:', proxies)
     return proxies
 
 def get_parsed_proxy(typ='http', debug=True):
     proxies = get_proxies(debug)
-    if typ not in proxies:
+    proxy = proxies.get(typ, None)
-        return
+    if proxy:
-    pattern = re.compile((
+        pattern = re.compile((
-        '(?:ptype://)?' \
+            '(?:ptype://)?' \
-        '(?:(?P<user>\w+):(?P<pass>.*)@)?' \
+            '(?:(?P<user>\w+):(?P<pass>.*)@)?' \
-        '(?P<host>[\w\-\.]+)' \
+            '(?P<host>[\w\-\.]+)' \
-        '(?::(?P<port>\d+))?').replace('ptype', typ)
+            '(?::(?P<port>\d+))?').replace('ptype', typ)
-    )
+        )
 
-    match = pattern.match(proxies['typ'])
+        match = pattern.match(proxies[typ])
-    if match:
+        if match:
-        try:
+            try:
-            ans = {
+                ans = {
-                    'host' : match.group('host'),
+                        'host' : match.group('host'),
-                    'port' : match.group('port'),
+                        'port' : match.group('port'),
-                    'user' : match.group('user'),
+                        'user' : match.group('user'),
-                    'pass' : match.group('pass')
+                        'pass' : match.group('pass')
-                }
+                    }
-            if ans['port']:
+                if ans['port']:
-                ans['port'] = int(ans['port'])
+                    ans['port'] = int(ans['port'])
-        except:
+            except:
-            if debug:
+                if debug:
-                traceback.print_exc()
+                    traceback.print_exc()
-            return
+            else:
-        if debug:
+                if debug:
-            prints('Using http proxy', ans)
+                    prints('Using http proxy', str(ans))
-        return ans
+                return ans
 
 
 def browser(honor_time=True, max_time=2, mobile_browser=False):

src/calibre/debug.py

@@ -34,9 +34,51 @@ Run an embedded python interpreter.
             help='Add a simple plugin (i.e. a plugin that consists of only a '
             '.py file), by specifying the path to the py file containing the '
             'plugin code.')
+    parser.add_option('--reinitialize-db', default=None,
+            help='Re-initialize the sqlite calibre database at the '
+            'specified path. Useful to recover from db corruption.')
 
     return parser
 
+def reinit_db(dbpath, callback=None):
+    if not os.path.exists(dbpath):
+        raise ValueError(dbpath + ' does not exist')
+    from calibre.library.sqlite import connect
+    from contextlib import closing
+    import shutil
+    conn = connect(dbpath, False)
+    uv = conn.get('PRAGMA user_version;', all=False)
+    conn.execute('PRAGMA writable_schema=ON')
+    conn.commit()
+    sql_lines = conn.dump()
+    conn.close()
+    dest = dbpath + '.tmp'
+    try:
+        with closing(connect(dest, False)) as nconn:
+            nconn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
+            nconn.commit()
+            if callable(callback):
+                callback(len(sql_lines), True)
+            for i, line in enumerate(sql_lines):
+                try:
+                    nconn.execute(line)
+                except:
+                    import traceback
+                    prints('SQL line %r failed with error:'%line)
+                    prints(traceback.format_exc())
+                    continue
+                finally:
+                    if callable(callback):
+                        callback(i, False)
+            nconn.execute('pragma user_version=%d'%int(uv))
+            nconn.commit()
+        os.remove(dbpath)
+        shutil.copyfile(dest, dbpath)
+    finally:
+        if os.path.exists(dest):
+            os.remove(dest)
+    prints('Database successfully re-initialized')
+
 def migrate(old, new):
     from calibre.utils.config import prefs
     from calibre.library.database import LibraryDatabase
@@ -122,6 +164,8 @@ def main(args=sys.argv):
         prints('CALIBRE_RESOURCES_PATH='+sys.resources_location)
         prints('CALIBRE_EXTENSIONS_PATH='+sys.extensions_location)
         prints('CALIBRE_PYTHON_PATH='+os.pathsep.join(sys.path))
+    elif opts.reinitialize_db is not None:
+        reinit_db(opts.reinitialize_db)
     else:
         from calibre import ipython
         ipython()

src/calibre/devices/misc.py

@@ -47,5 +47,5 @@ class KOBO(USBMS):
     VENDOR_NAME = 'KOBO_INC'
     WINDOWS_MAIN_MEM = '.KOBOEREADER'
 
-    EBOOK_DIR_MAIN = 'e-books'
+    EBOOK_DIR_MAIN = ''
 

src/calibre/devices/prs505/books.py

@@ -284,7 +284,12 @@ class BookList(_BookList):
         plitems = []
         for pl in self.playlists():
             for c in pl.childNodes:
-                if hasattr(c, 'tagName')  and c.tagName.endswith('item'):
+                if hasattr(c, 'tagName') and c.tagName.endswith('item') and \
+                    hasattr(c, 'getAttribute'):
+                    try:
+                        c.getAttribute('id')
+                    except: # Unlinked node
+                        continue
                     plitems.append(c)
         return plitems
 
@@ -385,9 +390,9 @@ class BookList(_BookList):
                 continue
             db_ids = [i.getAttribute('id') for i in pl.childNodes if hasattr(i, 'getAttribute')]
             pl_book_ids = [getattr(self.book_by_id(i), 'db_id', None) for i in db_ids]
-            map = {}
+            imap = {}
             for i, j in zip(pl_book_ids, db_ids):
-                map[i] = j
+                imap[i] = j
             pl_book_ids = [i for i in pl_book_ids if i is not None]
             ordered_ids = [i for i in self.tag_order[title] if i in pl_book_ids]
 
@@ -399,7 +404,7 @@ class BookList(_BookList):
                 child.unlink()
             for id in ordered_ids:
                 item = self.document.createElement(self.prefix+'item')
-                item.setAttribute('id', str(map[id]))
+                item.setAttribute('id', str(imap[id]))
                 pl.appendChild(item)
 
 def fix_ids(main, carda, cardb):

src/calibre/devices/prs505/driver.py

@@ -121,6 +121,14 @@ class PRS505(CLI, Device):
         self.report_progress(1.0, _('Getting list of books on device...'))
         return bl
 
+    def filename_callback(self, fname, mi):
+        if getattr(mi, 'application_id', None) is not None:
+            base = fname.rpartition('.')[0]
+            suffix = '_%s'%mi.application_id
+            if not base.endswith(suffix):
+                fname = base + suffix + '.' + fname.rpartition('.')[-1]
+        return fname
+
     def upload_books(self, files, names, on_card=None, end_session=True,
                      metadata=None):
 

src/calibre/devices/usbms/device.py

@@ -825,7 +825,10 @@ class Device(DeviceConfig, DevicePlugin):
         from calibre.library.save_to_disk import get_components
         if not isinstance(template, unicode):
             template = template.decode('utf-8')
-        extra_components = get_components(template, mdata, fname)
+        app_id = str(getattr(mdata, 'application_id', ''))
+        # The SONY readers need to have the db id in the created filename
+        extra_components = get_components(template, mdata, fname,
+                length=250-len(app_id)-1)
         if not extra_components:
             extra_components.append(sanitize(self.filename_callback(fname,
                 mdata)))

src/calibre/ebooks/comic/input.py

@@ -341,8 +341,15 @@ class ComicInput(InputFormatPlugin):
             if not os.path.exists('comics.txt'):
                 raise ValueError('%s is not a valid comic collection'
                         %stream.name)
-            raw = open('comics.txt', 'rb').read().decode('utf-8')
+            raw = open('comics.txt', 'rb').read()
-            raw.lstrip(unicode(codecs.BOM_UTF8, "utf8" ))
+            if raw.startswith(codecs.BOM_UTF16_BE):
+                raw = raw.decode('utf-16-be')[1:]
+            elif raw.startswith(codecs.BOM_UTF16_LE):
+                raw = raw.decode('utf-16-le')[1:]
+            elif raw.startswith(codecs.BOM_UTF8):
+                raw = raw.decode('utf-8')[1:]
+            else:
+                raw = raw.decode('utf-8')
             for line in raw.splitlines():
                 line = line.strip()
                 if not line:

src/calibre/ebooks/html/input.py

@@ -292,7 +292,7 @@ class HTMLInput(InputFormatPlugin):
                 encoding=opts.input_encoding)
 
     def is_case_sensitive(self, path):
-        if self._is_case_sensitive is not None:
+        if getattr(self, '_is_case_sensitive', None) is not None:
             return self._is_case_sensitive
         if not path or not os.path.exists(path):
             return islinux or isfreebsd

src/calibre/ebooks/metadata/ereader.py

@@ -8,6 +8,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import re
 import struct
 
 from calibre.ebooks.metadata import MetaInformation
@@ -44,12 +45,12 @@ def get_metadata(stream, extract_cover=True):
         if hr.compression in (2, 10) and hr.has_metadata == 1:
             try:
                 mdata = pheader.section_data(hr.metadata_offset)
-
+    
                 mdata = mdata.split('\x00')
-                mi.title = mdata[0]
+                mi.title = re.sub(r'[^a-zA-Z0-9 \._=\+\-!\?,\'\"]', '', mdata[0])
-                mi.authors = [mdata[1]]
+                mi.authors = [re.sub(r'[^a-zA-Z0-9 \._=\+\-!\?,\'\"]', '', mdata[1])]
-                mi.publisher = mdata[3]
+                mi.publisher = re.sub(r'[^a-zA-Z0-9 \._=\+\-!\?,\'\"]', '', mdata[3])
-                mi.isbn = mdata[4]
+                mi.isbn = re.sub(r'[^a-zA-Z0-9 \._=\+\-!\?,\'\"]', '', mdata[4])
             except:
                 pass
 

src/calibre/gui2/__init__.py

@@ -420,6 +420,7 @@ class FileDialog(QObject):
                        modal = True,
                        name = '',
                        mode = QFileDialog.ExistingFiles,
+                       default_dir='~'
                        ):
         QObject.__init__(self)
         ftext = ''
@@ -438,9 +439,10 @@ class FileDialog(QObject):
         self.selected_files = None
         self.fd = None
 
-        initial_dir = dynamic.get(self.dialog_name, os.path.expanduser('~'))
+        initial_dir = dynamic.get(self.dialog_name,
+                os.path.expanduser(default_dir))
         if not isinstance(initial_dir, basestring):
-            initial_dir = os.path.expanduser('~')
+            initial_dir = os.path.expanduser(default_dir)
         self.selected_files = []
         if mode == QFileDialog.AnyFile:
             f = unicode(QFileDialog.getSaveFileName(parent, title, initial_dir, ftext, ""))
@@ -475,9 +477,10 @@ class FileDialog(QObject):
         return tuple(self.selected_files)
 
 
-def choose_dir(window, name, title):
+def choose_dir(window, name, title, default_dir='~'):
-    fd = FileDialog(title, [], False, window, name=name,
+    fd = FileDialog(title=title, filters=[], add_all_files_filter=False,
-                    mode=QFileDialog.DirectoryOnly)
+            parent=window, name=name, mode=QFileDialog.DirectoryOnly,
+            default_dir=default_dir)
     dir = fd.get_files()
     if dir:
         return dir[0]

src/calibre/gui2/main.py

@@ -4,15 +4,18 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import sys, os, time, socket, traceback
 from functools import partial
 
-from PyQt4.Qt import QCoreApplication, QIcon, QMessageBox
+from PyQt4.Qt import QCoreApplication, QIcon, QMessageBox, QObject, QTimer, \
+        QThread, pyqtSignal, Qt, QProgressDialog, QString
 
-from calibre import prints
+from calibre import prints, plugins
-from calibre.constants import iswindows, __appname__, isosx
+from calibre.constants import iswindows, __appname__, isosx, filesystem_encoding
 from calibre.utils.ipc import ADDRESS, RC
 from calibre.gui2 import ORG_NAME, APP_UID, initialize_file_icon_provider, \
-    Application
+    Application, choose_dir, error_dialog, question_dialog
 from calibre.gui2.main_window import option_parser as _option_parser
 from calibre.utils.config import prefs, dynamic
+from calibre.library.database2 import LibraryDatabase2
+from calibre.library.sqlite import sqlite, DatabaseException
 
 def option_parser():
     parser = _option_parser('''\
@@ -48,25 +51,186 @@ def init_qt(args):
     app.setWindowIcon(QIcon(I('library.png')))
     return app, opts, args, actions
 
+def get_library_path():
+    library_path = prefs['library_path']
+    if library_path is None: # Need to migrate to new database layout
+        base = os.path.expanduser('~')
+        if iswindows:
+            base = plugins['winutil'][0].special_folder_path(
+                    plugins['winutil'][0].CSIDL_PERSONAL)
+            if not base or not os.path.exists(base):
+                from PyQt4.Qt import QDir
+                base = unicode(QDir.homePath()).replace('/', os.sep)
+        candidate = choose_dir(None, 'choose calibre library',
+                _('Choose a location for your calibre e-book library'),
+                default_dir=base)
+        if not candidate:
+            candidate = os.path.join(base, 'Calibre Library')
+        library_path = os.path.abspath(candidate)
+    if not os.path.exists(library_path):
+        try:
+            os.makedirs(library_path)
+        except:
+            error_dialog(None, _('Failed to create library'),
+                    _('Failed to create calibre library at: %r. Aborting.')%library_path,
+                    det_msg = traceback.print_exc(), show=True)
+            library_path = None
+    return library_path
+
+class DBRepair(QThread):
+
+    repair_done = pyqtSignal(object, object)
+    progress = pyqtSignal(object, object)
+
+    def __init__(self, library_path, parent, pd):
+        QThread.__init__(self, parent)
+        self.library_path = library_path
+        self.pd = pd
+        self.progress.connect(self._callback, type=Qt.QueuedConnection)
+
+    def _callback(self, num, is_length):
+        if is_length:
+            self.pd.setRange(0, num-1)
+            num = 0
+        self.pd.setValue(num)
+
+    def callback(self, num, is_length):
+        self.progress.emit(num, is_length)
+
+    def run(self):
+        from calibre.debug import reinit_db
+        try:
+            reinit_db(os.path.join(self.library_path, 'metadata.db'),
+                    self.callback)
+            db = LibraryDatabase2(self.library_path)
+            tb = None
+        except:
+            db, tb = None, traceback.format_exc()
+        self.repair_done.emit(db, tb)
+
+class GuiRunner(QObject):
+    '''Make sure an event loop is running before starting the main work of
+    initialization'''
+
+    def __init__(self, opts, args, actions, listener, app):
+        self.opts, self.args, self.listener, self.app = opts, args, listener, app
+        self.actions = actions
+        self.main = None
+        QObject.__init__(self)
+        self.timer = QTimer.singleShot(1, self.initialize)
+
+    def start_gui(self):
+        from calibre.gui2.ui import Main
+        main = Main(self.library_path, self.db, self.listener, self.opts, self.actions)
+        add_filesystem_book = partial(main.add_filesystem_book, allow_device=False)
+        sys.excepthook = main.unhandled_exception
+        if len(self.args) > 1:
+            p = os.path.abspath(self.args[1])
+            add_filesystem_book(p)
+        self.app.file_event_hook = add_filesystem_book
+        self.main = main
+
+    def initialization_failed(self):
+        print 'Catastrophic failure initializing GUI, bailing out...'
+        QCoreApplication.exit(1)
+        raise SystemExit(1)
+
+    def initialize_db_stage2(self, db, tb):
+        repair_pd = getattr(self, 'repair_pd', None)
+        if repair_pd is not None:
+            repair_pd.cancel()
+
+        if db is None and tb is not None:
+            # DB Repair failed
+            error_dialog(None, _('Repairing failed'),
+                    _('The database repair failed. Starting with '
+                        'a new empty library.'),
+                    det_msg=tb, show=True)
+        if db is None:
+            fname = _('Calibre Library')
+            if isinstance(fname, unicode):
+                try:
+                    fname = fname.encode(filesystem_encoding)
+                except:
+                    fname = 'Calibre Library'
+            x = os.path.expanduser('~'+os.sep+fname)
+            if not os.path.exists(x):
+                try:
+                    os.makedirs(x)
+                except:
+                    x = os.path.expanduser('~')
+            candidate = choose_dir(None, 'choose calibre library',
+                _('Choose a location for your new calibre e-book library'),
+                default_dir=x)
+
+            if not candidate:
+                self.initialization_failed()
+
+            try:
+                self.library_path = candidate
+                db = LibraryDatabase2(candidate)
+            except:
+                error_dialog(None, _('Bad database location'),
+                    _('Bad database location %r. calibre will now quit.'
+                     )%self.library_path,
+                    det_msg=traceback.format_exc(), show=True)
+                self.initialization_failed()
+
+        self.db = db
+        self.start_gui()
+
+    def initialize_db(self):
+        db = None
+        try:
+            db = LibraryDatabase2(self.library_path)
+        except (sqlite.Error, DatabaseException):
+            repair = question_dialog(None, _('Corrupted database'),
+                    _('Your calibre database appears to be corrupted. Do '
+                    'you want calibre to try and repair it automatically? '
+                    'If you say No, a new empty calibre library will be created.'),
+                    det_msg=traceback.format_exc()
+                    )
+            if repair:
+                self.repair_pd = QProgressDialog(_('Repairing database. This '
+                    'can take a very long time for a large collection'), QString(),
+                    0, 0)
+                self.repair_pd.setWindowModality(Qt.WindowModal)
+                self.repair_pd.show()
+
+                self.repair = DBRepair(self.library_path, self, self.repair_pd)
+                self.repair.repair_done.connect(self.initialize_db_stage2,
+                        type=Qt.QueuedConnection)
+                self.repair.start()
+                return
+        except:
+            error_dialog(None, _('Bad database location'),
+                    _('Bad database location %r. Will start with '
+                    ' a new, empty calibre library')%self.library_path,
+                    det_msg=traceback.format_exc(), show=True)
+
+        self.initialize_db_stage2(db, None)
+
+    def initialize(self, *args):
+        self.library_path = get_library_path()
+        if self.library_path is None:
+            self.initialization_failed()
+
+        self.initialize_db()
+
+
+
 def run_gui(opts, args, actions, listener, app):
-    from calibre.gui2.ui import Main
     initialize_file_icon_provider()
     if not dynamic.get('welcome_wizard_was_run', False):
         from calibre.gui2.wizard import wizard
         wizard().exec_()
         dynamic.set('welcome_wizard_was_run', True)
-    main = Main(listener, opts, actions)
+    runner = GuiRunner(opts, args, actions, listener, app)
-    add_filesystem_book = partial(main.add_filesystem_book, allow_device=False)
-    sys.excepthook = main.unhandled_exception
-    if len(args) > 1:
-        args[1] = os.path.abspath(args[1])
-        add_filesystem_book(args[1])
-    app.file_event_hook = add_filesystem_book
     ret = app.exec_()
-    if getattr(main, 'run_wizard_b4_shutdown', False):
+    if getattr(runner.main, 'run_wizard_b4_shutdown', False):
         from calibre.gui2.wizard import wizard
         wizard().exec_()
-    if getattr(main, 'restart_after_quit', False):
+    if getattr(runner.main, 'restart_after_quit', False):
         e = sys.executable if getattr(sys, 'frozen', False) else sys.argv[0]
         print 'Restarting with:', e, sys.argv
         if hasattr(sys, 'frameworks_dir'):
@@ -78,7 +242,7 @@ def run_gui(opts, args, actions, listener, app):
     else:
         if iswindows:
             try:
-                main.system_tray_icon.hide()
+                runner.main.system_tray_icon.hide()
             except:
                 pass
     return ret

src/calibre/gui2/main_window.py

@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import StringIO, traceback, sys
 
 from PyQt4.Qt import QMainWindow, QString, Qt, QFont, QCoreApplication, SIGNAL,\
-                     QAction, QMenu, QMenuBar, QIcon
+                     QAction, QMenu, QMenuBar, QIcon, pyqtSignal
 from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog
 from calibre.utils.config import OptionParser
 from calibre.gui2 import error_dialog
@@ -41,6 +41,8 @@ class MainWindow(QMainWindow):
     ___menu     = None
     __actions   = []
 
+    keyboard_interrupt = pyqtSignal()
+
     @classmethod
     def create_application_menubar(cls):
         mb = QMenuBar(None)
@@ -76,6 +78,9 @@ class MainWindow(QMainWindow):
         print 'Received signal:', repr(signal)
 
     def unhandled_exception(self, type, value, tb):
+        if type == KeyboardInterrupt:
+            self.keyboard_interrupt.emit()
+            return
         try:
             sio = StringIO.StringIO()
             traceback.print_exception(type, value, tb, file=sio)

src/calibre/gui2/ui.py

@@ -14,9 +14,9 @@ from xml.parsers.expat import ExpatError
 from Queue import Queue, Empty
 from threading import Thread
 from functools import partial
-from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
+from PyQt4.Qt import Qt, SIGNAL, QObject, QUrl, QTimer, \
                      QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \
-                     QToolButton, QDialog, QDesktopServices, QFileDialog, \
+                     QToolButton, QDialog, QDesktopServices, \
                      QSystemTrayIcon, QApplication, QKeySequence, QAction, \
                      QMessageBox, QStackedLayout, QHelpEvent, QInputDialog,\
                      QThread, pyqtSignal
@@ -125,9 +125,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.default_thumbnail = (pixmap.width(), pixmap.height(),
                 pixmap_to_data(pixmap))
 
-    def __init__(self, listener, opts, actions, parent=None):
+        self.last_time = datetime.datetime.now()
+    def __init__(self, library_path, db, listener, opts, actions, parent=None):
         self.last_time = datetime.datetime.now()
         self.preferences_action, self.quit_action = actions
+        self.library_path = library_path
         self.spare_servers = []
         self.must_restart_before_config = False
         MainWindow.__init__(self, opts, parent)
@@ -518,31 +520,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         if self.system_tray_icon.isVisible() and opts.start_in_tray:
             self.hide_windows()
         self.stack.setCurrentIndex(0)
-        try:
-            db = LibraryDatabase2(self.library_path)
-        except Exception:
-            import traceback
-            error_dialog(self, _('Bad database location'),
-                    _('Bad database location')+':'+self.library_path,
-                    det_msg=traceback.format_exc()).exec_()
-            fname = _('Calibre Library')
-            if isinstance(fname, unicode):
-                try:
-                    fname = fname.encode(filesystem_encoding)
-                except:
-                    fname = 'Calibre Library'
-            x = os.path.expanduser('~'+os.sep+fname)
-            if not os.path.exists(x):
-                os.makedirs(x)
-            dir = unicode(QFileDialog.getExistingDirectory(self,
-                            _('Choose a location for your ebook library.'),
-                            x))
-            if not dir:
-                QCoreApplication.exit(1)
-                raise SystemExit(1)
-            else:
-                self.library_path = dir
-                db = LibraryDatabase2(self.library_path)
         self.library_view.set_database(db)
         prefs['library_path'] = self.library_path
         self.library_view.restore_sort_at_startup(dynamic.get('sort_history', [('timestamp', Qt.DescendingOrder)]))
@@ -673,6 +650,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             v.resizeRowToContents(0)
             height = v.rowHeight(0)
             self.library_view.verticalHeader().setDefaultSectionSize(height)
+        self.keyboard_interrupt.connect(self.quit, type=Qt.QueuedConnection)
 
     def do_edit_categories(self):
         d = TagCategories(self, self.library_view.model().db)
@@ -2379,38 +2357,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         d.show()
         self._modeless_dialogs.append(d)
 
-
-    def initialize_database(self):
-        self.library_path = prefs['library_path']
-        if self.library_path is None: # Need to migrate to new database layout
-            base = os.path.expanduser('~')
-            if iswindows:
-                from calibre import plugins
-                from PyQt4.Qt import QDir
-                base = plugins['winutil'][0].special_folder_path(
-                        plugins['winutil'][0].CSIDL_PERSONAL)
-                if not base or not os.path.exists(base):
-                    base = unicode(QDir.homePath()).replace('/', os.sep)
-            dir = unicode(QFileDialog.getExistingDirectory(self,
-                        _('Choose a location for your ebook library.'), base))
-            if not dir:
-                dir = os.path.expanduser('~/Library')
-            self.library_path = os.path.abspath(dir)
-        if not os.path.exists(self.library_path):
-            try:
-                os.makedirs(self.library_path)
-            except:
-                self.library_path = os.path.expanduser('~/CalibreLibrary')
-                error_dialog(self, _('Invalid library location'),
-                     _('Could not access %s. Using %s as the library.')%
-                     (repr(self.library_path), repr(self.library_path))
-                             ).exec_()
-                if not os.path.exists(self.library_path):
-                    os.makedirs(self.library_path)
-
-
     def read_settings(self):
-        self.initialize_database()
         geometry = config['main_window_geometry']
         if geometry is not None:
             self.restoreGeometry(geometry)

src/calibre/library/__init__.py

@@ -26,3 +26,8 @@ def server_config(defaults=None):
             help=_('The maximum number of matches to return per OPDS query. '
             'This affects Stanza, WordPlayer, etc. integration.'))
     return c
+
+def db():
+    from calibre.library.database2 import LibraryDatabase2
+    from calibre.utils.config import prefs
+    return LibraryDatabase2(prefs['library_path'])

src/calibre/library/database2.py

@@ -106,6 +106,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.conn = connect(self.dbpath, self.row_factory)
         if self.user_version == 0:
             self.initialize_database()
+        # remember to add any filter to the connect method in sqlite.py as well
+        # so that various code taht connects directly will not complain about
+        # missing functions
         self.books_list_filter = self.conn.create_dynamic_filter('books_list_filter')
 
     def __init__(self, library_path, row_factory=False):
@@ -1457,7 +1460,7 @@ books_series_link      feeds
     def check_integrity(self, callback):
         callback(0., _('Checking SQL integrity...'))
         user_version = self.user_version
-        sql = self.conn.dump()
+        sql = '\n'.join(self.conn.dump())
         self.conn.close()
         dest = self.dbpath+'.tmp'
         if os.path.exists(dest):
@@ -1469,7 +1472,6 @@ books_series_link      feeds
             conn = ndb.conn
             conn.execute('create table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
             conn.commit()
-            conn.create_function(self.books_list_filter.name, 1, lambda x: 1)
             conn.executescript(sql)
             conn.commit()
             conn.execute('pragma user_version=%d'%user_version)

src/calibre/library/sqlite.py

@@ -117,6 +117,8 @@ class DBThread(Thread):
         self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
         self.conn.create_function('title_sort', 1, title_sort)
         self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
+        # Dummy functions for dynamically created filters
+        self.conn.create_function('books_list_filter', 1, lambda x: 1)
 
     def run(self):
         try:
@@ -128,7 +130,7 @@ class DBThread(Thread):
                     break
                 if func == 'dump':
                     try:
-                        ok, res = True, '\n'.join(self.conn.iterdump())
+                        ok, res = True, tuple(self.conn.iterdump())
                     except Exception, err:
                         ok, res = False, (err, traceback.format_exc())
                 elif func == 'create_dynamic_filter':