Merge from trunk

resources/recipes/winnipeg_free_press.recipe

@@ -0,0 +1,30 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class WinnipegFreePress(BasicNewsRecipe):
+    title          = u'Winnipeg Free Press'
+    __author__            = 'buyo'
+    description           = 'News from Winnipeg, Manitoba, Canada'
+    oldest_article = 1
+    max_articles_per_feed = 15
+    category              = 'News, Winnipeg, Canada'
+    cover_url             = 'http://media.winnipegfreepress.com/designimages/winnipegfreepress_WFP.gif'
+    no_stylesheets        = True
+    encoding              = 'UTF-8'
+    remove_javascript     = True
+    use_embedded_content  = False
+    language = 'en_CA'
+
+    feeds          = [(u'Breaking News', u'http://www.winnipegfreepress.com/rss?path=/breakingnews'),
+          (u'Local News',u'http://www.winnipegfreepress.com/rss?path=/local'),
+          (u'Breaking Business News',u'http://www.winnipegfreepress.com/rss?path=/business/finance'),
+          (u'Business',u'http://www.winnipegfreepress.com/rss?path=/business'),
+          (u'Editorials',u'http://www.winnipegfreepress.com/rss?path=/opinion/editorials'),
+          (u'Views from the West',u'http://www.winnipegfreepress.com/rss?path=/opinion/westview'),
+          (u'Life & Style',u'http://www.winnipegfreepress.com/rss?path=/life'),
+          (u'Food & Drink',u'http://www.winnipegfreepress.com/rss?path=/life/food')
+         ]
+
+    keep_only_tags = [
+              dict(name='div', attrs={'id':'article_header'}),
+                              dict(name='div', attrs={'class':'article'}),
+                              ]

src/calibre/devices/android/driver.py

@@ -41,7 +41,7 @@ class ANDROID(USBMS):
             0x502 : { 0x3203 : [0x0100]},
 
             # Dell
-            0x413c : { 0xb007 : [0x0100]},
+            0x413c : { 0xb007 : [0x0100, 0x0224]},
 
             # Eken?
             0x040d : { 0x0851 : [0x0001]},

src/calibre/devices/jetbook/driver.py

@@ -50,6 +50,8 @@ class JETBOOK(USBMS):
 
     def filename_callback(self, fname, mi):
         fileext = os.path.splitext(os.path.basename(fname))[1]
+        if fileext.lower() not in ('txt', 'pdf', 'fb2'):
+            return fname
         title = mi.title if mi.title else 'Unknown'
         title = title.replace(' ', '_')
         au = mi.format_authors()

src/calibre/devices/kobo/books.py

@@ -23,7 +23,7 @@ class Book(MetaInformation):
         'uuid', 'device_collections',
     ]
 
-    def __init__(self, prefix, lpath, title, authors, mime, date, ContentType, thumbnail_name, other=None):
+    def __init__(self, prefix, lpath, title, authors, mime, date, ContentType, thumbnail_name, size=None, other=None):
 
         MetaInformation.__init__(self, '')
         self.device_collections = []
@@ -42,10 +42,8 @@ class Book(MetaInformation):
         else:
             self.authors = [authors]
         self.mime = mime
-        try:
+
-            self.size = os.path.getsize(self.path)
+        self.size = size # will be set later if None
-        except OSError:
-            self.size = 0
         try:
             if ContentType == '6':
                 self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")

src/calibre/devices/kobo/driver.py

@@ -94,19 +94,19 @@ class KOBO(USBMS):
 
                 idx = bl_cache.get(lpath, None)
                 if idx is not None:
+                    bl_cache[lpath] = None
                     if ImageID is not None:
                         imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - NickelBookCover.parsed')
                         #print "Image name Normalized: " + imagename
                         if imagename is not None:
                             bl[idx].thumbnail = ImageWrapper(imagename)
-                    bl_cache[lpath] = None
                     if ContentType != '6':
                         if self.update_metadata_item(bl[idx]):
                             # print 'update_metadata_item returned true'
                             changed = True
                     bl[idx].device_collections = playlist_map.get(lpath, [])
                 else:
-                    book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
+                    book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
                     # print 'Update booklist'
                     if bl.add_book(book, replace_metadata=False):
                         changed = True
@@ -316,10 +316,10 @@ class KOBO(USBMS):
                 lpath = lpath[1:]
             #print "path: " + lpath
             #book = self.book_class(prefix, lpath, other=info)
-            lpath = self.normalize_path(prefix + lpath)
             book = Book(prefix, lpath, '', '', '', '', '', '', other=info)
             if book.size is None:
                 book.size = os.stat(self.normalize_path(path)).st_size
+            book._new_book = True # Must be before add_book
             booklists[blist].add_book(book, replace_metadata=True)
         self.report_progress(1.0, _('Adding books to device metadata listing...'))
 
@@ -380,3 +380,19 @@ class KOBO(USBMS):
 
         return USBMS.get_file(self, path, *args, **kwargs)
 
+    @classmethod
+    def book_from_path(cls, prefix, lpath, title, authors, mime, date, ContentType, ImageID):
+        from calibre.ebooks.metadata import MetaInformation
+
+        if cls.settings().read_metadata or cls.MUST_READ_METADATA:
+            mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, lpath)))
+        else:
+            from calibre.ebooks.metadata.meta import metadata_from_filename
+            mi = metadata_from_filename(cls.normalize_path(os.path.basename(lpath)),
+                                        cls.build_template_regexp())
+        if mi is None:
+            mi = MetaInformation(os.path.splitext(os.path.basename(lpath))[0],
+                    [_('Unknown')])
+        size = os.stat(cls.normalize_path(os.path.join(prefix, lpath))).st_size
+        book =  Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=size, other=mi)
+        return book

src/calibre/gui2/actions/choose_library.py

@@ -5,15 +5,16 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
+import os, shutil
 from functools import partial
 
-from PyQt4.Qt import QMenu, Qt
+from PyQt4.Qt import QMenu, Qt, QInputDialog
 
 from calibre import isbytestring
 from calibre.constants import filesystem_encoding
 from calibre.utils.config import prefs
-from calibre.gui2 import gprefs, warning_dialog, Dispatcher
+from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \
+    question_dialog
 from calibre.gui2.actions import InterfaceAction
 
 class LibraryUsageStats(object):
@@ -66,6 +67,13 @@ class LibraryUsageStats(object):
             loc = loc[:-1]
         return loc.split('/')[-1]
 
+    def rename(self, location, newloc):
+        newloc = self.canonicalize_path(newloc)
+        stats = self.stats.pop(location, None)
+        if stats is not None:
+            self.stats[newloc] = stats
+        self.write_stats()
+
 
 class ChooseLibraryAction(InterfaceAction):
 
@@ -80,7 +88,7 @@ class ChooseLibraryAction(InterfaceAction):
                 type=Qt.QueuedConnection)
 
         self.stats = LibraryUsageStats()
-        self.create_action(spec=(_('Switch to library...'), 'lt.png', None,
+        self.create_action(spec=(_('Switch/create library...'), 'lt.png', None,
             None), attr='action_choose')
         self.action_choose.triggered.connect(self.choose_library,
                 type=Qt.QueuedConnection)
@@ -90,7 +98,13 @@ class ChooseLibraryAction(InterfaceAction):
 
         self.quick_menu = QMenu(_('Quick switch'))
         self.quick_menu_action = self.choose_menu.addMenu(self.quick_menu)
-        self.qs_separator = self.choose_menu.addSeparator()
+        self.rename_menu = QMenu(_('Rename library'))
+        self.rename_menu_action = self.choose_menu.addMenu(self.rename_menu)
+        self.delete_menu = QMenu(_('Delete library'))
+        self.delete_menu_action = self.choose_menu.addMenu(self.delete_menu)
+
+        self.rename_separator = self.choose_menu.addSeparator()
+
         self.switch_actions = []
         for i in range(5):
             ac = self.create_action(spec=('', None, None, None),
@@ -123,9 +137,15 @@ class ChooseLibraryAction(InterfaceAction):
             ac.setVisible(False)
         self.quick_menu.clear()
         self.qs_locations = [i[1] for i in locations]
+        self.rename_menu.clear()
+        self.delete_menu.clear()
         for name, loc in locations:
             self.quick_menu.addAction(name, Dispatcher(partial(self.switch_requested,
                 loc)))
+            self.rename_menu.addAction(name, Dispatcher(partial(self.rename_requested,
+                name, loc)))
+            self.delete_menu.addAction(name, Dispatcher(partial(self.delete_requested,
+                name, loc)))
 
         for i, x in enumerate(locations[:len(self.switch_actions)]):
             name, loc = x
@@ -134,12 +154,50 @@ class ChooseLibraryAction(InterfaceAction):
             ac.setVisible(True)
 
         self.quick_menu_action.setVisible(bool(locations))
+        self.rename_menu_action.setVisible(bool(locations))
+        self.delete_menu_action.setVisible(bool(locations))
 
 
     def location_selected(self, loc):
         enabled = loc == 'library'
         self.qaction.setEnabled(enabled)
 
+    def rename_requested(self, name, location):
+        loc = location.replace('/', os.sep)
+        base = os.path.dirname(loc)
+        newname, ok = QInputDialog.getText(self.gui, _('Rename') + ' ' + name,
+                '<p>'+_('Choose a new name for the library <b>%s</b>. ')%name +
+                '<p>'+_('Note that the actual library folder will be renamed.'),
+                text=name)
+        newname = unicode(newname)
+        if not ok or not newname or newname == name:
+            return
+        newloc = os.path.join(base, newname)
+        if os.path.exists(newloc):
+            return error_dialog(self.gui, _('Already exists'),
+                    _('The folder %s already exists. Delete it first.') %
+                    newloc, show=True)
+        os.rename(loc, newloc)
+        self.stats.rename(location, newloc)
+        self.build_menus()
+
+    def delete_requested(self, name, location):
+        loc = location.replace('/', os.sep)
+        if not question_dialog(self.gui, _('Are you sure?'), '<p>'+
+                _('All files from %s will be '
+                '<b>permanently deleted</b>. Are you sure?') % loc,
+                show_copy_button=False):
+            return
+        exists = self.gui.library_view.model().db.exists_at(loc)
+        if exists:
+            try:
+                shutil.rmtree(loc, ignore_errors=True)
+            except:
+                pass
+        self.stats.remove(location)
+        self.build_menus()
+
+
     def switch_requested(self, location):
         if not self.change_library_allowed():
             return

src/calibre/gui2/actions/copy_to_library.py

@@ -5,6 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import os
 from functools import partial
 from threading import Thread
 
@@ -13,6 +14,7 @@ from PyQt4.Qt import QMenu, QToolButton
 from calibre.gui2.actions import InterfaceAction
 from calibre.gui2 import error_dialog, Dispatcher
 from calibre.gui2.dialogs.progress import ProgressDialog
+from calibre.utils.config import prefs
 
 class Worker(Thread):
 
@@ -38,6 +40,13 @@ class Worker(Thread):
 
         self.done()
 
+    def add_formats(self, id, paths, newdb, replace=True):
+        for path in paths:
+            fmt = os.path.splitext(path)[-1].replace('.', '').upper()
+            with open(path, 'rb') as f:
+                newdb.add_format(id, fmt, f, index_is_id=True,
+                        notify=False, replace=replace)
+
     def doit(self):
         from calibre.library.database2 import LibraryDatabase2
         newdb = LibraryDatabase2(self.loc)
@@ -49,12 +58,18 @@ class Worker(Thread):
             else: fmts = fmts.split(',')
             paths = [self.db.format_abspath(x, fmt, index_is_id=True) for fmt in
                     fmts]
-            newdb.import_book(mi, paths, notify=False, import_hooks=False)
+            added = False
-            co = self.db.conversion_options(x, 'PIPE')
+            if prefs['add_formats_to_existing']:
-            if co is not None:
+                identical_book_list = newdb.find_identical_books(mi)
-                newdb.set_conversion_options(x, 'PIPE', co)
+                if identical_book_list: # books with same author and nearly same title exist in newdb
-
+                    added = True
-
+                    for identical_book in identical_book_list:
+                        self.add_formats(identical_book, paths, newdb, replace=False)
+            if not added:
+                newdb.import_book(mi, paths, notify=False, import_hooks=False)
+                co = self.db.conversion_options(x, 'PIPE')
+                if co is not None:
+                    newdb.set_conversion_options(x, 'PIPE', co)
 
 
 class CopyToLibraryAction(InterfaceAction):

src/calibre/gui2/add.py

@@ -1,7 +1,7 @@
 '''
 UI for adding books to the database and saving books to disk
 '''
-import os, shutil, time, re
+import os, shutil, time
 from Queue import Queue, Empty
 from threading import Thread
 
@@ -94,14 +94,6 @@ class DBAdder(Thread): # {{{
         self.daemon = True
         self.input_queue = Queue()
         self.output_queue = Queue()
-        self.fuzzy_title_patterns = [(re.compile(pat), repl) for pat, repl in
-                [
-                    (r'[\[\](){}<>\'";,:#]', ''),
-                    (r'^(the|a|an) ', ''),
-                    (r'[-._]', ' '),
-                    (r'\s+', ' ')
-                ]
-        ]
         self.merged_books = set([])
 
     def run(self):
@@ -138,33 +130,6 @@ class DBAdder(Thread): # {{{
                 fmts[-1] = fmt
         return fmts
 
-    def fuzzy_title(self, title):
-        title = title.strip().lower()
-        for pat, repl in self.fuzzy_title_patterns:
-            title = pat.sub(repl, title)
-        return title
-
-    def find_identical_books(self, mi):
-        identical_book_ids = set([])
-        if mi.authors:
-            try:
-                query = u' and '.join([u'author:"=%s"'%(a.replace('"', '')) for a in
-                    mi.authors])
-            except ValueError:
-                return identical_book_ids
-            try:
-                book_ids = self.db.data.parse(query)
-            except:
-                import traceback
-                traceback.print_exc()
-                return identical_book_ids
-            for book_id in book_ids:
-                fbook_title = self.db.title(book_id, index_is_id=True)
-                fbook_title = self.fuzzy_title(fbook_title)
-                mbook_title = self.fuzzy_title(mi.title)
-                if fbook_title == mbook_title:
-                    identical_book_ids.add(book_id)
-        return identical_book_ids
 
     def add(self, id, opf, cover, name):
         formats = self.ids.pop(id)
@@ -191,7 +156,7 @@ class DBAdder(Thread): # {{{
             orig_formats = formats
             formats = [f for f in formats if not f.lower().endswith('.opf')]
             if prefs['add_formats_to_existing']:
-                identical_book_list = self.find_identical_books(mi)
+                identical_book_list = self.db.find_identical_books(mi)
 
                 if identical_book_list: # books with same author and nearly same title exist in db
                     self.merged_books.add(mi.title)

src/calibre/library/database2.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 '''
 The database used to store ebook metadata
 '''
-import os, sys, shutil, cStringIO, glob, time, functools, traceback
+import os, sys, shutil, cStringIO, glob, time, functools, traceback, re
 from itertools import repeat
 from math import floor
 
@@ -550,6 +550,43 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             return bool(self.conn.get('SELECT id FROM books where title=?', (title,), all=False))
         return False
 
+    def find_identical_books(self, mi):
+        fuzzy_title_patterns = [(re.compile(pat), repl) for pat, repl in
+                [
+                    (r'[\[\](){}<>\'";,:#]', ''),
+                    (r'^(the|a|an) ', ''),
+                    (r'[-._]', ' '),
+                    (r'\s+', ' ')
+                ]
+        ]
+
+        def fuzzy_title(title):
+            title = title.strip().lower()
+            for pat, repl in fuzzy_title_patterns:
+                title = pat.sub(repl, title)
+            return title
+
+        identical_book_ids = set([])
+        if mi.authors:
+            try:
+                query = u' and '.join([u'author:"=%s"'%(a.replace('"', '')) for a in
+                    mi.authors])
+            except ValueError:
+                return identical_book_ids
+            try:
+                book_ids = self.data.parse(query)
+            except:
+                import traceback
+                traceback.print_exc()
+                return identical_book_ids
+            for book_id in book_ids:
+                fbook_title = self.title(book_id, index_is_id=True)
+                fbook_title = fuzzy_title(fbook_title)
+                mbook_title = fuzzy_title(mi.title)
+                if fbook_title == mbook_title:
+                    identical_book_ids.add(book_id)
+        return identical_book_ids
+
     def has_cover(self, index, index_is_id=False):
         id = index if  index_is_id else self.id(index)
         path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')

src/calibre/utils/smtp.py

@@ -81,12 +81,7 @@ def sendmail(msg, from_, to, localhost=None, verbose=0, timeout=30,
         for x in to:
             return sendmail_direct(from_, x, msg, timeout, localhost, verbose)
     import smtplib
-    class SMTP_SSL(smtplib.SMTP_SSL): # Workaround for bug in smtplib.py
+    cls = smtplib.SMTP if encryption == 'TLS' else smtplib.SMTP_SSL
-        def _get_socket(self, host, port, timeout):
-            smtplib.SMTP_SSL._get_socket(self, host, port, timeout)
-            return self.sock
-
-    cls = smtplib.SMTP if encryption == 'TLS' else SMTP_SSL
     timeout = None # Non-blocking sockets sometimes don't work
     port = int(port)
     s = cls(timeout=timeout, local_hostname=localhost)