Pull from trunk

Changelog.yaml

@@ -4,6 +4,83 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+- version: 0.7.18
+  date: 2010-09-10
+
+  new features:
+    - title: "All new Preferences dialog, with nicer layout and the ability to restore settings to defaults"
+      type: major
+
+    - title: "Add series info when available to generated cover. Also auto-resize the logo on the cover to ensure all text fits"
+      tickets: [6724]
+
+    - title: "On device column: Now indicates when multiple copies of the same book are present on the device"
+
+    - title: "Driver for the Gemei GM2000"
+
+    - title: "Extract fb2 files from zip container when adding to calibre. Can be disables by disabling the Archive Extract file type plugin."
+      tickets: [6739]
+
+    - title: "Switch to using raster icons for a small speedup in startup time"
+
+  bug fixes:
+    - title: "On device column: Fix matching bug when multiple books in the library have the same title and author"
+
+    - title: "Content server: Use /mobile version for Kindle browser"
+
+    - title: "E-book viewer: When adding a bookmark, a default name is generated"
+      tickets: [6450]
+
+    - title: "Hide visible menus before clearing toolbar."
+      tickets: [6706]
+
+    - title: "Batch conversion: Don't overwrite the insert page break before setting"
+      tickets: [6729]
+
+    - title: "Catalog generation: Fixed improper title display in catalog when title contains ':'. Added 'ondevice' field to CSV/XML catalog output (only with connected device|folder|iTunes). Added optional 'Series' section to generated catalogs with hyperlinks between books and series. Tweaks to catalog formatting."
+
+    - title: "Fix regression when checking database integrity with custom columns introduced in 0.7.17"
+
+    - title: "Sending email: Ignore geenric records when trying to resolve domain"
+      tickets: [6723]
+
+    - title: "Fix a bug where the open state of the splitter was not being saved on shutdown if the splitter had been closed at startup and was opened by dragging the center line"
+
+    - title: "MOBI Output: Fix bug generating index when chapter names contained non ASCII characters"
+      tickets: [6595]
+
+    - title: "PDF Input: Fix handling of more non ascii characters"
+
+    - title: "Content server: Triple AJAX timeout for main book list to 30 seconds"
+
+    - title: "Use ImageMagick instead of Qt to generate thumbnails when sending covers to device. Should fix corrupted nook covers on some windows installs"
+
+    - title: "FB2 Output: Improve creation of sections and fix a couple of bugs that could result in invalid output"
+
+  new recipes:
+    - title: "Journal Gazette"
+      author: cynvision
+
+    - title: "Milenio"
+      author: bmsleight
+
+    - title: "Winnipeg Free Press"
+      author: buyo
+
+    - title: "Buckmasters in the kitchen, The Walrus Magazine and Kansas City Star"
+      author: Tony Stegall
+
+    - title: "Europa Sur"
+      author: "Darko Miletic"
+
+
+  improved recipes:
+    - El Pais
+    - La Jornada
+    - nrcnext
+    - WSJ (free)
+
+
 - version: 0.7.17
   date: 2010-09-03
 

resources/recipes/taz.recipe

@@ -13,50 +13,51 @@ from calibre.web.feeds.news import BasicNewsRecipe
 
 class TazDigiabo(BasicNewsRecipe):
 
-	title = u'Taz Digiabo'
+    title = u'Taz Digiabo'
-	description = u'Das EPUB DigiAbo der Taz'
+    description = u'Das EPUB DigiAbo der Taz'
-	language = 'de'
+    language = 'de'
-	lang = 'de-DE'
+    lang = 'de-DE'
 
-	__author__ = 'Lars Jacob'
+    __author__ = 'Lars Jacob'
-	needs_subscription = True
+    needs_subscription = True
 
-	conversion_options = {
+    conversion_options = {
-		'no_default_epub_cover' : True
+        'no_default_epub_cover' : True
-	}
+    }
 
-	def build_index(self):
+    def build_index(self):
-		if self.username is not None and self.password is not None:
+        if self.username is not None and self.password is not None:
-			domain = "http://www.taz.de"
+            domain = "http://www.taz.de"
 
-			url = domain + "/epub/"
+            url = domain + "/epub/"
 
-			auth_handler = urllib2.HTTPBasicAuthHandler()
+            auth_handler = urllib2.HTTPBasicAuthHandler()
-			auth_handler.add_password(realm='TAZ-ABO',
+            auth_handler.add_password(realm='TAZ-ABO',
-									  uri=url,
+                                      uri=url,
-									  user=self.username,
+                                      user=self.username,
-									  passwd=self.password)
+                                      passwd=self.password)
-			opener = urllib2.build_opener(auth_handler)
+            opener = urllib2.build_opener(auth_handler)
-			urllib2.install_opener(opener)
+            urllib2.install_opener(opener)
 
-			try:
+            try:
-				f = urllib2.urlopen(url)
+                f = urllib2.urlopen(url)
-			except urllib2.HTTPError:
+            except urllib2.HTTPError:
-				self.report_progress(0,_('Can\'t login to download issue'))
+                self.report_progress(0,_('Can\'t login to download issue'))
-				return
+                raise ValueError('Failed to login, check your username and'
+                        ' password')
 
-			tmp = tempfile.TemporaryFile()
+            tmp = tempfile.TemporaryFile()
-			self.report_progress(0,_('downloading epub'))
+            self.report_progress(0,_('downloading epub'))
-			tmp.write(f.read())
+            tmp.write(f.read())
 
-			zfile = zipfile.ZipFile(tmp, 'r')
+            zfile = zipfile.ZipFile(tmp, 'r')
-			self.report_progress(0,_('extracting epub'))
+            self.report_progress(0,_('extracting epub'))
 
-			zfile.extractall(self.output_dir)
+            zfile.extractall(self.output_dir)
 
-			tmp.close()
+            tmp.close()
-			index = os.path.join(self.output_dir, 'content.opf')
+            index = os.path.join(self.output_dir, 'content.opf')
 
-			self.report_progress(1,_('epub downloaded and extracted'))
+            self.report_progress(1,_('epub downloaded and extracted'))
 
-			return index
+            return index

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.17'
+__version__   = '0.7.18'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/devices/kobo/books.py

@@ -22,13 +22,15 @@ class Book(Book_):
         self.mime = mime
 
         self.size = size # will be set later if None
-        try:
+
-            if ContentType == '6':
+        if ContentType == '6':
-                self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")
+            self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")
-            else:
+        else:
+            try:
                 self.datetime = time.gmtime(os.path.getctime(self.path))
-        except:
+            except:
-            self.datetime = time.gmtime()
+                self.datetime = time.gmtime()
+
         if thumbnail_name is not None:
             self.thumbnail = ImageWrapper(thumbnail_name)
         self.tags = []

src/calibre/devices/kobo/driver.py

@@ -106,11 +106,14 @@ class KOBO(USBMS):
                             changed = True
                     bl[idx].device_collections = playlist_map.get(lpath, [])
                 else:
-                    book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
+                    if ContentType == '6':
+                        book =  Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=1048576)
+                    else:
+                        book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
                     # print 'Update booklist'
+                    book.device_collections = playlist_map.get(book.lpath, [])
                     if bl.add_book(book, replace_metadata=False):
                         changed = True
-                    book.device_collections = playlist_map.get(book.lpath, [])
             except: # Probably a path encoding error
                 import traceback
                 traceback.print_exc()
@@ -231,21 +234,9 @@ class KOBO(USBMS):
             path = self.normalize_path(path)
             # print "Delete file normalized path: " + path
             extension =  os.path.splitext(path)[1]
+            ContentType = self.get_content_type_from_extension(extension)
             
-            if extension == '.kobo':
+            ContentID = self.contentid_from_path(path, ContentType)
-                # Kobo books do not have book files.  They do have some images though
-                #print "kobo book"
-                ContentType = 6
-                ContentID = self.contentid_from_path(path, ContentType)
-            elif extension == '.pdf' or extension == '.epub':
-                # print "ePub or pdf"
-                ContentType = 16
-                #print "Path: " + path
-                ContentID = self.contentid_from_path(path, ContentType)
-                # print "ContentID: " + ContentID
-            else: # if extension == '.html' or extension == '.txt':
-                ContentType = 999 # Yet another hack: to get around Kobo changing how ContentID is stored
-                ContentID = self.contentid_from_path(path, ContentType)
 
             ImageID = self.delete_via_sql(ContentID, ContentType)
             #print " We would now delete the Images for" + ImageID
@@ -343,6 +334,17 @@ class KOBO(USBMS):
         ContentID = ContentID.replace("\\", '/')
         return ContentID
 
+    def get_content_type_from_extension(self, extension):
+        if extension == '.kobo':
+            # Kobo books do not have book files.  They do have some images though
+            #print "kobo book"
+            ContentType = 6
+        elif extension == '.pdf' or extension == '.epub':
+            # print "ePub or pdf"
+            ContentType = 16
+        else: # if extension == '.html' or extension == '.txt':
+            ContentType = 999 # Yet another hack: to get around Kobo changing how ContentID is stored
+        return ContentType
 
     def path_from_contentid(self, ContentID, ContentType, oncard):
         path = ContentID

src/calibre/devices/prs505/sony_cache.py

@@ -89,6 +89,10 @@ class XMLCache(object):
                         raw, strip_encoding_pats=True, assume_utf8=True,
                         verbose=DEBUG)[0],
                         parser=parser)
+            if self.roots[source_id] is None:
+                raise Exception(('The SONY database at %s is corrupted. Try '
+                        ' disconnecting and reconnecting your reader.')%path)
+
         # }}}
 
         recs = self.roots[0].xpath('//*[local-name()="records"]')

src/calibre/ebooks/metadata/opf2.py

@@ -1106,7 +1106,8 @@ class OPFCreator(Metadata):
             spine.set('toc', 'ncx')
         if self.spine is not None:
             for ref in self.spine:
-                spine.append(E.itemref(idref=ref.id))
+                if ref.id is not None:
+                    spine.append(E.itemref(idref=ref.id))
         guide = E.guide()
         if self.guide is not None:
             for ref in self.guide:

src/calibre/gui2/device.py

@@ -760,8 +760,8 @@ class DeviceMixin(object): # {{{
 
     def refresh_ondevice_info(self, device_connected, reset_only = False):
         '''
-        Force the library view to refresh, taking into consideration
+        Force the library view to refresh, taking into consideration new
-        books information
+        device books information
         '''
         self.book_on_device(None, reset=True)
         if reset_only:
@@ -791,12 +791,14 @@ class DeviceMixin(object): # {{{
                     self.booklists())
             model.paths_deleted(paths)
             self.upload_booklists()
-        # Clear the ondevice info so it will be recomputed
+        # Force recomputation the library's ondevice info. We need to call
+        # set_books_in_library even though books were not added because
+        # the deleted book might have been an exact match.
+        self.set_books_in_library(self.booklists(), reset=True)
         self.book_on_device(None, None, reset=True)
-        # We want to reset all the ondevice flags in the library. Use a big
+        # We need to reset the ondevice flags in the library. Use a big hammer,
-        # hammer, so we don't need to worry about whether some succeeded or not
+        # so we don't need to worry about whether some succeeded or not.
-        self.library_view.model().refresh()
+        self.refresh_ondevice_info(device_connected=True, reset_only=False)
-
 
     def dispatch_sync_event(self, dest, delete, specific):
         rows = self.library_view.selectionModel().selectedRows()
@@ -1286,8 +1288,14 @@ class DeviceMixin(object): # {{{
             books_to_be_deleted = memory[1]
             self.library_view.model().delete_books_by_id(books_to_be_deleted)
 
-        self.set_books_in_library(self.booklists(),
+        # There are some cases where sending a book to the device overwrites a
-                reset=bool(books_to_be_deleted))
+        # book already there with a different book. This happens frequently in
+        # news. When this happens, the book match indication will be wrong
+        # because the UUID changed. Force both the device and the library view
+        # to refresh the flags.
+        self.set_books_in_library(self.booklists(), reset=True)
+        self.book_on_device(None, reset=True)
+        self.refresh_ondevice_info(device_connected = True)
 
         view = self.card_a_view if on_card == 'carda' else self.card_b_view if on_card == 'cardb' else self.memory_view
         view.model().resort(reset=False)
@@ -1295,92 +1303,53 @@ class DeviceMixin(object): # {{{
         for f in files:
             getattr(f, 'close', lambda : True)()
 
-        self.book_on_device(None, reset=True)
-        if metadata:
-            changed = set([])
-            for mi in metadata:
-                id_ = getattr(mi, 'application_id', None)
-                if id_ is not None:
-                    changed.add(id_)
-            if changed:
-                self.library_view.model().refresh_ids(list(changed))
-
     def book_on_device(self, id, format=None, reset=False):
         '''
         Return an indication of whether the given book represented by its db id
-        is on the currently connected device. It returns a 4 element list. The
+        is on the currently connected device. It returns a 5 element list. The
         first three elements represent memory locations main, carda, and cardb,
         and are true if the book is identifiably in that memory. The fourth
-        is the a count of how many instances of the book were found across all
+        is a count of how many instances of the book were found across all
-        the memory locations.
+        the memory locations. The fifth is a set of paths to the
+        matching books on the device.
         '''
-        loc = [None, None, None, 0]
+        loc = [None, None, None, 0, set([])]
 
         if reset:
-            self.book_db_title_cache = None
+            self.book_db_id_cache = None
-            self.book_db_uuid_cache = None
             self.book_db_id_counts = None
+            self.book_db_uuid_path_map = None
             return
 
-        if self.book_db_title_cache is None:
+        if not hasattr(self, 'db_book_uuid_cache'):
-            self.book_db_title_cache = []
+            return loc
-            self.book_db_uuid_cache = []
+
+        if self.book_db_id_cache is None:
+            self.book_db_id_cache = []
             self.book_db_id_counts = {}
+            self.book_db_uuid_path_map = {}
             for i, l in enumerate(self.booklists()):
-                self.book_db_title_cache.append({})
+                self.book_db_id_cache.append(set())
-                self.book_db_uuid_cache.append(set())
                 for book in l:
-                    book_title = book.title.lower() if book.title else ''
-                    book_title = re.sub('(?u)\W|[_]', '', book_title)
-                    if book_title not in self.book_db_title_cache[i]:
-                        self.book_db_title_cache[i][book_title] = \
-                                {'authors':set(), 'db_ids':set(), 'uuids':set()}
-                    book_authors = authors_to_string(book.authors).lower()
-                    book_authors = re.sub('(?u)\W|[_]', '', book_authors)
-                    self.book_db_title_cache[i][book_title]['authors'].add(book_authors)
                     db_id = getattr(book, 'application_id', None)
                     if db_id is None:
                         db_id = book.db_id
                     if db_id is not None:
-                        self.book_db_title_cache[i][book_title]['db_ids'].add(db_id)
                         # increment the count of books on the device with this
                         # db_id.
+                        self.book_db_id_cache[i].add(db_id)
+                        if db_id not in self.book_db_uuid_path_map:
+                            self.book_db_uuid_path_map[db_id] = set()
+                        if getattr(book, 'lpath', False):
+                            self.book_db_uuid_path_map[db_id].add(book.lpath)
                         c = self.book_db_id_counts.get(db_id, 0)
                         self.book_db_id_counts[db_id] = c + 1
-                    uuid = getattr(book, 'uuid', None)
-                    if uuid is not None:
-                        self.book_db_uuid_cache[i].add(uuid)
 
-        mi = self.library_view.model().db.get_metadata(id, index_is_id=True)
         for i, l in enumerate(self.booklists()):
-            if mi.uuid in self.book_db_uuid_cache[i]:
+            if id in self.book_db_id_cache[i]:
                 loc[i] = True
-                continue
+                loc[3] = self.book_db_id_counts.get(id, 0)
-            db_title = re.sub('(?u)\W|[_]', '', mi.title.lower())
+                loc[4] |= self.book_db_uuid_path_map[id]
-            cache = self.book_db_title_cache[i].get(db_title, None)
-            if cache:
-                if id in cache['db_ids']:
-                    loc[i] = True
-                    continue
-                if mi.authors and \
-                        re.sub('(?u)\W|[_]', '', authors_to_string(mi.authors).lower()) \
-                        in cache['authors']:
-                    # We really shouldn't get here, because set_books_in_library
-                    # should have set the db_ids for the books, and therefore
-                    # the if just above should have found them. Mark the book
-                    # anyway, and print a message about the situation
-                    loc[i] = True
-                    prints('book_on_device: matched title/author but not db_id!',
-                            mi.title, authors_to_string(mi.authors))
-                    continue
-                # Also check author sort, because it can be used as author in
-                # some formats
-                if mi.author_sort and \
-                        re.sub('(?u)\W|[_]', '', mi.author_sort.lower()) \
-                        in cache['authors']:
-                    loc[i] = True
-                    continue
-        loc[3] = self.book_db_id_counts.get(id, 0)
         return loc
 
     def set_books_in_library(self, booklists, reset=False):
@@ -1390,46 +1359,52 @@ class DeviceMixin(object): # {{{
         it sets the application_id for matched books. Book_on_device uses that
         to both speed up matching and to count matches.
         '''
+
+        string_pat = re.compile('(?u)\W|[_]')
+        def clean_string(x):
+            x = x.lower() if x else ''
+            return string_pat.sub('', x)
+
         # Force a reset if the caches are not initialized
         if reset or not hasattr(self, 'db_book_title_cache'):
             # It might be possible to get here without having initialized the
             # library view. In this case, simply give up
-            if not hasattr(self, 'library_view') or self.library_view is None:
+            try:
-                return
+                db = self.library_view.model().db
-            db = getattr(self.library_view.model(), 'db', None)
+            except:
-            if db is None:
                 return
             # Build a cache (map) of the library, so the search isn't On**2
             self.db_book_title_cache = {}
             self.db_book_uuid_cache = {}
             for id in db.data.iterallids():
                 mi = db.get_metadata(id, index_is_id=True)
-                title = re.sub('(?u)\W|[_]', '', mi.title.lower())
+                title = clean_string(mi.title)
                 if title not in self.db_book_title_cache:
                     self.db_book_title_cache[title] = \
                                 {'authors':{}, 'author_sort':{}, 'db_ids':{}}
+                # If there are multiple books in the library with the same title
+                # and author, then remember the last one. That is OK, because as
+                # we can't tell the difference between the books, one is as good
+                # as another.
                 if mi.authors:
-                    authors = authors_to_string(mi.authors).lower()
+                    authors = clean_string(authors_to_string(mi.authors))
-                    authors = re.sub('(?u)\W|[_]', '', authors)
                     self.db_book_title_cache[title]['authors'][authors] = mi
                 if mi.author_sort:
-                    aus = mi.author_sort.lower()
+                    aus = clean_string(mi.author_sort)
-                    aus = re.sub('(?u)\W|[_]', '', aus)
                     self.db_book_title_cache[title]['author_sort'][aus] = mi
                 self.db_book_title_cache[title]['db_ids'][mi.application_id] = mi
                 self.db_book_uuid_cache[mi.uuid] = mi
 
         # Now iterate through all the books on the device, setting the
-        # in_library field. Fastest and most accurate key is the uuid. Second is
+        # in_library field. If the UUID matches a book in the library, then
-        # the application_id, which is really the db key, but as this can
+        # do not consider that book for other matching. In all cases set
-        # accidentally match across libraries we also verify the title. The
+        # the application_id to the db_id of the matching book. This value
-        # db_id exists on Sony devices. Fallback is title and author match.
+        # will be used by books_on_device to indicate matches.
-        # We set the application ID so that we can reproduce book matching,
-        # necessary for identifying copies of books.
 
         update_metadata = prefs['manage_device_metadata'] == 'on_connect'
         for booklist in booklists:
             for book in booklist:
+                book.in_library = None
                 if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
                     if update_metadata:
                         book.smart_update(self.db_book_uuid_cache[book.uuid],
@@ -1439,20 +1414,23 @@ class DeviceMixin(object): # {{{
                     book.application_id = \
                         self.db_book_uuid_cache[book.uuid].application_id
                     continue
-
+                # No UUID exact match. Try metadata matching.
-                book_title = book.title.lower() if book.title else ''
+                book_title = clean_string(book.title)
-                book_title = re.sub('(?u)\W|[_]', '', book_title)
-                book.in_library = None
                 d = self.db_book_title_cache.get(book_title, None)
                 if d is not None:
+                    # At this point we know that the title matches. The book
+                    # will match if any of the db_id, author, or author_sort
+                    # also match.
                     if getattr(book, 'application_id', None) in d['db_ids']:
                         book.in_library = True
-                        # application already matches db_id, so no need to set it
+                        # app_id already matches a db_id. No need to set it.
                         if update_metadata:
                             book.smart_update(d['db_ids'][book.application_id],
                                               replace_metadata=True)
                         continue
-                    if book.db_id in d['db_ids']:
+                    # Sonys know their db_id independent of the application_id
+                    # in the metadata cache. Check that as well.
+                    if getattr(book, 'db_id', None) in d['db_ids']:
                         book.in_library = True
                         book.application_id = \
                                     d['db_ids'][book.db_id].application_id
@@ -1460,11 +1438,15 @@ class DeviceMixin(object): # {{{
                             book.smart_update(d['db_ids'][book.db_id],
                                               replace_metadata=True)
                         continue
+                    # We now know that the application_id is not right. Set it
+                    # to None to prevent book_on_device from accidentally
+                    # matching on it. It will be set to a correct value below if
+                    # the book is matched with one in the library
+                    book.application_id = None
                     if book.authors:
                         # Compare against both author and author sort, because
                         # either can appear as the author
-                        book_authors = authors_to_string(book.authors).lower()
+                        book_authors = clean_string(authors_to_string(book.authors))
-                        book_authors = re.sub('(?u)\W|[_]', '', book_authors)
                         if book_authors in d['authors']:
                             book.in_library = True
                             book.application_id = \
@@ -1479,6 +1461,9 @@ class DeviceMixin(object): # {{{
                             if update_metadata:
                                 book.smart_update(d['author_sort'][book_authors],
                                                   replace_metadata=True)
+                else:
+                    # Book definitely not matched. Clear its application ID
+                    book.application_id = None
                 # Set author_sort if it isn't already
                 asort = getattr(book, 'author_sort', None)
                 if not asort and book.authors:

src/calibre/gui2/library/models.py

@@ -121,10 +121,11 @@ class BooksModel(QAbstractTableModel): # {{{
     def set_device_connected(self, is_connected):
         self.device_connected = is_connected
         self.db.refresh_ondevice()
+        self.refresh()
+        self.research()
         if is_connected and self.sorted_on[0] == 'ondevice':
             self.resort()
 
-
     def set_book_on_device_func(self, func):
         self.book_on_device = func
 

src/calibre/library/database2.py

@@ -648,7 +648,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         count = 0
         on = self.book_on_device(id)
         if on is not None:
-            m, a, b, count = on
+            m, a, b, count = on[:4]
             if m is not None:
                 loc.append(_('Main'))
             if a is not None:

src/calibre/utils/filenames.py

@@ -55,6 +55,9 @@ def shorten_components_to(length, components):
         else:
             if x is components[-1]:
                 b, _, e = x.rpartition('.')
+                if not b and e:
+                    b = e
+                    e = ''
                 r = b[:-delta]+e
                 if r.startswith('.'): r = x[0]+r
             else: