Merge from trunk

src/calibre/devices/android/driver.py

@@ -60,7 +60,12 @@ class ANDROID(USBMS):
             0x1004 : { 0x61cc : [0x100] },
 
             # Archos
-            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]},
+            0x0e79 : {
+                0x1400 : [0x0222, 0x0216],
+                0x1419 : [0x0216],
+                0x1420 : [0x0216],
+                0x1422 : [0x0216]
+            },
 
             # Huawei
             # Disabled as this USB id is used by various USB flash drives
@@ -84,10 +89,10 @@ class ANDROID(USBMS):
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
             'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
-            'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD']
+            'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD', '7']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
-            'A70S', 'A101IT']
+            'A70S', 'A101IT', '7']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/ebooks/metadata/book/base.py

@@ -30,6 +30,7 @@ NULL_VALUES = {
                 'author_sort_map': {},
                 'authors'      : [_('Unknown')],
                 'title'        : _('Unknown'),
+                'user_categories' : {},
                 'language'     : 'und'
 }
 

src/calibre/ebooks/metadata/opf2.py

@@ -471,8 +471,11 @@ def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8))
 
 
 def dump_user_categories(cats):
+    if not cats:
+        cats = {}
     from calibre.ebooks.metadata.book.json_codec import object_to_unicode
-    return json.dumps(object_to_unicode(cats))
+    return json.dumps(object_to_unicode(cats), ensure_ascii=False,
+            skipkeys=True)
 
 class OPF(object): # {{{
 
@@ -1182,7 +1185,7 @@ class OPFCreator(Metadata):
             a(CAL_ELEM('calibre:timestamp', self.timestamp.isoformat()))
         if self.publication_type is not None:
             a(CAL_ELEM('calibre:publication_type', self.publication_type))
-        if self.user_categories is not None:
+        if self.user_categories:
             from calibre.ebooks.metadata.book.json_codec import object_to_unicode
             a(CAL_ELEM('calibre:user_categories',
                        json.dumps(object_to_unicode(self.user_categories))))
@@ -1311,7 +1314,7 @@ def metadata_to_opf(mi, as_string=True):
     if mi.title_sort:
         meta('title_sort', mi.title_sort)
     if mi.user_categories:
-        meta('user_categories', json.dumps(mi.user_categories))
+        meta('user_categories', dump_user_categories(mi.user_categories))
 
     serialize_user_metadata(metadata, mi.get_all_user_metadata(False))
 

src/calibre/ebooks/metadata/sources/base.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re
+import re, threading
 
 from calibre.customize import Plugin
 from calibre.utils.logging import ThreadSafeLog, FileStream
@@ -30,7 +30,21 @@ class Source(Plugin):
 
     touched_fields = frozenset()
 
+    def __init__(self, *args, **kwargs):
+        Plugin.__init__(self, *args, **kwargs)
+        self._isbn_to_identifier_cache = {}
+        self.cache_lock = threading.RLock()
+
     # Utility functions {{{
+
+    def cache_isbn_to_identifier(self, isbn, identifier):
+        with self.cache_lock:
+            self._isbn_to_identifier_cache[isbn] = identifier
+
+    def cached_isbn_to_identifier(self, isbn):
+        with self.cache_lock:
+            return self._isbn_to_identifier_cache.get(isbn, None)
+
     def get_author_tokens(self, authors, only_first_author=True):
         '''
         Take a list of authors and return a list of tokens useful for an

src/calibre/ebooks/metadata/sources/google.py

@@ -13,6 +13,7 @@ from functools import partial
 
 from lxml import etree
 
+from calibre.ebooks.metadata import check_isbn
 from calibre.ebooks.metadata.sources.base import Source
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.chardet import xml_to_unicode
@@ -69,6 +70,7 @@ def to_metadata(browser, log, entry_, timeout):
 
 
     id_url = entry_id(entry_)[0].text
+    google_id = id_url.split('/')[-1]
     title_ = ': '.join([x.text for x in title(entry_)]).strip()
     authors = [x.text.strip() for x in creator(entry_) if x.text]
     if not authors:
@@ -78,6 +80,7 @@ def to_metadata(browser, log, entry_, timeout):
         return None
 
     mi = Metadata(title_, authors)
+    mi.identifiers = {'google':google_id}
     try:
         raw = get_details(browser, id_url, timeout)
         feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw),
@@ -103,9 +106,12 @@ def to_metadata(browser, log, entry_, timeout):
         t = str(x.text).strip()
         if t[:5].upper() in ('ISBN:', 'LCCN:', 'OCLC:'):
             if t[:5].upper() == 'ISBN:':
-                isbns.append(t[5:])
+                t = check_isbn(t[5:])
+                if t:
+                    isbns.append(t)
     if isbns:
         mi.isbn = sorted(isbns, key=len)[-1]
+    mi.all_isbns = isbns
 
     # Tags
     try:
@@ -133,20 +139,6 @@ def to_metadata(browser, log, entry_, timeout):
     return mi
 
 
-def get_all_details(br, log, entries, abort, result_queue, timeout):
-    for i in entries:
-        try:
-            ans = to_metadata(br, log, i, timeout)
-            if isinstance(ans, Metadata):
-                result_queue.put(ans)
-        except:
-            log.exception(
-                'Failed to get metadata for identify entry:',
-                etree.tostring(i))
-        if abort.is_set():
-            break
-
-
 class GoogleBooks(Source):
 
     name = 'Google Books'
@@ -185,6 +177,36 @@ class GoogleBooks(Source):
             'min-viewability':'none',
             })
 
+    def cover_url_from_identifiers(self, identifiers):
+        goog = identifiers.get('google', None)
+        if goog is None:
+            isbn = identifiers.get('isbn', None)
+            goog = self.cached_isbn_to_identifier(isbn)
+        if goog is not None:
+            return ('http://books.google.com/books?id=%s&printsec=frontcover&img=1' %
+                goog)
+
+    def is_cover_image_valid(self, raw):
+        # When no cover is present, returns a PNG saying image not available
+        # Try for example google identifier llNqPwAACAAJ
+        # I have yet to see an actual cover in PNG format
+        return raw and len(raw) > 17000 and raw[1:4] != 'PNG'
+
+    def get_all_details(self, br, log, entries, abort, result_queue, timeout):
+        for i in entries:
+            try:
+                ans = to_metadata(br, log, i, timeout)
+                if isinstance(ans, Metadata):
+                    result_queue.put(ans)
+                    for isbn in ans.all_isbns:
+                        self.cache_isbn_to_identifier(isbn,
+                                ans.identifiers['google'])
+            except:
+                log.exception(
+                    'Failed to get metadata for identify entry:',
+                    etree.tostring(i))
+            if abort.is_set():
+                break
 
     def identify(self, log, result_queue, abort, title=None, authors=None,
             identifiers={}, timeout=5):
@@ -207,8 +229,8 @@ class GoogleBooks(Source):
             return as_unicode(e)
 
         # There is no point running these queries in threads as google
-        # throttles requests returning Forbidden errors
+        # throttles requests returning 403 Forbidden errors
-        get_all_details(br, log, entries, abort, result_queue, timeout)
+        self.get_all_details(br, log, entries, abort, result_queue, timeout)
 
         return None
 
@@ -218,8 +240,14 @@ if __name__ == '__main__':
             title_test)
     test_identify_plugin(GoogleBooks.name,
         [
+
             (
-                {'title': 'Great Expectations', 'authors':['Charles Dickens']},
+                {'identifiers':{'isbn': '0743273567'}},
-                [title_test('Great Expectations', exact=True)]
+                [title_test('The great gatsby', exact=True)]
             ),
+
+            #(
+            #    {'title': 'Great Expectations', 'authors':['Charles Dickens']},
+            #    [title_test('Great Expectations', exact=True)]
+            #),
     ])

src/calibre/ebooks/oeb/base.py

@@ -515,7 +515,7 @@ class Metadata(object):
                          'publisher', 'relation', 'rights', 'source',
                          'subject', 'title', 'type'])
     CALIBRE_TERMS = set(['series', 'series_index', 'rating', 'timestamp',
-                         'publication_type'])
+                         'publication_type', 'title_sort'])
     OPF_ATTRS     = {'role': OPF('role'), 'file-as': OPF('file-as'),
                      'scheme': OPF('scheme'), 'event': OPF('event'),
                      'type': XSI('type'), 'lang': XML('lang'), 'id': 'id'}

src/calibre/ebooks/oeb/transforms/metadata.py

@@ -18,7 +18,8 @@ def meta_info_to_oeb_metadata(mi, m, log, override_input_metadata=False):
     if mi.title_sort:
         if not m.title:
             m.add('title', mi.title_sort)
-        m.title[0].file_as = mi.title_sort
+        m.clear('title_sort')
+        m.add('title_sort', mi.title_sort)
     if not mi.is_null('authors'):
         m.filter('creator', lambda x : x.role.lower() in ['aut', ''])
         for a in mi.authors:

src/calibre/gui2/dialogs/drm_error.ui

@@ -44,7 +44,8 @@
     <widget class="QLabel" name="msg">
      <property name="text">
       <string>&lt;p&gt;This book is locked by &lt;b&gt;DRM&lt;/b&gt;. To learn more about DRM and why you cannot read or convert this book in calibre, 
-&lt;a href=&quot;http://bugs.calibre-ebook.com/wiki/DRM&quot;&gt;click here&lt;/a&gt;.</string>
+          &lt;a href=&quot;http://drmfree.calibre-ebook.com/about#drm&quot;&gt;click here&lt;/a&gt;.&lt;p&gt;A large number of recent, DRM free releases are 
+          available at &lt;a href=&quot;http://drmfree.calibre-ebook.com&quot;&gt;Open Books&lt;/a&gt;.</string>
      </property>
      <property name="wordWrap">
       <bool>true</bool>

src/calibre/gui2/email.py

@@ -209,7 +209,6 @@ class EmailMixin(object): # {{{
 
     def __init__(self):
         self.emailer = Emailer(self.job_manager)
-        self.emailer.start()
 
     def send_by_mail(self, to, fmts, delete_from_library, send_ids=None,
             do_auto_convert=True, specific_format=None):
@@ -255,6 +254,8 @@ class EmailMixin(object): # {{{
 
         to_s = list(repeat(to, len(attachments)))
         if attachments:
+            if not self.emailer.is_alive():
+                self.emailer.start()
             self.emailer.send_mails(jobnames,
                     Dispatcher(partial(self.email_sent, remove=remove)),
                     attachments, to_s, subjects, texts, attachment_names)
@@ -325,6 +326,8 @@ class EmailMixin(object): # {{{
             files, auto = self.library_view.model().\
                     get_preferred_formats_from_ids([id_], fmts)
             return files
+        if not self.emailer.is_alive():
+            self.emailer.start()
         sent_mails = self.emailer.email_news(mi, remove,
                 get_fmts, self.email_sent)
         if sent_mails:

src/calibre/gui2/preferences/look_feel.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>670</width>
-    <height>392</height>
+    <height>422</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -136,7 +136,7 @@
      <item>
       <widget class="QLabel" name="label_6">
        <property name="text">
-        <string>Tags browser category partitioning method:</string>
+        <string>Tags browser category &amp;partitioning method:</string>
        </property>
        <property name="buddy">
         <cstring>opt_tags_browser_partition_method</cstring>
@@ -157,7 +157,7 @@ if you never want subcategories</string>
      <item>
       <widget class="QLabel" name="label_6">
        <property name="text">
-        <string>Collapse when more items than:</string>
+        <string>&amp;Collapse when more items than:</string>
        </property>
        <property name="buddy">
         <cstring>opt_tags_browser_collapse_at</cstring>
@@ -193,7 +193,7 @@ up into sub-categories. If the partition method is set to disable, this value is
    <item row="8" column="0">
     <widget class="QLabel" name="label_81">
      <property name="text">
-      <string>Categories with hierarchical items:</string>
+      <string>Categories with &amp;hierarchical items:</string>
      </property>
      <property name="buddy">
       <cstring>opt_categories_using_hierarchy</cstring>
@@ -205,9 +205,9 @@ up into sub-categories. If the partition method is set to disable, this value is
      <property name="toolTip">
       <string>A comma-separated list of columns in which items containing
 periods are displayed in the tag browser trees. For example, if
-this box contains 'tags' then tags of the form 'mystery.English'
+this box contains 'tags' then tags of the form 'Mystery.English'
-and 'mystery.Thriller' will be displayed with English and Thriller
+and 'Mystery.Thriller' will be displayed with English and Thriller
-both under the label 'mystery'. If 'tags' is not in this box,
+both under 'Mystery'. If 'tags' is not in this box,
 then the tags will be displayed each on their own line.</string>
      </property>
     </widget>

src/calibre/gui2/tag_view.py

@@ -262,7 +262,6 @@ class TagsView(QTreeView): # {{{
                 tag_name = t.name
                 tag_id = t.id
                 can_edit = getattr(t, 'can_edit', True)
-                print can_edit, getattr(t, 'original_name', t.name), t.name
                 while item.type != TagTreeItem.CATEGORY:
                     item = item.parent
 

src/calibre/gui2/ui.py

@@ -633,7 +633,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             mb.stop()
 
         self.hide_windows()
-        self.emailer.stop()
+        if self.emailer.is_alive():
+            self.emailer.stop()
         try:
             try:
                 if self.content_server is not None:

src/calibre/library/database2.py

@@ -174,7 +174,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.prefs = DBPrefs(self)
         defs = self.prefs.defaults
         defs['gui_restriction'] = defs['cs_restriction'] = ''
-        defs['categories_using_hierarchy'] = ''
+        defs['categories_using_hierarchy'] = []
 
         # Migrate saved search and user categories to db preference scheme
         def migrate_preference(key, default):