Sync to trunk.

recipes/economist.recipe

@@ -77,30 +77,21 @@ class Economist(BasicNewsRecipe):
                 continue
             self.log('Found section: %s'%section_title)
             articles = []
-            for h5 in section.findAll('h5'):
-                article_title = self.tag_to_string(h5).strip()
-                if not article_title:
-                    continue
-                data = h5.findNextSibling(attrs={'class':'article'})
-                if data is None: continue
-                a = data.find('a', href=True)
-                if a is None: continue
-                url = a['href']
-                if url.startswith('/'): url = 'http://www.economist.com'+url
-                url += '/print'
-                article_title += ': %s'%self.tag_to_string(a).strip()
-                articles.append({'title':article_title, 'url':url,
-                    'description':'', 'date':''})
-            if not articles:
-                # We have last or first section
-                for art in section.findAll(attrs={'class':'article'}):
-                    a = art.find('a', href=True)
+            subsection = ''
+            for node in section.findAll(attrs={'class':'article'}):
+                subsec = node.findPreviousSibling('h5')
+                if subsec is not None:
+                    subsection = self.tag_to_string(subsec)
+                prefix = (subsection+': ') if subsection else ''
+                a = node.find('a', href=True)
                 if a is not None:
                     url = a['href']
                     if url.startswith('/'): url = 'http://www.economist.com'+url
                     url += '/print'
                     title = self.tag_to_string(a)
                     if title:
+                        title = prefix + title
+                        self.log('\tFound article:', title)
                         articles.append({'title':title, 'url':url,
                         'description':'', 'date':''})
 

recipes/economist_free.recipe

@@ -69,30 +69,21 @@ class Economist(BasicNewsRecipe):
                 continue
             self.log('Found section: %s'%section_title)
             articles = []
-            for h5 in section.findAll('h5'):
-                article_title = self.tag_to_string(h5).strip()
-                if not article_title:
-                    continue
-                data = h5.findNextSibling(attrs={'class':'article'})
-                if data is None: continue
-                a = data.find('a', href=True)
-                if a is None: continue
-                url = a['href']
-                if url.startswith('/'): url = 'http://www.economist.com'+url
-                url += '/print'
-                article_title += ': %s'%self.tag_to_string(a).strip()
-                articles.append({'title':article_title, 'url':url,
-                    'description':'', 'date':''})
-            if not articles:
-                # We have last or first section
-                for art in section.findAll(attrs={'class':'article'}):
-                    a = art.find('a', href=True)
+            subsection = ''
+            for node in section.findAll(attrs={'class':'article'}):
+                subsec = node.findPreviousSibling('h5')
+                if subsec is not None:
+                    subsection = self.tag_to_string(subsec)
+                prefix = (subsection+': ') if subsection else ''
+                a = node.find('a', href=True)
                 if a is not None:
                     url = a['href']
                     if url.startswith('/'): url = 'http://www.economist.com'+url
                     url += '/print'
                     title = self.tag_to_string(a)
                     if title:
+                        title = prefix + title
+                        self.log('\tFound article:', title)
                         articles.append({'title':title, 'url':url,
                         'description':'', 'date':''})
 

resources/default_tweaks.py

@@ -63,7 +63,7 @@ authors_completer_append_separator = False
 # end of an author name. The case of the suffix is ignored and trailing
 # periods are automatically handled.
 # The author name copy words are a set of words which if they occur in an
-# author name cause the automatically geenrated author sort string to be
+# author name cause the automatically generated author sort string to be
 # identical to the author name. This means that the sort for a string like Acme
 # Inc. will be Acme Inc. instead of Inc., Acme
 author_sort_copy_method = 'comma'

setup/check.py

@@ -63,10 +63,10 @@ class Check(Command):
             for f in x[-1]:
                 y = self.j(x[0], f)
                 mtime = os.stat(y).st_mtime
-                if f.endswith('.py') and f not in ('ptempfile.py', 'feedparser.py',
-                    'pyparsing.py', 'markdown.py') and \
-                    'genshi' not in y and cache.get(y, 0) != mtime and \
-                    'prs500/driver.py' not in y:
+                if (f.endswith('.py') and f not in ('ptempfile.py', 'feedparser.py',
+                    'pyparsing.py', 'markdown.py') and
+                    'genshi' not in y and cache.get(y, 0) != mtime and
+                    'prs500/driver.py' not in y):
                         yield y, mtime
 
         for x in os.walk(self.j(self.d(self.SRC), 'recipes')):

src/calibre/db/backend.py

@@ -25,7 +25,8 @@ from calibre.utils.config import to_json, from_json, prefs, tweaks
 from calibre.utils.date import utcfromtimestamp, parse_date
 from calibre.utils.filenames import is_case_sensitive
 from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
-        SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, CompositeTable)
+        SizeTable, FormatsTable, AuthorsTable, IdentifiersTable,
+        CompositeTable, LanguagesTable)
 # }}}
 
 '''
@@ -604,11 +605,12 @@ class DB(object):
         for col in ('series', 'publisher', 'rating'):
             tables[col] = ManyToOneTable(col, self.field_metadata[col].copy())
 
-        for col in ('authors', 'tags', 'formats', 'identifiers'):
+        for col in ('authors', 'tags', 'formats', 'identifiers', 'languages'):
             cls = {
                     'authors':AuthorsTable,
                     'formats':FormatsTable,
                     'identifiers':IdentifiersTable,
+                    'languages':LanguagesTable,
                   }.get(col, ManyToManyTable)
             tables[col] = cls(col, self.field_metadata[col].copy())
 

src/calibre/db/cache.py

@@ -13,7 +13,8 @@ from functools import wraps, partial
 
 from calibre.db.locking import create_locks, RecordLock
 from calibre.db.fields import create_field
-from calibre.ebooks.book.base import Metadata
+from calibre.db.tables import VirtualTable
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import now
 
 def api(f):
@@ -189,7 +190,8 @@ class Cache(object):
                 if table.metadata['datatype'] == 'composite':
                     self.composites.add(field)
 
-            self.fields['ondevice'] = create_field('ondevice', None)
+            self.fields['ondevice'] = create_field('ondevice',
+                    VirtualTable('ondevice'))
 
     @read_api
     def field_for(self, name, book_id, default_value=None):
@@ -345,8 +347,9 @@ class Cache(object):
                     as_path=as_path)
 
     @read_api
-    def multisort(self, fields):
-        all_book_ids = frozenset(self._all_book_ids())
+    def multisort(self, fields, ids_to_sort=None):
+        all_book_ids = frozenset(self._all_book_ids() if ids_to_sort is None
+                else ids_to_sort)
         get_metadata = partial(self._get_metadata, get_user_categories=False)
 
         sort_keys = tuple(self.fields[field[0]].sort_keys_for_books(get_metadata,

src/calibre/db/fields.py

@@ -51,9 +51,13 @@ class Field(object):
 
     def __iter__(self):
         '''
-        Iterate over the ids for all values in this field
+        Iterate over the ids for all values in this field.
+
+        WARNING: Some fields such as composite fields and virtual
+        fields like ondevice do not have ids for their values, in such
+        cases this is an empty iterator.
         '''
-        raise NotImplementedError()
+        return iter(())
 
     def sort_keys_for_books(self, get_metadata, all_book_ids):
         '''
@@ -78,9 +82,6 @@ class OneToOneField(Field):
     def __iter__(self):
         return self.table.book_col_map.iterkeys()
 
-    def iter_book_ids(self):
-        return self.table.book_col_map.iterkeys()
-
     def sort_keys_for_books(self, get_metadata, all_book_ids):
         return {id_ : self._sort_key(self.book_col_map.get(id_, '')) for id_ in
                 all_book_ids}
@@ -154,9 +155,6 @@ class OnDeviceField(OneToOneField):
     def __iter__(self):
         return iter(())
 
-    def iter_book_ids(self):
-        return iter(())
-
     def sort_keys_for_books(self, get_metadata, all_book_ids):
         return {id_ : self.for_book(id_) for id_ in
                 all_book_ids}

src/calibre/db/tables.py

@@ -13,6 +13,7 @@ from dateutil.tz import tzoffset
 
 from calibre.constants import plugins
 from calibre.utils.date import parse_date, local_tz, UNDEFINED_DATE
+from calibre.utils.localization import lang_map
 from calibre.ebooks.metadata import author_to_author_sort
 
 _c_speedup = plugins['speedup'][0]
@@ -54,6 +55,19 @@ class Table(object):
         self.link_table = (link_table if link_table else
                 'books_%s_link'%self.metadata['table'])
 
+class VirtualTable(Table):
+
+    '''
+    A dummy table used for fields that only exist in memory like ondevice
+    '''
+
+    def __init__(self, name, table_type=ONE_ONE, datatype='text'):
+        metadata = {'datatype':datatype, 'table':name}
+        self.table_type = table_type
+        Table.__init__(self, name, metadata)
+
+
+
 class OneToOneTable(Table):
 
     '''
@@ -210,3 +224,9 @@ class IdentifiersTable(ManyToManyTable):
         for key in tuple(self.col_book_map.iterkeys()):
             self.col_book_map[key] = tuple(self.col_book_map[key])
 
+class LanguagesTable(ManyToManyTable):
+
+    def read_id_maps(self, db):
+        ManyToManyTable.read_id_maps(self, db)
+        lm = lang_map()
+        self.lang_name_map = {x:lm.get(x, x) for x in self.id_map.itervalues()}

src/calibre/devices/android/driver.py

@@ -81,6 +81,7 @@ class ANDROID(USBMS):
 
             # LG
             0x1004 : {
+                    0x61c5 : [0x100, 0x226, 0x9999],
                     0x61cc : [0x100],
                     0x61ce : [0x100],
                     0x618e : [0x226, 0x9999, 0x100]

src/calibre/ebooks/markdown/blockprocessors.py

@@ -355,7 +355,7 @@ class HashHeaderProcessor(BlockProcessor):
                 blocks.insert(0, after)
         else:
             # This should never happen, but just in case...
-            message(CRITICAL, "We've got a problem header!")
+            print("We've got a problem header!")
 
 
 class SetextHeaderProcessor(BlockProcessor):
@@ -407,7 +407,7 @@ class HRProcessor(BlockProcessor):
             # Recursively parse lines before hr so they get parsed first.
             self.parser.parseBlocks(parent, ['\n'.join(prelines)])
         # create hr
-        hr = markdown.etree.SubElement(parent, 'hr')
+        markdown.etree.SubElement(parent, 'hr')
         # check for lines in block after hr.
         lines = lines[len(prelines)+1:]
         if len(lines):

src/calibre/ebooks/markdown/commandline.py

@@ -9,7 +9,7 @@ Markdown is called from the command line.
 import markdown
 import sys
 import logging
-from logging import DEBUG, INFO, WARN, ERROR, CRITICAL
+from logging import DEBUG, INFO, CRITICAL
 
 EXECUTABLE_NAME_FOR_USAGE = "python markdown.py"
 """ The name used in the usage statement displayed for python versions < 2.3.

src/calibre/ebooks/markdown/etree_loader.py

@@ -8,9 +8,11 @@ def importETree():
     etree_in_c = None
     try: # Is it Python 2.5+ with C implemenation of ElementTree installed?
         import xml.etree.cElementTree as etree_in_c
+        etree_in_c
     except ImportError:
         try: # Is it Python 2.5+ with Python implementation of ElementTree?
             import xml.etree.ElementTree as etree
+            etree
         except ImportError:
             try: # An earlier version of Python with cElementTree installed?
                 import cElementTree as etree_in_c

src/calibre/ebooks/markdown/extensions/def_list.py

@@ -80,7 +80,7 @@ class DefListIndentProcessor(markdown.blockprocessors.ListIndentProcessor):
     ITEM_TYPES = ['dd']
     LIST_TYPES = ['dl']
 
-    def create_item(parent, block):
+    def create_item(self, parent, block):
         """ Create a new dd and parse the block with it as the parent. """
         dd = markdown.etree.SubElement(parent, 'dd')
         self.parser.parseBlocks(dd, [block])

src/calibre/ebooks/markdown/extensions/footnotes.py

@@ -106,7 +106,7 @@ class FootnoteExtension(markdown.Extension):
 
         div = etree.Element("div")
         div.set('class', 'footnote')
-        hr = etree.SubElement(div, "hr")
+        etree.SubElement(div, "hr")
         ol = etree.SubElement(div, "ol")
 
         for id in self.footnotes.keys():
@@ -199,7 +199,6 @@ class FootnotePreprocessor(markdown.preprocessors.Preprocessor):
 
         """
         items = []
-        item = -1
         i = 0 # to keep track of where we are
 
         def detab(line):
@@ -277,7 +276,6 @@ class FootnoteTreeprocessor(markdown.treeprocessors.Treeprocessor):
                     ind = element.getchildren().find(child)
                     element.getchildren().insert(ind + 1, footnotesDiv)
                     child.tail = None
-                fnPlaceholder.parent.replaceChild(fnPlaceholder, footnotesDiv)
             else:
                 root.append(footnotesDiv)
 

src/calibre/ebooks/markdown/extensions/headerid.py

@@ -66,7 +66,6 @@ Dependencies:
 """
 
 import calibre.ebooks.markdown.markdown as markdown
-from calibre.ebooks.markdown.markdown import etree
 import re
 from string import ascii_lowercase, digits, punctuation
 
@@ -119,7 +118,7 @@ class HeaderIdProcessor(markdown.blockprocessors.BlockProcessor):
                 blocks.insert(0, after)
         else:
             # This should never happen, but just in case...
-            message(CRITICAL, "We've got a problem header!")
+            print ("We've got a problem header!")
 
     def _get_meta(self):
         """ Return meta data suported by this ext as a tuple """

src/calibre/ebooks/markdown/inlinepatterns.py

@@ -47,6 +47,7 @@ from urlparse import urlparse, urlunparse
 import sys
 if sys.version >= "3.0":
     from html import entities as htmlentitydefs
+    htmlentitydefs
 else:
     import htmlentitydefs
 
@@ -215,7 +216,6 @@ class HtmlPattern (Pattern):
     """ Store raw inline html and return a placeholder. """
     def handleMatch (self, m):
         rawhtml = m.group(2)
-        inline = True
         place_holder = self.markdown.htmlStash.store(rawhtml)
         return place_holder
 

src/calibre/ebooks/markdown/odict.py

@@ -156,7 +156,7 @@ class OrderedDict(dict):
                 self.keyOrder.insert(i, key)
             else:
                 self.keyOrder.append(key)
-        except Error:
+        except Exception as e:
             # restore to prevent data loss and reraise
             self.keyOrder.insert(n, key)
-            raise Error
+            raise e

src/calibre/ebooks/markdown/treeprocessors.py

@@ -185,7 +185,7 @@ class InlineProcessor(Treeprocessor):
                     result.append(node)
 
                 else: # wrong placeholder
-                    end = index + len(prefix)
+                    end = index + len(self.__placeholder_prefix)
                     linkText(data[strartIndex:end])
                     strartIndex = end
             else:

src/calibre/gui2/actions/__init__.py

@@ -220,12 +220,11 @@ class InterfaceAction(QObject):
             ac.setStatusTip(description)
             ac.setWhatsThis(description)
 
-        ac.calibre_shortcut_unique_name = None
+        ac.calibre_shortcut_unique_name = unique_name
         if shortcut is not False:
             self.gui.keyboard.register_shortcut(unique_name,
                 shortcut_name, default_keys=keys,
                 action=ac, description=description, group=self.action_spec[0])
-            ac.calibre_shortcut_unique_name = unique_name
         if triggered is not None:
             ac.triggered.connect(triggered)
         return ac

src/calibre/gui2/keyboard.py

@@ -125,11 +125,14 @@ class Manager(QObject): # {{{
         #pprint.pprint(self.keys_map)
 
     def replace_action(self, unique_name, new_action):
+        '''
+        Replace the action associated with a shortcut.
+        Once you're done calling replace_action() for all shortcuts you want
+        replaced, call finalize() to have the shortcuts assigned to the replaced
+        actions.
+        '''
         sc = self.shortcuts[unique_name]
-        ac = sc['action']
-        if ac is not None:
-            new_action.setShortcuts(ac.shortcuts())
-            ac.setShortcuts([])
+        sc['action'] = new_action
 
 # }}}
 

src/calibre/library/server/ajax.py

@@ -83,6 +83,10 @@ def category_url(prefix, cid):
 
 def icon_url(prefix, name):
     return absurl(prefix, '/browse/icon/'+name)
+
+def books_in_url(prefix, category, cid):
+    return absurl(prefix, '/ajax/books_in/%s/%s'%(
+        encode_name(category), encode_name(cid)))
 # }}}
 
 class AjaxServer(object):
@@ -114,7 +118,7 @@ class AjaxServer(object):
 
 
     # Get book metadata {{{
-    def ajax_book_to_json(self, book_id):
+    def ajax_book_to_json(self, book_id, get_category_urls=True):
         mi = self.db.get_metadata(book_id, index_is_id=True)
         try:
             mi.rating = mi.rating/2.
@@ -151,18 +155,46 @@ class AjaxServer(object):
         data['other_formats'] = {fmt: absurl(self.opts.url_prefix, u'/get/%s/%d'%(fmt, book_id)) for fmt
                 in other_fmts}
 
+        if get_category_urls:
+            category_urls = data['category_urls'] = {}
+            ccache = self.categories_cache()
+            for key in mi.all_field_keys():
+                fm = mi.metadata_for_field(key)
+                if (fm and fm['is_category'] and not fm['is_csp'] and
+                        key != 'formats' and fm['datatype'] not in ['rating']):
+                    categories = mi.get(key)
+                    if isinstance(categories, basestring):
+                        categories = [categories]
+                    if categories is None:
+                        categories = []
+                    dbtags = {}
+                    for category in categories:
+                        for tag in ccache.get(key, []):
+                            if tag.original_name == category:
+                                dbtags[category] = books_in_url(self.opts.url_prefix,
+                                    tag.category if tag.category else key,
+                                    tag.original_name if tag.id is None else
+                                    unicode(tag.id))
+                                break
+                    category_urls[key] = dbtags
+
         return data, mi.last_modified
 
     @Endpoint(set_last_modified=False)
-    def ajax_book(self, book_id):
+    def ajax_book(self, book_id, category_urls='true'):
         '''
         Return the metadata of the book as a JSON dictionary.
+
+        If category_urls == 'true' the returned dictionary also contains a
+        mapping of category names to URLs that return the list of books in the
+        given category.
         '''
         cherrypy.response.timeout = 3600
 
         try:
             book_id = int(book_id)
-            data, last_modified = self.ajax_book_to_json(book_id)
+            data, last_modified = self.ajax_book_to_json(book_id,
+                    get_category_urls=category_urls.lower()=='true')
         except:
             raise cherrypy.HTTPError(404, 'No book with id: %r'%book_id)
 
@@ -172,7 +204,7 @@ class AjaxServer(object):
         return data
 
     @Endpoint(set_last_modified=False)
-    def ajax_books(self, ids=None):
+    def ajax_books(self, ids=None, category_urls='true'):
         '''
         Return the metadata for a list of books specified as a comma separated
         list of ids. The metadata is returned as a dictionary mapping ids to
@@ -192,9 +224,11 @@ class AjaxServer(object):
                     ' of integers')
         ans = {}
         lm = None
+        gcu = category_urls.lower()=='true'
         for book_id in ids:
             try:
-                data, last_modified = self.ajax_book_to_json(book_id)
+                data, last_modified = self.ajax_book_to_json(book_id,
+                        get_category_urls=gcu)
             except:
                 ans[book_id] = None
             else:
@@ -431,9 +465,9 @@ class AjaxServer(object):
             'name':item_names.get(x, x.original_name),
             'average_rating': x.avg_rating,
             'count': x.count,
-            'url': absurl(self.opts.url_prefix, '/ajax/books_in/%s/%s'%(
-                encode_name(x.category if x.category else toplevel),
-                encode_name(x.original_name if x.id is None else unicode(x.id)))),
+            'url': books_in_url(self.opts.url_prefix,
+                x.category if x.category else toplevel,
+                x.original_name if x.id is None else unicode(x.id)),
             'has_children': x.original_name in children,
             } for x in items]
 

src/calibre/library/server/base.py

@@ -49,6 +49,8 @@ class DispatchController(object): # {{{
         elif self.prefix:
             self.dispatcher.connect(name+'prefix_extra', self.prefix, self,
                     **kwargs)
+            self.dispatcher.connect(name+'prefix_extra_trailing',
+                    self.prefix+'/', self, **kwargs)
         self.dispatcher.connect(name, route, self, **kwargs)
         self.funcs.append(expose(func))
 

src/calibre/library/server/browse.py

@@ -695,8 +695,8 @@ class BrowseServer(object):
                         for tag in dbtags:
                             tval = ('<a title="Browse books by {3}: {0}"'
                             ' href="{1}" class="details_category_link">{2}</a>')
-                            href='/browse/matches/%s/%s' % \
-                            (quote(tag.category), quote(str(tag.id)))
+                            href='%s/browse/matches/%s/%s' % \
+                            (self.opts.url_prefix, quote(tag.category), quote(str(tag.id)))
                             vals.append(tval.format(xml(tag.name, True),
                                 xml(href, True),
                                 xml(val if len(dbtags) == 1 else tag.name),