Merge branch 'kovidgoyal/master'

recipes/acrimed.recipe

@@ -0,0 +1,30 @@
+# vim:fileencoding=utf-8
+from __future__ import unicode_literals
+
+__license__   = 'GPL v3'
+__copyright__ = '2012'
+'''
+acrimed.org
+'''
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Acrimed(BasicNewsRecipe):
+    title                  = u'Acrimed'
+    __author__             = 'Gaëtan Lehmann'
+    oldest_article         = 30
+    max_articles_per_feed  = 100
+    auto_cleanup           = True
+    auto_cleanup_keep      = '//div[@class="crayon article-chapo-4112 chapo"]'
+    language               = 'fr'
+    masthead_url           = 'http://www.acrimed.org/IMG/siteon0.gif'
+    feeds                  = [(u'Acrimed', u'http://www.acrimed.org/spip.php?page=backend')]
+
+    preprocess_regexps     = [
+        (re.compile(r'<title>(.*) - Acrimed \| Action Critique M.*dias</title>'), lambda m: '<title>' + m.group(1) + '</title>'),
+        (re.compile(r'<h2>(.*) - Acrimed \| Action Critique M.*dias</h2>'), lambda m: '<h2>' + m.group(1) + '</h2>')]
+
+    extra_css              = """
+                                .chapo{font-style:italic; margin: 1em 0 0.5em}
+                            """

recipes/lamebook.recipe

@@ -13,6 +13,8 @@ class LamebookRecipe(BasicNewsRecipe):
     language = 'en'
     use_embedded_content  = False
     publication_type = 'blog'
+    reverse_article_order = True
+    encoding = 'utf-8'
 
     keep_only_tags = [
         dict(name='div', attrs={'class':'entry'})

src/calibre/db/cache.py

@@ -161,7 +161,8 @@ class Cache(object):
     def _get_metadata(self, book_id, get_user_categories=True):  # {{{
         mi = Metadata(None, template_cache=self.formatter_template_cache)
         author_ids = self._field_ids_for('authors', book_id)
-        aut_list = [self._author_data(i) for i in author_ids]
+        adata = self._author_data(author_ids)
+        aut_list = [adata[i] for i in author_ids]
         aum = []
         aus = {}
         aul = {}
@@ -363,6 +364,9 @@ class Cache(object):
     @read_api
     def all_field_names(self, field):
         ''' Frozen set of all fields names (should only be used for many-one and many-many fields) '''
+        if field == 'formats':
+            return frozenset(self.fields[field].table.col_book_map)
+
         try:
             return frozenset(self.fields[field].table.id_map.itervalues())
         except AttributeError:
@@ -385,17 +389,21 @@ class Cache(object):
             raise ValueError('%s is not a many-one or many-many field' % field)
 
     @read_api
-    def author_data(self, author_id):
+    def get_item_name(self, field, item_id):
+        return self.fields[field].table.id_map[item_id]
+
+    @read_api
+    def author_data(self, author_ids=None):
         '''
         Return author data as a dictionary with keys: name, sort, link
 
-        If no author with the specified id is found an empty dictionary is
+        If no authors with the specified ids are found an empty dictionary is
-        returned.
+        returned. If author_ids is None, data for all authors is returned.
         '''
-        try:
+        af = self.fields['authors']
-            return self.fields['authors'].author_data(author_id)
+        if author_ids is None:
-        except (KeyError, IndexError):
+            author_ids = tuple(af.table.id_map)
-            return {}
+        return {aid:af.author_data(aid) for aid in author_ids if aid in af.table.id_map}
 
     @read_api
     def format_metadata(self, book_id, fmt, allow_cache=True):

src/calibre/db/legacy.py

@@ -6,10 +6,11 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import os, traceback
+import os, traceback, types
 from functools import partial
 from future_builtins import zip
 
+from calibre import force_unicode
 from calibre.db import _get_next_series_num_for_list, _get_series_values
 from calibre.db.adding import (
     find_books_in_directory, import_book_directory_multiple,
@@ -62,17 +63,43 @@ class LibraryDatabase(object):
             setattr(self, prop, partial(self.get_property,
                     loc=self.FIELD_MAP[fm]))
 
+        MT = lambda func: types.MethodType(func, self, LibraryDatabase)
+
         for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'):
             setattr(self, meth, getattr(self.new_api, meth))
 
+        # Legacy API to get information about many-(one, many) fields
         for field in ('authors', 'tags', 'publisher', 'series'):
+            def getter(field):
+                def func(self):
+                    return self.new_api.all_field_names(field)
+                return func
             name = field[:-1] if field in {'authors', 'tags'} else field
-            setattr(self, 'all_%s_names' % name, partial(self.new_api.all_field_names, field))
+            setattr(self, 'all_%s_names' % name, MT(getter(field)))
+            self.all_formats = MT(lambda self:self.new_api.all_field_names('formats'))
 
         for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems():
             setattr(self, func, partial(self.field_id_map, field))
-        self.all_tags = lambda : list(self.all_tag_names())
+        self.all_tags = MT(lambda self: list(self.all_tag_names()))
+        self.get_authors_with_ids = MT(
+            lambda self: [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()])
+        for field in ('tags', 'series', 'publishers', 'ratings', 'languages'):
+            def getter(field):
+                fname = field[:-1] if field in {'publishers', 'ratings'} else field
+                def func(self):
+                    return [[tid, tag] for tid, tag in self.new_api.get_id_map(fname).iteritems()]
+                return func
+            setattr(self, 'get_%s_with_ids' % field,
+                    MT(getter(field)))
+        for field in ('author', 'tag', 'series'):
+            def getter(field):
+                field = field if field == 'series' else (field+'s')
+                def func(self, item_id):
+                    return self.new_api.get_item_name(field, item_id)
+                return func
+            setattr(self, '%s_name' % field, MT(getter(field)))
 
+        # Legacy field API
         for func in (
             'standard_field_keys', 'custom_field_keys', 'all_field_keys',
             'searchable_fields', 'sortable_field_keys',
@@ -82,6 +109,12 @@ class LibraryDatabase(object):
         self.metadata_for_field = self.field_metadata.get
 
         self.last_update_check = self.last_modified()
+        self.book_on_device_func = None
+        # Cleaning is not required anymore
+        self.clean = self.clean_custom = MT(lambda self:None)
+        self.clean_standard_field = MT(lambda self, field, commit=False:None)
+        # apsw operates in autocommit mode
+        self.commit = MT(lambda self:None)
 
     def close(self):
         self.backend.close()
@@ -286,8 +319,71 @@ class LibraryDatabase(object):
         mi = self.new_api.get_metadata(book_id, get_cover=key == 'cover')
         return mi.get(key, default)
 
-    # Private interface {{{
+    def authors_sort_strings(self, index, index_is_id=False):
+        book_id = index if index_is_id else self.data.index_to_id(index)
+        with self.new_api.read_lock:
+            authors = self.new_api._field_ids_for('authors', book_id)
+            adata = self.new_api._author_data(authors)
+            return [adata[aid]['sort'] for aid in authors]
 
+    def author_sort_from_book(self, index, index_is_id=False):
+        return ' & '.join(self.authors_sort_strings(index, index_is_id=index_is_id))
+
+    def authors_with_sort_strings(self, index, index_is_id=False):
+        book_id = index if index_is_id else self.data.index_to_id(index)
+        with self.new_api.read_lock:
+            authors = self.new_api._field_ids_for('authors', book_id)
+            adata = self.new_api._author_data(authors)
+            return [(aid, adata[aid]['name'], adata[aid]['sort'], adata[aid]['link']) for aid in authors]
+
+    def book_on_device(self, book_id):
+        if callable(self.book_on_device_func):
+            return self.book_on_device_func(book_id)
+        return None
+
+    def book_on_device_string(self, book_id):
+        loc = []
+        count = 0
+        on = self.book_on_device(book_id)
+        if on is not None:
+            m, a, b, count = on[:4]
+            if m is not None:
+                loc.append(_('Main'))
+            if a is not None:
+                loc.append(_('Card A'))
+            if b is not None:
+                loc.append(_('Card B'))
+        return ', '.join(loc) + ((_(' (%s books)')%count) if count > 1 else '')
+
+    def set_book_on_device_func(self, func):
+        self.book_on_device_func = func
+
+    def books_in_series(self, series_id):
+        with self.new_api.read_lock:
+            book_ids = self.new_api._books_for_field('series', series_id)
+            ff = self.new_api._field_for
+            return sorted(book_ids, key=lambda x:ff('series_index', x))
+
+    def books_in_series_of(self, index, index_is_id=False):
+        book_id = index if index_is_id else self.data.index_to_id(index)
+        series_ids = self.new_api.field_ids_for('series', book_id)
+        if not series_ids:
+            return []
+        return self.books_in_series(series_ids[0])
+
+    def books_with_same_title(self, mi, all_matches=True):
+        title = mi.title
+        ans = set()
+        if title:
+            title = icu_lower(force_unicode(title))
+            for book_id, x in self.new_api.get_id_map('title').iteritems():
+                if icu_lower(x) == title:
+                    ans.add(book_id)
+                    if not all_matches:
+                        break
+        return ans
+
+    # Private interface {{{
     def __iter__(self):
         for row in self.data.iterall():
             yield row

src/calibre/db/tests/legacy.py

@@ -32,7 +32,11 @@ class ET(object):
         return newres
 
 def compare_argspecs(old, new, attr):
-    ok = len(old.args) == len(new.args) and len(old.defaults or ()) == len(new.defaults or ()) and old.args[-len(old.defaults or ()):] == new.args[-len(new.defaults or ()):]  # noqa
+    # We dont compare the names of the non-keyword arguments as they are often
+    # different and they dont affect the usage of the API.
+    num = len(old.defaults or ())
+
+    ok = len(old.args) == len(new.args) and old.defaults == new.defaults and (num == 0 or old.args[-num:] == new.args[-num:])
     if not ok:
         raise AssertionError('The argspec for %s does not match. %r != %r' % (attr, old, new))
 
@@ -143,12 +147,12 @@ class LegacyTest(BaseTest):
             'all_tag_names':[()],
             'all_series_names':[()],
             'all_publisher_names':[()],
-            'all_authors':[()],
+            '!all_authors':[()],
-            'all_tags2':[()],
+            '!all_tags2':[()],
-            'all_tags':[()],
+            '@all_tags':[()],
-            'all_publishers':[()],
+            '!all_publishers':[()],
-            'all_titles':[()],
+            '!all_titles':[()],
-            'all_series':[()],
+            '!all_series':[()],
             'standard_field_keys':[()],
             'all_field_keys':[()],
             'searchable_fields':[()],
@@ -156,15 +160,32 @@ class LegacyTest(BaseTest):
             'metadata_for_field':[('title',), ('tags',)],
             'sortable_field_keys':[()],
             'custom_field_keys':[(True,), (False,)],
-            'get_usage_count_by_id':[('authors',), ('tags',), ('series',), ('publisher',), ('#tags',), ('languages',)],
+            '!get_usage_count_by_id':[('authors',), ('tags',), ('series',), ('publisher',), ('#tags',), ('languages',)],
             'get_field':[(1, 'title'), (2, 'tags'), (0, 'rating'), (1, 'authors'), (2, 'series'), (1, '#tags')],
+            'all_formats':[()],
+            'get_authors_with_ids':[()],
+            '!get_tags_with_ids':[()],
+            '!get_series_with_ids':[()],
+            '!get_publishers_with_ids':[()],
+            '!get_ratings_with_ids':[()],
+            '!get_languages_with_ids':[()],
+            'tag_name':[(3,)],
+            'author_name':[(3,)],
+            'series_name':[(3,)],
+            'authors_sort_strings':[(0,), (1,), (2,)],
+            'author_sort_from_book':[(0,), (1,), (2,)],
+            'authors_with_sort_strings':[(0,), (1,), (2,)],
+            'book_on_device_string':[(1,), (2,), (3,)],
+            'books_in_series_of':[(0,), (1,), (2,)],
+            'books_with_same_title':[(Metadata(db.title(0)),), (Metadata(db.title(1)),), (Metadata('1234'),)],
         }.iteritems():
             for a in args:
                 fmt = lambda x: x
-                if meth in {'get_usage_count_by_id', 'all_series', 'all_authors', 'all_tags2', 'all_publishers', 'all_titles'}:
+                if meth[0] in {'!', '@'}:
-                    fmt = dict
+                    fmt = {'!':dict, '@':frozenset}[meth[0]]
-                elif meth in {'all_tags'}:
+                    meth = meth[1:]
-                    fmt = frozenset
+                elif meth == 'get_authors_with_ids':
+                    fmt = lambda val:{x[0]:tuple(x[1:]) for x in val}
                 self.assertEqual(fmt(getattr(db, meth)(*a)), fmt(getattr(ndb, meth)(*a)),
                                  'The method: %s() returned different results for argument %s' % (meth, a))
         db.close()
@@ -242,9 +263,16 @@ class LegacyTest(BaseTest):
             '_set_title', '_set_custom', '_update_author_in_cache',
             # Feeds are now stored in the config folder
             'get_feeds', 'get_feed', 'update_feed', 'remove_feeds', 'add_feed', 'set_feeds',
+            # Obsolete/broken methods
+            'author_id',  # replaced by get_author_id
+            'books_for_author',  # broken
+            'books_in_old_database',  # unused
+
+            # Internal API
+            'clean_user_categories',  'cleanup_tags',  'books_list_filter',
         }
         SKIP_ARGSPEC = {
-            '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors', 'all_tags',
+            '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors',
         }
 
         missing = []
@@ -309,3 +337,4 @@ class LegacyTest(BaseTest):
         T(('n', object()))
         old.close()
     # }}}
+

src/calibre/db/view.py

@@ -189,10 +189,8 @@ class View(object):
         id_ = idx if index_is_id else self.index_to_id(idx)
         with self.cache.read_lock:
             ids = self.cache._field_ids_for('authors', id_)
-            ans = []
+            adata = self.cache._author_data(ids)
-            for id_ in ids:
+            ans = [':::'.join((adata[aid]['name'], adata[aid]['sort'], adata[aid]['link'])) for aid in ids if aid in adata]
-                data = self.cache._author_data(id_)
-                ans.append(':::'.join((data['name'], data['sort'], data['link'])))
         return ':#:'.join(ans) if ans else default_value
 
     def multisort(self, fields=[], subsort=False, only_ids=None):

src/calibre/devices/mtp/driver.py

@@ -83,7 +83,7 @@ class MTP_DEVICE(BASE):
         return name in {
             'alarms', 'android', 'dcim', 'movies', 'music', 'notifications',
             'pictures', 'ringtones', 'samsung', 'sony', 'htc', 'bluetooth',
-            'games', 'lost.dir', 'video', 'whatsapp', 'image'}
+            'games', 'lost.dir', 'video', 'whatsapp', 'image', 'com.zinio.mobile.android.reader'}
 
     def configure_for_kindle_app(self):
         proxy = self.prefs
@@ -161,7 +161,8 @@ class MTP_DEVICE(BASE):
             self.driveinfo = {}
             for sid, location_code in ((self._main_id, 'main'), (self._carda_id,
                 'A'), (self._cardb_id, 'B')):
-                if sid is None: continue
+                if sid is None:
+                    continue
                 self._update_drive_info(self.filesystem_cache.storage(sid), location_code)
         return self.driveinfo
 
@@ -470,7 +471,8 @@ class MTP_DEVICE(BASE):
 
     def remove_books_from_metadata(self, paths, booklists):
         self.report_progress(0, _('Removing books from metadata'))
-        class NextPath(Exception): pass
+        class NextPath(Exception):
+            pass
 
         for i, path in enumerate(paths):
             try:
@@ -549,3 +551,4 @@ if __name__ == '__main__':
         dev.shutdown()
 
 
+