Merge from trunk

resources/recipes/science_based_medicine.recipe

@@ -0,0 +1,42 @@
+#!/usr/bin/env  python
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Tag
+
+class SBM(BasicNewsRecipe):
+    title                 = 'Science Based Medicine'
+    __author__            = 'BuzzKill'
+    description           = 'Exploring issues and controversies in the relationship between science and medicine'
+    oldest_article        = 5
+    max_articles_per_feed = 15
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    publisher             = 'SBM'
+    category              = 'science, sbm, ebm, blog, pseudoscience'
+    language              = 'en'
+
+    lang                  = 'en-US'
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : lang
+                        , 'pretty_print'     : True
+                        }
+
+    keep_only_tags = [
+                      dict(name='a', attrs={'title':re.compile(r'Posts by.*', re.DOTALL|re.IGNORECASE)}),
+                      dict(name='div', attrs={'class':'entry'})
+                      ]
+
+    feeds = [(u'Science Based Medicine', u'http://www.sciencebasedmedicine.org/?feed=rss2')]
+
+    def preprocess_html(self, soup):
+        mtag = Tag(soup,'meta',[('http-equiv','Content-Type'),('context','text/html; charset=utf-8')])
+        soup.head.insert(0,mtag)
+        soup.html['lang'] = self.lang
+        return self.adeify_images(soup)
+

resources/recipes/zeitde.recipe

@@ -60,8 +60,8 @@ class ZeitDe(BasicNewsRecipe):
         for tag in soup.findAll(name=['ul','li']):
             tag.name = 'div'
 
-        soup.html['xml:lang'] = self.lang
+        soup.html['xml:lang'] = self.language.replace('_', '-')
-        soup.html['lang']     = self.lang
+        soup.html['lang']     = self.language.replace('_', '-')
         mtag = '<meta http-equiv="Content-Type" content="text/html; charset=' + self.encoding + '">'
         soup.head.insert(0,mtag)
         return soup

src/calibre/devices/errors.py

@@ -36,6 +36,11 @@ class UserFeedback(DeviceError):
        self.details = details
        self.msg = msg
 
+class OpenFeedback(DeviceError):
+    def __init__(self, msg):
+        self.feedback_msg = msg
+        DeviceError.__init__(self, msg)
+
 class DeviceBusy(ProtocolError):
     """ Raised when device is busy """
     def __init__(self, uerr=""):

src/calibre/devices/interface.py

@@ -216,6 +216,9 @@ class DevicePlugin(Plugin):
         an implementation of
         this function that should serve as a good example for USB Mass storage
         devices.
+
+        This method can raise an OpenFeedback exception to display a message to
+        the user.
         '''
         raise NotImplementedError()
 

src/calibre/gui2/device.py

@@ -12,7 +12,7 @@ from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, \
 from calibre.customize.ui import available_input_formats, available_output_formats, \
     device_plugins
 from calibre.devices.interface import DevicePlugin
-from calibre.devices.errors import UserFeedback
+from calibre.devices.errors import UserFeedback, OpenFeedback
 from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
 from calibre.utils.ipc.job import BaseJob
 from calibre.devices.scanner import DeviceScanner
@@ -122,7 +122,8 @@ def device_name_for_plugboards(device_class):
 
 class DeviceManager(Thread): # {{{
 
-    def __init__(self, connected_slot, job_manager, open_feedback_slot, sleep_time=2):
+    def __init__(self, connected_slot, job_manager, open_feedback_slot,
+            open_feedback_msg, sleep_time=2):
         '''
         :sleep_time: Time to sleep between device probes in secs
         '''
@@ -143,6 +144,7 @@ class DeviceManager(Thread): # {{{
         self.ejected_devices  = set([])
         self.mount_connection_requests = Queue.Queue(0)
         self.open_feedback_slot = open_feedback_slot
+        self.open_feedback_msg = open_feedback_msg
 
     def report_progress(self, *args):
         pass
@@ -163,6 +165,9 @@ class DeviceManager(Thread): # {{{
                 dev.reset(detected_device=detected_device,
                     report_progress=self.report_progress)
                 dev.open()
+            except OpenFeedback, e:
+                self.open_feedback_msg(dev.get_gui_name(), e.feedback_msg)
+                continue
             except:
                 tb = traceback.format_exc()
                 if DEBUG or tb not in self.reported_errors:
@@ -594,11 +599,16 @@ class DeviceMixin(object): # {{{
                 _('Error communicating with device'), ' ')
         self.device_error_dialog.setModal(Qt.NonModal)
         self.device_manager = DeviceManager(Dispatcher(self.device_detected),
-                self.job_manager, Dispatcher(self.status_bar.show_message))
+                self.job_manager, Dispatcher(self.status_bar.show_message),
+                Dispatcher(self.show_open_feedback))
         self.device_manager.start()
         if tweaks['auto_connect_to_folder']:
             self.connect_to_folder_named(tweaks['auto_connect_to_folder'])
 
+    def show_open_feedback(self, devname, msg):
+        self.__of_dev_mem__ = d = info_dialog(self, devname, msg)
+        d.show()
+
     def auto_convert_question(self, msg, autos):
         autos = u'\n'.join(map(unicode, map(force_unicode, autos)))
         return self.ask_a_yes_no_question(

src/calibre/gui2/tag_view.py

@@ -101,6 +101,7 @@ class TagsView(QTreeView): # {{{
                                 hidden_categories=self.hidden_categories,
                                 search_restriction=None,
                                 drag_drop_finished=self.drag_drop_finished)
+        self.pane_is_visible = True # because TagsModel.init did a recount
         self.sort_by = sort_by
         self.tag_match = tag_match
         self.db = db

src/calibre/gui2/ui.py

@@ -234,6 +234,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
 
         ######################### Search Restriction ##########################
         SearchRestrictionMixin.__init__(self)
+        if db.prefs['gui_restriction']:
             self.apply_named_search_restriction(db.prefs['gui_restriction'])
 
         ########################### Cover Flow ################################

src/calibre/library/database2.py

@@ -10,11 +10,10 @@ import os, sys, shutil, cStringIO, glob, time, functools, traceback, re
 from itertools import repeat
 from math import floor
 from Queue import Queue
-from operator import itemgetter
 
 from PyQt4.QtGui import QImage
 
-
+from calibre import prints
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.library.database import LibraryDatabase
@@ -1039,25 +1038,142 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     tn=field['table'], col=field['link_column']), (id_,))
         return set(x[0] for x in ans)
 
+########## data structures for get_categories
+
     CATEGORY_SORTS = ('name', 'popularity', 'rating')
 
-    def get_categories(self, sort='name', ids=None, icon_map=None):
+    class TCat_Tag(object):
-        self.books_list_filter.change([] if not ids else ids)
 
-        categories = {}
+        def __init__(self, name, sort):
+            self.n = name
+            self.s = sort
+            self.c  = 0
+            self.rt = 0
+            self.rc = 0
+            self.id = None
+
+        def set_all(self, c, rt, rc, id):
+            self.c  = c
+            self.rt = rt
+            self.rc = rc
+            self.id = id
+
+        def __str__(self):
+            return unicode(self)
+
+        def __unicode__(self):
+            return 'n=%s s=%s c=%d rt=%d rc=%d id=%s'%\
+                            (self.n, self.s, self.c, self.rt, self.rc, self.id)
+
+
+    def get_categories(self, sort='name', ids=None, icon_map=None):
+        #start = last = time.clock()
         if icon_map is not None and type(icon_map) != TagsIcons:
             raise TypeError('icon_map passed to get_categories must be of type TagIcons')
+        if sort not in self.CATEGORY_SORTS:
+            raise ValueError('sort ' + sort + ' not a valid value')
+
+        self.books_list_filter.change([] if not ids else ids)
+        id_filter = None if not ids else frozenset(ids)
 
         tb_cats = self.field_metadata
-        #### First, build the standard and custom-column categories ####
+        tcategories = {}
+        tids = {}
+        md = []
+
+        # First, build the maps. We need a category->items map and an
+        # item -> (item_id, sort_val) map to use in the books loop
         for category in tb_cats.keys():
             cat = tb_cats[category]
-            if not cat['is_category'] or cat['kind'] in ['user', 'search']:
+            if not cat['is_category'] or cat['kind'] in ['user', 'search'] \
+                    or category in ['news', 'formats']:
                 continue
+            # Get the ids for the item values
+            if not cat['is_custom']:
+                funcs = {
+                        'authors'  : self.get_authors_with_ids,
+                        'series'   : self.get_series_with_ids,
+                        'publisher': self.get_publishers_with_ids,
+                        'tags'     : self.get_tags_with_ids,
+                        'rating'   : self.get_ratings_with_ids,
+                    }
+                func = funcs.get(category, None)
+                if func:
+                    list = func()
+                else:
+                    raise ValueError(category + ' has no get with ids function')
+            else:
+                list = self.get_custom_items_with_ids(label=cat['label'])
+            tids[category] = {}
+            if category == 'authors':
+                for l in list:
+                    (id, val, sort_val) = (l[0], l[1], l[2])
+                    tids[category][val] = (id, sort_val)
+            else:
+                for l in list:
+                    (id, val) = (l[0], l[1])
+                    tids[category][val] = (id, val)
+            # add an empty category to the category map
+            tcategories[category] = {}
+            # create a list of category/field_index for the books scan to use.
+            # This saves iterating through field_metadata for each book
+            md.append((category, cat['rec_index'], cat['is_multiple']))
+
+        #print 'end phase "collection":', time.clock() - last, 'seconds'
+        #last = time.clock()
+
+        # Now scan every book looking for category items.
+        # Code below is duplicated because it shaves off 10% of the loop time
+        id_dex = self.FIELD_MAP['id']
+        rating_dex = self.FIELD_MAP['rating']
+        tag_class = LibraryDatabase2.TCat_Tag
+        for book in self.data.iterall():
+            if id_filter and book[id_dex] not in id_filter:
+                continue
+            rating = book[rating_dex]
+            # We kept track of all possible category field_map positions above
+            for (cat, dex, mult) in md:
+                if book[dex] is None:
+                    continue
+                if not mult:
+                    val = book[dex]
+                    try:
+                        (item_id, sort_val) = tids[cat][val] # let exceptions fly
+                        item = tcategories[cat].get(val, None)
+                        if not item:
+                            item = tag_class(val, sort_val)
+                            tcategories[cat][val] = item
+                        item.c += 1
+                        item.id = item_id
+                        if rating > 0:
+                            item.rt += rating
+                            item.rc += 1
+                    except:
+                        prints('get_categories: item', val, 'is not in', cat, 'list!')
+                else:
+                    vals = book[dex].split(mult)
+                    for val in vals:
+                        try:
+                            (item_id, sort_val) = tids[cat][val] # let exceptions fly
+                            item = tcategories[cat].get(val, None)
+                            if not item:
+                                item = tag_class(val, sort_val)
+                                tcategories[cat][val] = item
+                            item.c += 1
+                            item.id = item_id
+                            if rating > 0:
+                                item.rt += rating
+                                item.rc += 1
+                        except:
+                            prints('get_categories: item', val, 'is not in', cat, 'list!')
+
+        #print 'end phase "books":', time.clock() - last, 'seconds'
+        #last = time.clock()
+
+        # Now do news
+        tcategories['news'] = {}
+        cat = tb_cats['news']
         tn = cat['table']
-            categories[category] = []   #reserve the position in the ordered list
-            if tn is None:              # Nothing to do for the moment
-                continue
         cn = cat['column']
         if ids is None:
             query = '''SELECT id, {0}, count, avg_rating, sort
@@ -1065,17 +1181,32 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         else:
             query = '''SELECT id, {0}, count, avg_rating, sort
                        FROM tag_browser_filtered_{1}'''.format(cn, tn)
-            if sort == 'popularity':
+        # results will be sorted later
-                query += ' ORDER BY count DESC, sort ASC'
-            elif sort == 'name':
-                query += ' ORDER BY sort COLLATE icucollate'
-            else:
-                query += ' ORDER BY avg_rating DESC, sort ASC'
         data = self.conn.get(query)
+        for r in data:
+            item = LibraryDatabase2.TCat_Tag(r[1], r[1])
+            item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0])
+            tcategories['news'][r[1]] = item
+
+        #print 'end phase "news":', time.clock() - last, 'seconds'
+        #last = time.clock()
+
+        # Build the real category list by iterating over the temporary copy
+        # and building the Tag instances.
+        categories = {}
+        tag_class = Tag
+        for category in tb_cats.keys():
+            if category not in tcategories:
+                continue
+            cat = tb_cats[category]
+
+            # prepare the place where we will put the array of Tags
+            categories[category] = []
 
             # icon_map is not None if get_categories is to store an icon and
             # possibly a tooltip in the tag structure.
-            icon, tooltip = None, ''
+            icon = None
+            tooltip = ''
             label = tb_cats.key_to_label(category)
             if icon_map:
                 if not tb_cats.is_custom_field(category):
@@ -1087,23 +1218,46 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     tooltip = self.custom_column_label_map[label]['name']
 
             datatype = cat['datatype']
-            avgr = itemgetter(3)
+            avgr = lambda x: 0.0 if x.rc == 0 else x.rt/x.rc
-            item_not_zero_func = lambda x: x[2] > 0
+            # Duplicate the build of items below to avoid using a lambda func
+            # in the main Tag loop. Saves a few %
             if datatype == 'rating':
-                # eliminate the zero ratings line as well as count == 0
-                item_not_zero_func = (lambda x: x[1] > 0 and x[2] > 0)
                 formatter = (lambda x:u'\u2605'*int(x/2))
-                avgr = itemgetter(1)
+                avgr = lambda x : x.n
+                # eliminate the zero ratings line as well as count == 0
+                items = [v for v in tcategories[category].values() if v.c > 0 and v.n != 0]
             elif category == 'authors':
                 # Clean up the authors strings to human-readable form
                 formatter = (lambda x: x.replace('|', ','))
+                items = [v for v in tcategories[category].values() if v.c > 0]
             else:
                 formatter = (lambda x:unicode(x))
+                items = [v for v in tcategories[category].values() if v.c > 0]
 
-            categories[category] = [Tag(formatter(r[1]), count=r[2], id=r[0],
+            # sort the list
-                                        avg=avgr(r), sort=r[4], icon=icon,
+            if sort == 'name':
+                def get_sort_key(x):
+                    sk = x.s
+                    if isinstance(sk, unicode):
+                        sk = sort_key(sk)
+                    return sk
+                kf = get_sort_key
+                reverse=False
+            elif sort == 'popularity':
+                kf = lambda x: x.c
+                reverse=True
+            else:
+                kf = avgr
+                reverse=True
+            items.sort(key=kf, reverse=reverse)
+
+            categories[category] = [tag_class(formatter(r.n), count=r.c, id=r.id,
+                                        avg=avgr(r), sort=r.s, icon=icon,
                                         tooltip=tooltip, category=category)
-                                    for r in data if item_not_zero_func(r)]
+                                    for r in items]
+
+        #print 'end phase "tags list":', time.clock() - last, 'seconds'
+        #last = time.clock()
 
         # Needed for legacy databases that have multiple ratings that
         # map to n stars
@@ -1189,8 +1343,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 icon_map['search'] = icon_map['search']
             categories['search'] = items
 
+        #print 'last phase ran in:', time.clock() - last, 'seconds'
+        #print 'get_categories ran in:', time.clock() - start, 'seconds'
+
         return categories
 
+    ############# End get_categories
+
     def tags_older_than(self, tag, delta):
         tag = tag.lower().strip()
         now = nowf()
@@ -1486,6 +1645,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     # Note: we generally do not need to refresh_ids because library_view will
     # refresh everything.
 
+    def get_ratings_with_ids(self):
+        result = self.conn.get('SELECT id,rating FROM ratings')
+        if not result:
+            return []
+        return result
+
     def dirty_books_referencing(self, field, id, commit=True):
         # Get the list of books to dirty -- all books that reference the item
         table = self.field_metadata[field]['table']