Merge from trunk

recipes/bild_de.recipe

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 from calibre.web.feeds.recipes import BasicNewsRecipe
 class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
     title          = u'Bild.de'
     __author__  = 'schuster'
     oldest_article = 1
-    max_articles_per_feed = 50
+    max_articles_per_feed = 100
     no_stylesheets         = True
     use_embedded_content   = False
     language               = 'de'
@@ -12,11 +13,25 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
 # get cover from myspace
     cover_url = 'http://a3.l3-images.myspacecdn.com/images02/56/0232f842170b4d349779f8379c27e073/l.jpg'
+    masthead_url = 'http://a3.l3-images.myspacecdn.com/images02/56/0232f842170b4d349779f8379c27e073/l.jpg'
 
 # set what to fetch on the site
     remove_tags_before =  dict(name = 'h2', attrs={'id':'cover'})
     remove_tags_after = dict(name ='div', attrs={'class':'back'})
 
+
+# remove things on the site that we don't want
+    remove_tags = [dict(name='div', attrs={'class':'credit'}),
+        dict(name='div', attrs={'class':'index'}),
+        dict(name='div', attrs={'id':'zstart31'}),
+        dict(name='div', attrs={'class':'hentry'}),
+        dict(name='div', attrs={'class':'back'}),
+        dict(name='div', attrs={'class':'pagination'}),
+        dict(name='div', attrs={'class':'header'}),
+        dict(name='div', attrs={'class':'element floatL'}),
+        dict(name='div', attrs={'class':'stWrap'})
+]
+
 #  thanx to kiklop74 for code (see sticky thread -> Recipes - Re-usable code)
 # this one removes a lot of direct-link's
     def preprocess_html(self, soup):
@@ -42,5 +57,18 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
                           (u'Unterhaltung', u'http://rss.bild.de/bild-unterhaltung.xml'),
                           (u'Sport', u'http://rss.bild.de/bild-sport.xml'),
                           (u'Lifestyle', u'http://rss.bild.de/bild-lifestyle.xml'),
-                          (u'Ratgeber', u'http://rss.bild.de/bild-ratgeber.xml')
+                          (u'Ratgeber', u'http://rss.bild.de/bild-ratgeber.xml'),
+                          (u'Reg. - Berlin', u'http://rss.bild.de/bild-berlin.xml'),
+                          (u'Reg. - Bremen', u'http://rss.bild.de/bild-bremen.xml'),
+                          (u'Reg. - Dresden', u'http://rss.bild.de/bild-dresden.xml'),
+                          (u'Reg. - Düsseldorf', u'http://rss.bild.de/bild-duesseldorf.xml'),
+                          (u'Reg. - Frankfurt-Main', u'http://rss.bild.de/bild-frankfurt-main.xml'),
+                          (u'Reg. - Hamburg', u'http://rss.bild.de/bild-hamburg.xml'),
+                          (u'Reg. - Hannover', u'http://rss.bild.de/bild-hannover.xml'),
+                          (u'Reg. - Köln', u'http://rss.bild.de/bild-koeln.xml'),
+                          (u'Reg. - Leipzig', u'http://rss.bild.de/bild-leipzig.xml'),
+                          (u'Reg. - München', u'http://rss.bild.de/bild-muenchen.xml'),
+                          (u'Reg. - Ruhrgebiet', u'http://rss.bild.de/bild-ruhrgebiet.xml'),
+                          (u'Reg. - Stuttgart', u'http://rss.bild.de/bild-stuttgart.xml')
 ]
+

recipes/guardian.recipe

@@ -28,7 +28,7 @@ class Guardian(BasicNewsRecipe):
     # List of section titles to ignore
     # For example: ['Sport']
     ignore_sections = []
-
+    
     timefmt = ' [%a, %d %b %Y]'
     keep_only_tags = [
                       dict(name='div', attrs={'id':["content","article_header","main-article-info",]}),
@@ -87,8 +87,14 @@ class Guardian(BasicNewsRecipe):
         idx = soup.find('div', id='book-index')
         for s in idx.findAll('strong', attrs={'class':'book'}):
             a = s.find('a', href=True)
-            yield (self.tag_to_string(a), a['href'])
+            section_title = self.tag_to_string(a)
-
+            if not section_title in self.ignore_sections:
+                prefix = ''
+                if section_title != 'Main section':
+                    prefix = section_title + ': '
+                for subsection in s.parent.findAll('a', attrs={'class':'book-section'}):
+                    yield (prefix + self.tag_to_string(subsection), subsection['href'])
+    
     def find_articles(self, url):
         soup = self.index_to_soup(url)
         div = soup.find('div', attrs={'class':'book-index'})
@@ -109,15 +115,12 @@ class Guardian(BasicNewsRecipe):
                         'title': title, 'url':url, 'description':desc,
                         'date' : strftime('%a, %d %b'),
                         }
-
+    
     def parse_index(self):
         try:
             feeds = []
             for title, href in self.find_sections():
-                if not title in self.ignore_sections:
+                feeds.append((title, list(self.find_articles(href))))
-                    feeds.append((title, list(self.find_articles(href))))
             return feeds
         except:
             raise NotImplementedError
-
-

recipes/telegraph_uk.recipe

@@ -49,6 +49,7 @@ class TelegraphUK(BasicNewsRecipe):
                          (u'UK News'        , u'http://www.telegraph.co.uk/news/uknews/rss'                                      )
                         ,(u'World News'     , u'http://www.telegraph.co.uk/news/worldnews/rss'                                   )
                         ,(u'Politics'       , u'http://www.telegraph.co.uk/news/newstopics/politics/rss'                         )
+                        ,(u'Finance'        , u'http://www.telegraph.co.uk/finance/rss'                                          )
                         ,(u'Technology News', u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologynews/rss'   )
                         ,(u'UK News'        , u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologyreviews/rss')
                         ,(u'Science News'   , u'http://www.telegraph.co.uk/scienceandtechnology/science/sciencenews/rss'         )

recipes/time_magazine.recipe

@@ -10,8 +10,8 @@ import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Time(BasicNewsRecipe):
-    recipe_disabled = ('This recipe has been disabled as TIME no longer'
+    #recipe_disabled = ('This recipe has been disabled as TIME no longer'
-            ' publish complete articles on the web.')
+    #        ' publish complete articles on the web.')
     title                 = u'Time'
     __author__            = 'Kovid Goyal and Sujata Raman'
     description           = 'Weekly magazine'

recipes/zaobao.recipe

@@ -82,7 +82,7 @@ class ZAOBAO(BasicNewsRecipe):
         return soup
 
     def parse_feeds(self):
-        self.log_debug(_('ZAOBAO overrided parse_feeds()'))
+        self.log(_('ZAOBAO overrided parse_feeds()'))
         parsed_feeds = BasicNewsRecipe.parse_feeds(self)
 
         for id, obj in enumerate(self.INDEXES):
@@ -99,7 +99,7 @@ class ZAOBAO(BasicNewsRecipe):
                     a_title = self.tag_to_string(a)
                     date = ''
                     description = ''
-                    self.log_debug(_('adding %s at %s')%(a_title,a_url))
+                    self.log(_('adding %s at %s')%(a_title,a_url))
                     articles.append({
                                     'title':a_title,
                                     'date':date,
@@ -110,23 +110,23 @@ class ZAOBAO(BasicNewsRecipe):
             pfeeds = feeds_from_index([(title, articles)], oldest_article=self.oldest_article,
                                      max_articles_per_feed=self.max_articles_per_feed)
 
-            self.log_debug(_('adding %s to feed')%(title))
+            self.log(_('adding %s to feed')%(title))
             for feed in pfeeds:
-                self.log_debug(_('adding feed: %s')%(feed.title))
+                self.log(_('adding feed: %s')%(feed.title))
                 feed.description = self.DESC_SENSE
                 parsed_feeds.append(feed)
                 for a, article in enumerate(feed):
-                    self.log_debug(_('added article %s from %s')%(article.title, article.url))
+                    self.log(_('added article %s from %s')%(article.title, article.url))
-                self.log_debug(_('added feed %s')%(feed.title))
+                self.log(_('added feed %s')%(feed.title))
 
         for i, feed in enumerate(parsed_feeds):
             # workaorund a strange problem: Somethimes the xml encoding is not apllied correctly by parse()
             weired_encoding_detected = False
             if not isinstance(feed.description, unicode) and self.encoding and feed.description:
-                self.log_debug(_('Feed %s is not encoded correctly, manually replace it')%(feed.title))
+                self.log(_('Feed %s is not encoded correctly, manually replace it')%(feed.title))
                 feed.description = feed.description.decode(self.encoding, 'replace')
             elif feed.description.find(self.DESC_SENSE) == -1 and self.encoding and feed.description:
-                self.log_debug(_('Feed %s is weired encoded, manually redo all')%(feed.title))
+                self.log(_('Feed %s is weired encoded, manually redo all')%(feed.title))
                 feed.description = feed.description.encode('cp1252', 'replace').decode(self.encoding, 'replace')
                 weired_encoding_detected = True
 
@@ -148,7 +148,7 @@ class ZAOBAO(BasicNewsRecipe):
                         article.text_summary = article.text_summary.encode('cp1252', 'replace').decode(self.encoding, 'replace')
 
                 if article.title == "Untitled article":
-                    self.log_debug(_('Removing empty article %s from %s')%(article.title, article.url))
+                    self.log(_('Removing empty article %s from %s')%(article.title, article.url))
                     # remove the article
                     feed.articles[a:a+1] = []
         return parsed_feeds

resources/default_tweaks.py

@@ -77,7 +77,7 @@ categories_use_field_for_author_name = 'author'
 # ascii ordering. This tweak controls when that switch happens. Set it to zero
 # to always use ascii ordering. Set it to something larger than zero to switch
 # to ascii ordering for performance reasons.
-completion_change_to_ascii_sorting = 1000
+completion_change_to_ascii_sorting = 2500
 
 #: Control partitioning of Tag Browser
 # When partitioning the tags browser, the format of the subcategory label is

src/calibre/devices/apple/driver.py

@@ -2392,6 +2392,16 @@ class ITUNES(DriverBase):
                 self.iTunes.Windows[0].Minimized = True
             self.initial_status = 'launched'
 
+            try:
+                # Pre-emptive test to confirm functional iTunes automation interface
+                foo = self.iTunes.Version
+                foo
+            except:
+                self.iTunes = None
+                raise OpenFeedback('Unable to connect to iTunes.\n' +
+                             ' iTunes automation interface non-responsive, ' +
+                             'recommend reinstalling iTunes')
+
             # Read the current storage path for iTunes media from the XML file
             media_dir = ''
             string = None
@@ -2988,7 +2998,6 @@ class ITUNES(DriverBase):
             newmi = book
         return newmi
 
-
 class ITUNES_ASYNC(ITUNES):
     '''
     This subclass allows the user to interact directly with iTunes via a menu option

src/calibre/devices/misc.py

@@ -224,13 +224,16 @@ class TREKSTOR(USBMS):
     FORMATS     = ['epub', 'txt', 'pdf']
 
     VENDOR_ID   = [0x1e68]
-    PRODUCT_ID  = [0x0041, 0x0042]
+    PRODUCT_ID  = [0x0041, 0x0042,
+            0x003e # This is for the EBOOK_PLAYER_5M https://bugs.launchpad.net/bugs/792091
+            ]
     BCD         = [0x0002]
 
     EBOOK_DIR_MAIN = 'Ebooks'
 
     VENDOR_NAME = 'TREKSTOR'
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_PLAYER_7'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['EBOOK_PLAYER_7',
+            'EBOOK_PLAYER_5M']
 
 class EEEREADER(USBMS):
 

src/calibre/gui2/complete.py

@@ -12,20 +12,23 @@ from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
 from calibre.gui2.widgets import EnComboBox, LineEditECM
-from calibre.utils.config import tweaks
+from calibre.utils.config_base import tweaks
 
 class CompleteModel(QAbstractListModel):
 
     def __init__(self, parent=None):
         QAbstractListModel.__init__(self, parent)
         self.items = []
+        self.sorting = QCompleter.UnsortedModel
 
     def set_items(self, items):
         items = [unicode(x.strip()) for x in items]
         if len(items) < tweaks['completion_change_to_ascii_sorting']:
-            self.items = list(sorted(items, key=lambda x: sort_key(x)))
+            self.items = sorted(items, key=lambda x: sort_key(x))
+            self.sorting = QCompleter.UnsortedModel
         else:
-            self.items = list(sorted(items, key=lambda x: x))
+            self.items = sorted(items, key=lambda x:x.lower())
+            self.sorting = QCompleter.CaseInsensitivelySortedModel
         self.lowered_items = [lower(x) for x in self.items]
         self.reset()
 
@@ -66,6 +69,7 @@ class MultiCompleteLineEdit(QLineEdit, LineEditECM):
         c.setWidget(self)
         c.setCompletionMode(QCompleter.PopupCompletion)
         c.setCaseSensitivity(Qt.CaseInsensitive)
+        c.setModelSorting(self._model.sorting)
         c.setCompletionRole(Qt.DisplayRole)
         p = c.popup()
         p.setMouseTracking(True)
@@ -78,10 +82,6 @@ class MultiCompleteLineEdit(QLineEdit, LineEditECM):
 
     # Interface {{{
     def update_items_cache(self, complete_items):
-        if len(complete_items) < tweaks['completion_change_to_ascii_sorting']:
-            self._completer.setModelSorting(QCompleter.UnsortedModel)
-        else:
-            self._completer.setModelSorting(QCompleter.CaseInsensitivelySortedModel)
         self.all_items = complete_items
 
     def set_separator(self, sep):
@@ -153,6 +153,7 @@ class MultiCompleteLineEdit(QLineEdit, LineEditECM):
             return self._model.items
         def fset(self, items):
             self._model.set_items(items)
+            self._completer.setModelSorting(self._model.sorting)
         return property(fget=fget, fset=fset)
 
 class MultiCompleteComboBox(EnComboBox):

src/calibre/library/database2.py

@@ -595,7 +595,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 if f is None:
                     continue
                 with tempfile.SpooledTemporaryFile(max_size=100*(1024**2)) as stream:
-                    shutil.copyfileobj(f, stream)
+                    with f:
+                        shutil.copyfileobj(f, stream)
                     stream.seek(0)
                     self.add_format(id, format, stream, index_is_id=True,
                         path=tpath, notify=False)