Sync to trunk.

Changelog.yaml

@@ -19,6 +19,68 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.3
+  date: 2011-05-27
+
+  new features:
+    - title: "Allow the coloring of columns in the book list."
+      description: "You can either create a custom column with a fixed set of values and assign a color to each value, or you can use the calibre template language to color any column in arbitrarily powerful ways. For example, you can have the title appear in red if the book has a particular tag."
+      type: major
+
+    - title: "Support for the Nook Simple Reader"
+
+    - title: "Get Books, new stores: Virtualo, lulu.net"
+
+    - title: "A store chooser dialog for Get Books (click the little preferences icon at the bottom of the Get Books screen)."
+
+    - title: "Add a merge_lists, and, or, not template functions to the calibre template language"
+
+    - title: "EPUB Output: Change any white-space:pre declarations in the CSS to pre-wrap to accomodate readers that cannot scroll horizontally."
+      tickets: [786722]
+
+    - title: "Windows installer: Remember and use previous installation folder when upgrading. Note that this will work for future upgrades, after this one."
+  
+  bug fixes:
+    - title: "MOBI Output: Fix hidden tags with id attributes also hiding their trailing text"
+      tickets: [788570]
+
+    - title: "Fix switching from one news source to another via a search not saving changes to the scheduling of the first source"
+      tickets: [774849]
+
+    - title: "Dont allow user to use non email usernames when setting up Hotmail or Gmail accounts"
+
+    - title: "Amazon metadata download: Use separate identifiers for country specific downloads so that the links to Amazon in the Book details panel work when downloading metadata from country specific amazon websites."
+      tickets: [786146]
+
+    - title: "Nicer error message when user attempts to set title/author via Edit metadata dialog and one of the files is open in another program."
+
+    - title: "Fix {id} not working in send to device templates"
+
+    - title: "Windows: If creating a bytestring temp dir fails, create a unicode one and hope the rest of calibre can handle it"
+
+    - title: "Get Books: Fix some results from Amazon missing."
+      tickets: [785962]
+
+  improved recipes:
+    - Kathermini
+    - Faz.net
+    - The Washington Post
+    - El Mundo
+    - Marca
+    - The Nation
+
+  new recipes:
+    - title: Various German news sources
+      author: schuster
+
+    - title: "George R. R. Martin's Blog"
+      author: Darko Miletic
+
+    - title: "Focus (DE) and National Geographic"
+      auhtor: Anonymous
+
+
+
 - version: 0.8.2
   date: 2011-05-20
 

recipes/faznet.recipe

@@ -1,35 +1,63 @@
-from calibre.web.feeds.recipes import BasicNewsRecipe
+__license__   = 'GPL v3'
-class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+__copyright__ = '2008-2011, Kovid Goyal <kovid at kovidgoyal.net>, Darko Miletic <darko at gmail.com>'
+'''
+Profile to download FAZ.NET
+'''
 
-    title          = u'Faz.net'
+from calibre.web.feeds.news import BasicNewsRecipe
-    __author__  = 'schuster'
+
-    oldest_article = 1
+class FazNet(BasicNewsRecipe):
+    title                 = 'FAZ.NET'
+    __author__            = 'Kovid Goyal, Darko Miletic'
     description           = 'Frankfurter Allgemeine Zeitung'
-    max_articles_per_feed = 100
+    publisher             = 'Frankfurter Allgemeine Zeitung GmbH'
-    no_stylesheets         = True
+    category              = 'news, politics, Germany'
-    use_embedded_content   = False
+    use_embedded_content  = False
-    language               = 'de'
+    language = 'de'
-    remove_javascript      = True
-    cover_url = 'http://www.faz.net/f30/Images/Logos/logo.gif'
 
-    remove_tags = [dict(attrs={'class':['LinkBoxModulSmall', 'ModulLesermeinungenFooter', 'ModulArtikelServices', 'SocialMediaUnten', 'ArrowLinkRight', 'ModulVerlagsInfo', 'AdData', 'FazFooter', 'Date']}),
+    max_articles_per_feed = 30
-                dict(id=['FAZNavHeader', 'FAZNavMain', 'RightColumn', 'FazFooter', 'BreadCrumbs', 'FAZNavSubMain', 'FAZImgEvent']),
+    no_stylesheets        = True
-                dict(name=['jksrdt'])]
+    encoding              = 'utf-8'
+    remove_javascript     = True
 
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
 
-    feeds          = [(u'Politik', u'http://www.faz.net/s/RubA24ECD630CAE40E483841DB7D16F4211/Tpl~Epartner~SRss_.xml'),
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
-          (u'Wirtschaft', u'http://www.faz.net/s/RubC9401175958F4DE28E143E68888825F6/Tpl~Epartner~SRss_.xml'),
-          (u'Feuilleton', u'http://www.faz.net/s/RubCC21B04EE95145B3AC877C874FB1B611/Tpl~Epartner~SRss_.xml'),
-          (u'Sport', u'http://www.faz.net/s/Rub9F27A221597D4C39A82856B0FE79F051/Tpl~Epartner~SRss_.xml'),
-          (u'Gesellschaft', u'http://www.faz.net/s/Rub02DBAA63F9EB43CEB421272A670A685C/Tpl~Epartner~SRss_.xml'),
-          (u'Finanzen', u'http://www.faz.net/s/Rub4B891837ECD14082816D9E088A2D7CB4/Tpl~Epartner~SRss_.xml'),
-          (u'Wissen', u'http://www.faz.net/s/Rub7F4BEE0E0C39429A8565089709B70C44/Tpl~Epartner~SRss_.xml'),
-          (u'Reise', u'http://www.faz.net/s/RubE2FB5CA667054BDEA70FB3BC45F8D91C/Tpl~Epartner~SRss_.xml'),
-          (u'Technik & Motor', u'http://www.faz.net/s/Rub01E4D53776494844A85FDF23F5707AD8/Tpl~Epartner~SRss_.xml'),
-          (u'Beruf & Chance', u'http://www.faz.net/s/RubB1E10A8367E8446897468EDAA6EA0504/Tpl~Epartner~SRss_.xml'),
-          (u'Kunstmarkt', u'http://www.faz.net/s/RubBC09F7BF72A2405A96718ECBFB68FBFE/Tpl~Epartner~SRss_.xml'),
-          (u'Immobilien ', u'http://www.faz.net/s/RubFED172A9E10F46B3A5F01B02098C0C8D/Tpl~Epartner~SRss_.xml'),
-          (u'Rhein-Main Zeitung', u'http://www.faz.net/s/RubABE881A6669742C2A5EBCB5D50D7EBEE/Tpl~Epartner~SRss_.xml'),
-          (u'Atomdebatte ', u'http://www.faz.net/s/Rub469C43057F8C437CACC2DE9ED41B7950/Tpl~Epartner~SRss_.xml')
-    ]
 
+    keep_only_tags = [dict(name='div', attrs={'class':'Article'})]
+
+    remove_tags = [
+                     dict(name=['object','link','embed','base'])
+                    ,dict(name='div',
+                        attrs={'class':['LinkBoxModulSmall','ModulVerlagsInfo',
+                            'ArtikelServices', 'ModulLesermeinungenFooter',
+                            'ModulArtikelServices', 'BoxTool Aufklappen_Grau',
+                            'SocialMediaUnten', ]}),
+                    dict(id=['KurzLinkMenu', 'ArtikelServicesMenu']),
+                  ]
+
+    feeds = [
+              ('FAZ.NET Aktuell', 'http://www.faz.net/s/RubF3CE08B362D244869BE7984590CB6AC1/Tpl~Epartner~SRss_.xml'),
+              ('Politik', 'http://www.faz.net/s/RubA24ECD630CAE40E483841DB7D16F4211/Tpl~Epartner~SRss_.xml'),
+              ('Wirtschaft', 'http://www.faz.net/s/RubC9401175958F4DE28E143E68888825F6/Tpl~Epartner~SRss_.xml'),
+              ('Feuilleton', 'http://www.faz.net/s/RubCC21B04EE95145B3AC877C874FB1B611/Tpl~Epartner~SRss_.xml'),
+              ('Sport', 'http://www.faz.net/s/Rub9F27A221597D4C39A82856B0FE79F051/Tpl~Epartner~SRss_.xml'),
+              ('Gesellschaft', 'http://www.faz.net/s/Rub02DBAA63F9EB43CEB421272A670A685C/Tpl~Epartner~SRss_.xml'),
+              ('Finanzen', 'http://www.faz.net/s/Rub4B891837ECD14082816D9E088A2D7CB4/Tpl~Epartner~SRss_.xml'),
+              ('Wissen', 'http://www.faz.net/s/Rub7F4BEE0E0C39429A8565089709B70C44/Tpl~Epartner~SRss_.xml'),
+              ('Reise', 'http://www.faz.net/s/RubE2FB5CA667054BDEA70FB3BC45F8D91C/Tpl~Epartner~SRss_.xml'),
+              ('Technik & Motor', 'http://www.faz.net/s/Rub01E4D53776494844A85FDF23F5707AD8/Tpl~Epartner~SRss_.xml'),
+              ('Beruf & Chance', 'http://www.faz.net/s/RubB1E10A8367E8446897468EDAA6EA0504/Tpl~Epartner~SRss_.xml')
+            ]
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
+        soup.head.insert(0,mtag)
+        del soup.body['onload']
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

recipes/kathemerini.recipe

@@ -5,14 +5,16 @@ class Kathimerini(BasicNewsRecipe):
     __author__             = 'Pan'
     description            = 'News from Greece'
     max_articles_per_feed  = 100
-    oldest_article = 100
+    oldest_article         = 100
     publisher              = 'Kathimerini'
     category               = 'news, GR'
     language               = 'el'
+    encoding               = 'windows-1253'
+    conversion_options     = { 'linearize_tables': True}
     no_stylesheets         = True
-    remove_tags_before = dict(name='td',attrs={'class':'news'})
+    remove_tags_before     = dict(name='td',attrs={'class':'news'})
-    remove_tags_after = dict(name='td',attrs={'class':'news'})
+    remove_tags_after      = dict(name='td',attrs={'class':'news'})
-    remove_attributes = ['width', 'src','header','footer']
+    remove_attributes      = ['width', 'src','header','footer']
 
     feeds = [(u'\u03a0\u03bf\u03bb\u03b9\u03c4\u03b9\u03ba\u03ae',
   'http://wk.kathimerini.gr/xml_files/politics.xml'),
@@ -34,4 +36,3 @@ class Kathimerini(BasicNewsRecipe):
     def print_version(self, url):
         return url.replace('http://news.kathimerini.gr/4dcgi/', 'http://news.kathimerini.gr/4dcgi/4dcgi/')
 
-

resources/template-functions.json

@@ -20,6 +20,7 @@
     "lookup": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if len(args) == 2: # here for backwards compatibility\n        if val:\n            return formatter.vformat('{'+args[0].strip()+'}', [], kwargs)\n        else:\n            return formatter.vformat('{'+args[1].strip()+'}', [], kwargs)\n    if (len(args) % 2) != 1:\n        raise ValueError(_('lookup requires either 2 or an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)\n        if re.search(args[i], val):\n            return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)\n        i += 2\n", 
     "template": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    template = template.replace('[[', '{').replace(']]', '}')\n    return formatter.__class__().safe_format(template, kwargs, 'TEMPLATE', mi)\n", 
     "print": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    print args\n    return None\n", 
+    "merge_lists": "def evaluate(self, formatter, kwargs, mi, locals, list1, list2, separator):\n    l1 = [l.strip() for l in list1.split(separator) if l.strip()]\n    l2 = [l.strip() for l in list2.split(separator) if l.strip()]\n    lcl1 = set([icu_lower(l) for l in l1])\n    res = []\n    for i in l1:\n        res.append(i)\n    for i in l2:\n        if icu_lower(i) not in lcl1:\n            res.append(i)\n    return ', '.join(sorted(res, key=sort_key))\n", 
     "titlecase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return titlecase(val)\n", 
     "subitems": "def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index):\n    if not val:\n        return ''\n    si = int(start_index)\n    ei = int(end_index)\n    items = [v.strip() for v in val.split(',')]\n    rv = set()\n    for item in items:\n        component = item.split('.')\n        try:\n            if ei == 0:\n                rv.add('.'.join(component[si:]))\n            else:\n                rv.add('.'.join(component[si:ei]))\n        except:\n            pass\n    return ', '.join(sorted(rv, key=sort_key))\n", 
     "sublist": "def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index, sep):\n    if not val:\n        return ''\n    si = int(start_index)\n    ei = int(end_index)\n    val = val.split(sep)\n    try:\n        if ei == 0:\n            return sep.join(val[si:])\n        else:\n            return sep.join(val[si:ei])\n    except:\n        return ''\n", 

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 2)
+numeric_version = (0, 8, 3)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/gui2/actions/add.py

@@ -317,6 +317,7 @@ class AddAction(InterfaceAction):
                     _('Uploading books to device.'), 2000)
         if getattr(self._adder, 'number_of_books_added', 0) > 0:
             self.gui.library_view.model().books_added(self._adder.number_of_books_added)
+            self.gui.library_view.set_current_row(0)
             if hasattr(self.gui, 'db_images'):
                 self.gui.db_images.reset()
             self.gui.tags_view.recount()
@@ -338,7 +339,6 @@ class AddAction(InterfaceAction):
                 self.gui.library_view.model().current_changed(current_idx,
                         current_idx)
 
-
         if getattr(self._adder, 'critical', None):
             det_msg = []
             for name, log in self._adder.critical.items():

src/calibre/gui2/wizard/__init__.py

@@ -164,7 +164,7 @@ class Sony900(Sony505):
 
 class Nook(Sony505):
     id = 'nook'
-    name = 'Nook'
+    name = 'Nook and Nook Simple Reader'
     manufacturer = 'Barnes & Noble'
     output_profile = 'nook'
 

src/calibre/manual/template_lang.rst

@@ -253,6 +253,7 @@ The following functions are available in addition to those described in single-f
     
     * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables.
     * ``not(value)`` -- returns the string "1" if the value is empty, otherwise returns the empty string. This function works well with test or first_non_empty. You can have as many values as you want.
+    * ``merge_lists(list1, list2, separator)`` -- return a list made by merging the items in list1 and list2, removing duplicate items using a case-insensitive compare. If items differ in case, the one in list1 is used. The items in list1 and list2 are separated by separator, as are the items in the returned list.
     * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers.
     * ``or(value, value, ...)`` -- returns the string "1" if any value is not empty, otherwise returns the empty string. This function works well with test or first_non_empty. You can have as many values as you want.
     * ``print(a, b, ...)`` -- prints the arguments to standard output. Unless you start calibre from the command line (``calibre-debug -g``), the output will go to a black hole.
@@ -271,7 +272,7 @@ Function classification summary:
     * If-then-else: ``contains``, ``test``
     * Iterating over values: ``first_non_empty``, ``lookup``, ``switch``
     * List lookup: ``in_list``, ``list_item``, ``select``, 
-    * List manipulation: ``count``, ``sublist``, ``subitems``
+    * List manipulation: ``count``, ``merge_lists``, ``sublist``, ``subitems``
     * Recursion: ``eval``, ``template``
     * Relational: ``cmp`` , ``strcmp`` for strings
     * String case changes: ``lowercase``, ``uppercase``, ``titlecase``, ``capitalize``

src/calibre/utils/formatter_functions.py

@@ -642,6 +642,30 @@ class BuiltinNot(BuiltinFormatterFunction):
             i += 1
         return ''
 
+class BuiltinMergeLists(BuiltinFormatterFunction):
+    name = 'merge_lists'
+    arg_count = 3
+    doc = _('merge_lists(list1, list2, separator) -- '
+            'return a list made by merging the items in list1 and list2, '
+            'removing duplicate items using a case-insensitive compare. If '
+            'items differ in case, the one in list1 is used. '
+            'The items in list1 and list2 are separated by separator, as are '
+            'the items in the returned list.')
+
+    def evaluate(self, formatter, kwargs, mi, locals, list1, list2, separator):
+        l1 = [l.strip() for l in list1.split(separator) if l.strip()]
+        l2 = [l.strip() for l in list2.split(separator) if l.strip()]
+        lcl1 = set([icu_lower(l) for l in l1])
+
+        res = []
+        for i in l1:
+            res.append(i)
+        for i in l2:
+            if icu_lower(i) not in lcl1:
+                res.append(i)
+        return ', '.join(sorted(res, key=sort_key))
+
+
 builtin_add         = BuiltinAdd()
 builtin_and         = BuiltinAnd()
 builtin_assign      = BuiltinAssign()
@@ -660,6 +684,7 @@ builtin_in_list     = BuiltinInList()
 builtin_list_item   = BuiltinListitem()
 builtin_lookup      = BuiltinLookup()
 builtin_lowercase   = BuiltinLowercase()
+builtin_merge_lists = BuiltinMergeLists()
 builtin_multiply    = BuiltinMultiply()
 builtin_not         = BuiltinNot()
 builtin_or          = BuiltinOr()

src/calibre/utils/localization.py

@@ -29,8 +29,11 @@ def get_lang():
     lang = os.environ.get('CALIBRE_OVERRIDE_LANG', lang)
     if lang is not None:
         return lang
-    lang = locale.getdefaultlocale(['LANGUAGE', 'LC_ALL', 'LC_CTYPE',
+    try:
+        lang = locale.getdefaultlocale(['LANGUAGE', 'LC_ALL', 'LC_CTYPE',
                                     'LC_MESSAGES', 'LANG'])[0]
+    except:
+        pass # This happens on Ubuntu apparently
     if lang is None and os.environ.has_key('LANG'): # Needed for OS X
         try:
             lang = os.environ['LANG']