KG wip, including charles addition of custom_book_data

COPYRIGHT

@@ -4,6 +4,11 @@ License: GPL-3
  The full text of the GPL is distributed as in
  /usr/share/common-licenses/GPL-3 on Debian systems.
 
+Files: src/calibre/ebooks/pdf/*.h,*.cpp
+License: GPL-2 or later
+ The full text of the GPL is distributed as in
+ /usr/share/common-licenses/GPL-2 on Debian systems.
+
 Files: src/calibre/ebooks/BeautifulSoup.py
 Copyright: Copyright (c) 2004-2007, Leonard Richardson
 License: BSD

resources/default_tweaks.py

@@ -55,6 +55,32 @@ author_sort_copy_method = 'invert'
 #   categories_use_field_for_author_name = 'author_sort'
 categories_use_field_for_author_name = 'author'
 
+# Control how the tags pane displays categories containing many items. If the
+# number of items is larger than categories_collapse_more_than, a sub-category
+# will be added. If sorting by name, then the subcategories can be organized by
+# first letter (categories_collapse_model = 'first letter') or into equal-sized
+# groups (categories_collapse_model = 'partition'). If sorting by average rating
+# or by popularity, then 'partition' is always used. The addition of
+# subcategories can be disabled by setting categories_collapse_more_than = 0.
+# When using partition, the format of the subcategory label is controlled by a
+# template: categories_collapsed_name_template if sorting by name,
+# categories_collapsed_rating_template if sorting by average rating, and
+# categories_collapsed_popularity_template if sorting by popularity. There are
+# two variables available to the template: first and last. The variable 'first'
+# is the initial item in the subcategory, and the variable 'last' is the final
+# item in the subcategory. Both variables are 'objects'; they each have multiple
+# values that are obtained by using a suffix. For example, first.name for an
+# author category will be the name of the author. The sub-values available are:
+#  name: the printable name of the item
+#  count: the number of books that references this item
+#  avg_rating: the averate rating of all the books referencing this item
+#  sort: the sort value. For authors, this is the author_sort for that author
+#  category: the category (e.g., authors, series) that the item is in.
+categories_collapse_more_than = 50
+categories_collapsed_name_template = '{first.name:shorten(4,'',0)}{last.name::shorten(4,'',0)| - |}'
+categories_collapsed_rating_template = '{first.avg_rating:4.2f}{last.avg_rating:4.2f| - |}'
+categories_collapsed_popularity_template = '{first.count:d}{last.count:d| - |}'
+categories_collapse_model = 'first letter'
 
 # Set whether boolean custom columns are two- or three-valued.
 #  Two-values for true booleans
@@ -135,32 +161,53 @@ auto_connect_to_folder = ''
 # metadata management is set to automatic. Collections on Sonys are named
 # depending upon whether the field is standard or custom. A collection derived
 # from a standard field is named for the value in that field. For example, if
-# the standard 'series' column contains the name 'Darkover', then the series
+# the standard 'series' column contains the value 'Darkover', then the
-# will be named 'Darkover'. A collection derived from a custom field will have
+# collection name is 'Darkover'. A collection derived from a custom field will
-# the name of the field added to the value. For example, if a custom series
+# have the name of the field added to the value. For example, if a custom series
 # column named 'My Series' contains the name 'Darkover', then the collection
-# will be named 'Darkover (My Series)'. If two books have fields that generate
+# will by default be named 'Darkover (My Series)'. For purposes of this
-# the same collection name, then both books will be in that collection. This
+# documentation, 'Darkover' is called the value and 'My Series' is called the
-# tweak lets you specify for a standard or custom field the value to be put
+# category. If two books have fields that generate the same collection name,
-# inside the parentheses. You can use it to add a parenthetical description to a
+# then both books will be in that collection.
+# This set of tweaks lets you specify for a standard or custom field how
+# the collections are to be named. You can use it to add a description to a
 # standard field, for example 'Foo (Tag)' instead of the 'Foo'. You can also use
 # it to force multiple fields to end up in the same collection. For example, you
 # could force the values in 'series', '#my_series_1', and '#my_series_2' to
 # appear in collections named 'some_value (Series)', thereby merging all of the
-# fields into one set of collections. The syntax of this tweak is
+# fields into one set of collections.
-# {'field_lookup_name':'name_to_use', 'lookup_name':'name', ...}
+# There are two related tweaks. The first determines the category name to use
-# Example 1: I want three series columns to be merged into one set of
+# for a metadata field.  The second is a template, used to determines how the
-# collections. If the column lookup names are 'series', '#series_1' and
+# value and category are combined to create the collection name.
-# '#series_2', and if I want nothing in the parenthesis, then the value to use
+# The syntax of the first tweak, sony_collection_renaming_rules, is:
-# in the tweak value would be:
+# {'field_lookup_name':'category_name_to_use', 'lookup_name':'name', ...}
-# sony_collection_renaming_rules={'series':'', '#series_1':'', '#series_2':''}
+# The second tweak, sony_collection_name_template, is a template. It uses the
-# Example 2: I want the word '(Series)' to appear on collections made from
+# same template language as plugboards and save templates. This tweak controls
-# series, and the word '(Tag)' to appear on collections made from tags. Use:
+# how the value and category are combined together to make the collection name.
-# sony_collection_renaming_rules={'series':'Series', 'tags':'Tag'}
+# The only two fields available are {category} and {value}. The {value} field is
-# Example 3: I want 'series' and '#myseries' to be merged, and for the
+# never empty. The {category} field can be empty. The default is to put the
-# collection name to have '(Series)' appended. The renaming rule is:
+# value first, then the category enclosed in parentheses, it is isn't empty:
-# sony_collection_renaming_rules={'series':'Series', '#myseries':'Series'}
+# '{value} {category:|(|)}'
+# Examples: The first three examples assume that the second tweak
+# has not been changed.
+# 1: I want three series columns to be merged into one set of collections. The
+# column lookup names are 'series', '#series_1' and '#series_2'. I want nothing
+# in the parenthesis. The value to use in the tweak value would be:
+#    sony_collection_renaming_rules={'series':'', '#series_1':'', '#series_2':''}
+# 2: I want the word '(Series)' to appear on collections made from series, and
+# the word '(Tag)' to appear on collections made from tags. Use:
+#    sony_collection_renaming_rules={'series':'Series', 'tags':'Tag'}
+# 3: I want 'series' and '#myseries' to be merged, and for the collection name
+# to have '(Series)' appended. The renaming rule is:
+#    sony_collection_renaming_rules={'series':'Series', '#myseries':'Series'}
+# 4: Same as example 2, but instead of having the category name in parentheses
+# and appended to the value, I want it prepended and separated by a colon, such
+# as in Series: Darkover. I must change the template used to format the category name
+# The resulting two tweaks are:
+#    sony_collection_renaming_rules={'series':'Series', 'tags':'Tag'}
+#    sony_collection_name_template='{category:||: }{value}'
 sony_collection_renaming_rules={}
+sony_collection_name_template='{value}{category:| (|)}'
 
 
 # Specify how sony collections are sorted. This tweak is only applicable if
@@ -244,8 +291,10 @@ generate_cover_title_font = None
 generate_cover_foot_font = None
 
 
-# Behavior of doubleclick on the books list. Choices:
+# Behavior of doubleclick on the books list. Choices: open_viewer, do_nothing,
-# open_viewer, do_nothing, edit_cell. Default: open_viewer.
+# edit_cell, edit_metadata. Selecting edit_metadata has the side effect of
+# disabling editing a field using a single click.
+# Default: open_viewer.
 # Example: doubleclick_on_library_view = 'do_nothing'
 doubleclick_on_library_view = 'open_viewer'
 
@@ -265,4 +314,12 @@ locale_for_sorting =  ''
 # Set whether to use one or two columns for custom metadata when editing
 # metadata  one book at a time. If True, then the fields are laid out using two
 # columns. If False, one column is used.
-metadata_single_use_2_cols_for_custom_fields = True
+metadata_single_use_2_cols_for_custom_fields = True
+
+# The number of seconds to wait before sending emails when using a
+# public email server like gmail or hotmail. Default is: 5 minutes
+# Setting it to lower may cause the server's SPAM controls to kick in,
+# making email sending fail. Changes will take effect only after a restart of
+# calibre.
+public_smtp_relay_delay = 301
+

resources/recipes/business_insider.recipe

@@ -0,0 +1,69 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.businessinsider.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Business_insider(BasicNewsRecipe):
+    title                 = 'Business Insider'
+    __author__            = 'Darko Miletic'
+    description           = 'Noticias de Argentina y el resto del mundo'
+    publisher             = 'Business Insider, Inc.'
+    category              = 'news, politics, finances, world'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = True
+    language              = 'en'
+    remove_empty_feeds    = True
+    publication_type      = 'newsportal'
+    masthead_url          = 'http://static.businessinsider.com/assets/images/logos/tbi_print.jpg'
+    extra_css             = """
+                               body{font-family: Arial,Helvetica,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [
+                      dict(name=['meta','link'])
+                     ,dict(attrs={'class':'feedflare'})
+                  ]
+    remove_attributes=['lang','border']
+
+
+    feeds = [
+              (u'Latest'      , u'http://feeds2.feedburner.com/businessinsider'                          )
+             ,(u'Markets'     , u'http://feeds.feedburner.com/TheMoneyGame'                              )
+             ,(u'Wall Street' , u'http://feeds.feedburner.com/clusterstock'                              )
+             ,(u'Tech'        , u'http://feeds.feedburner.com/typepad/alleyinsider/silicon_alley_insider')
+             ,(u'The Wire'    , u'http://feeds.feedburner.com/businessinsider/thewire'                   )
+             ,(u'War Room'    , u'http://feeds.feedburner.com/businessinsider/warroom'                   )
+             ,(u'Sports'      , u'http://feeds.feedburner.com/businessinsider/sportspage'                )
+             ,(u'Tools'       , u'http://feeds.feedburner.com/businessinsider/tools'                     )
+             ,(u'Travel'      , u'http://feeds.feedburner.com/businessinsider/travel'                    )
+            ]
+
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            if item['href'].startswith('http://feedads'):
+               item.extract()
+            else:
+                if item.string is not None:
+                   tstr = item.string
+                   item.replaceWith(tstr)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup

resources/recipes/el_periodico.recipe

@@ -0,0 +1,109 @@
+#!/usr/bin/env  python
+# -*- coding: utf-8 -*-
+
+__license__     = 'GPL v3'
+__copyright__   = '04 December 2010, desUBIKado'
+__author__      = 'desUBIKado'
+__description__ = 'Daily newspaper from Aragon'
+__version__     = 'v0.05'
+__date__        = '07, December 2010'
+'''
+elperiodicodearagon.com
+'''
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class elperiodicodearagon(BasicNewsRecipe):
+    title                 = u'El Periodico de Aragon'
+    __author__            = u'desUBIKado'
+    description           = u'Noticias desde Aragon'
+    publisher             = u'elperiodicodearagon.com'
+    category              = u'news, politics, Spain, Aragon'
+    oldest_article        = 2
+    delay                 = 0
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    language              = 'es'
+    encoding              = 'utf8'
+    remove_empty_feeds    = True
+    remove_javascript     = True
+
+
+    conversion_options = {
+                             'comments'  : description
+                            ,'tags'      : category
+                            ,'language'  : language
+                            ,'publisher' : publisher
+                         }
+
+    feeds              = [(u'Arag\xf3n', u'http://elperiodicodearagon.com/RSS/2.xml'),
+                          (u'Internacional', u'http://elperiodicodearagon.com/RSS/4.xml'),
+                          (u'Espa\xf1a', u'http://elperiodicodearagon.com/RSS/3.xml'),
+                          (u'Econom\xeda', u'http://elperiodicodearagon.com/RSS/5.xml'),
+                          (u'Deportes', u'http://elperiodicodearagon.com/RSS/7.xml'),
+                          (u'Real Zaragoza', u'http://elperiodicodearagon.com/RSS/10.xml'),
+                          (u'Opini\xf3n', u'http://elperiodicodearagon.com/RSS/103.xml'),
+                          (u'Escenarios', u'http://elperiodicodearagon.com/RSS/105.xml'),
+                          (u'Sociedad', u'http://elperiodicodearagon.com/RSS/104.xml'),
+                          (u'Gente', u'http://elperiodicodearagon.com/RSS/330.xml')]
+
+
+    extra_css = '''
+                    h3{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:xx-large;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    dd{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+		'''
+
+    remove_attributes = ['height','width']
+
+    keep_only_tags     = [dict(name='div', attrs={'id':'contenidos'})]
+
+
+    # Quitar toda la morralla
+
+    remove_tags        = [dict(name='ul', attrs={'class':'herramientasDeNoticia'}),
+                          dict(name='span', attrs={'class':'MasInformacion '}),
+                          dict(name='span', attrs={'class':'MasInformacion'}),
+                          dict(name='div', attrs={'class':'Middle'}),
+                          dict(name='div', attrs={'class':'MenuCabeceraRZaragoza'}),
+                          dict(name='div', attrs={'id':'MenuCabeceraRZaragoza'}),
+                          dict(name='div', attrs={'class':'MenuEquipo'}),
+                          dict(name='div', attrs={'class':'TemasRelacionados'}),
+                          dict(name='div', attrs={'class':'GaleriaEnNoticia'}),
+                          dict(name='div', attrs={'class':'Recorte'}),
+                          dict(name='div', attrs={'id':'NoticiasenRecursos'}),
+                          dict(name='div', attrs={'id':'NoticiaEnPapel'}),
+                          dict(name='p', attrs={'class':'RecorteEnNoticias'}),
+                          dict(name='div', attrs={'id':'Comparte'}),
+                          dict(name='div', attrs={'id':'CajaComparte'}),
+                          dict(name='a', attrs={'class':'EscribirComentario'}),
+                          dict(name='a', attrs={'class':'AvisoComentario'}),
+                          dict(name='div', attrs={'class':'CajaAvisoComentario'}),
+                          dict(name='div', attrs={'class':'navegaNoticias'}),
+                          dict(name='div', attrs={'id':'PaginadorDiCom'}),
+                          dict(name='div', attrs={'id':'CajaAccesoCuentaUsuario'}),
+                          dict(name='div', attrs={'id':'CintilloComentario'}),
+                          dict(name='div', attrs={'id':'EscribeComentario'}),
+                          dict(name='div', attrs={'id':'FormularioComentario'}),
+                          dict(name='div', attrs={'id':'FormularioNormas'})]
+
+    # Recuperamos la portada de papel (la imagen format=1 tiene mayor resolucion)
+
+    def get_cover_url(self):
+        index = 'http://pdf.elperiodicodearagon.com/'
+        soup = self.index_to_soup(index)
+        for image in soup.findAll('img',src=True):
+           if image['src'].startswith('http://pdf.elperiodicodearagon.com/funciones/portada-preview.php?eid='):
+              return image['src'].rstrip('format=2') + 'format=1'
+        return None
+
+    # Para quitar espacios entre la noticia y los comentarios (lineas 1 y 2)
+    # El indice no apuntaba correctamente al empiece de la noticia (linea 3)
+
+    preprocess_regexps = [
+        (re.compile(r'<p>&nbsp;</p>', re.DOTALL|re.IGNORECASE), lambda match: ''),
+        (re.compile(r'<p> </p>', re.DOTALL|re.IGNORECASE), lambda match: ''),
+        (re.compile(r'<p id="">', re.DOTALL|re.IGNORECASE), lambda match: '<p>')
+        ]

resources/recipes/el_universal.recipe

@@ -1,7 +1,5 @@
-#!/usr/bin/env  python
-
 __license__   = 'GPL v3'
-__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
 eluniversal.com.mx
 '''
@@ -18,75 +16,25 @@ class ElUniversal(BasicNewsRecipe):
     category              = 'news, politics, Mexico'
     no_stylesheets        = True
     use_embedded_content  = False
-    encoding              = 'cp1252'
+    encoding              = 'utf8'
     remove_javascript     = True
-    language = 'es'
+    remove_empty_feeds    = True
+    publication_type      = 'newspaper'
+    language              = 'es'
 
     extra_css = '''
-                    body{font-family:Arial,Helvetica,sans-serif; font-size:x-small;}
+                    body{font-family:Arial,Helvetica,sans-serif}
-                    .geoGris30{font-family:Georgia,"Times New Roman",Times,serif; font-size:large; color:#003366; font-weight:bold;}
+                    .noteTitle{font-family: Georgia,"Times New Roman",Times,serif; color: #336699; font-size: xx-large; font-weight: bold}
-                    .arnegro16{font-family:Georgia,"Times New Roman",Times,serif; font-weight:bold; font-size:small;}
+                    .noteInfo{display: block; color: gray}
-                    .tbazull2{font-family:"trebuchet ms",Arial,Helvetica,sans-serif; color:#336699; font-size:xx-small;}
-                    .tbgrisf11{font-family:"trebuchet ms",Arial,Helvetica,sans-serif; color: #666666; font-size:xx-small;}
-                    .verrojo13{font-family:"trebuchet ms",Arial,Helvetica,sans-serif; color: #CC0033; font-size:xx-small;}
-                    .trnegro13{font-family:"trebuchet ms",Arial,Helvetica,sans-serif; font-size:xx-small;}
-                    .txt-fotogaleria{font-family:"trebuchet ms",Arial,Helvetica,sans-serif; font-size:xx-small;}
                 '''
-    keep_only_tags = [ dict(name='table', attrs={'width':"633"}),dict(name='table', attrs={'width':"629"}),]
+    keep_only_tags = [ dict(name='div', attrs={'id':'noteContent'})]
-
+    remove_tags_after = dict(attrs={'class':'noteText'})
     remove_tags  = [
-                    dict(name='table', attrs={'bgcolor':"#f5f5f5"}),
+                     dict(attrs={'class':'noteExtras'}),
-                     dict(name='td', attrs={'bgcolor':"#f7f8f9"}),
+                     dict(name=['meta','iframe','base','embed','object']),
-                    dict(name='td', attrs={'bgcolor':"#f5f5f5"}),
+                     dict(attrs={'id':'tm_box'})
-                    dict(name='table', attrs={'width':"302"}),
+                   ]
-                    dict(name='table', attrs={'width':"214"}),
+    remove_attributes=['lang','onclick']
-                    dict(name='table', attrs={'width':"112"}),
-                    dict(name='table', attrs={'width':"980"}),
-                    dict(name='td', attrs={'height':"1"}),
-                    dict(name='td', attrs={'height':"4"}),
-                     dict(name='td', attrs={'height':"20"}),
-                     dict(name='td', attrs={'height':"10"}),
-                    dict(name='td', attrs={'class':["trrojo11","trbris11","trrojo12","arrojo12s","tbazul13"]}),
-                    dict(name='div', attrs={'id':["mapg","ver_off_todosloscom","todosloscom"]}),
-                    dict(name='span', attrs={'class':["trazul18b","trrojo11","trnaranja11","trbris11","georojo18b","geogris18"]}),
-                    dict(name='span', attrs={'class':["detalles-opinion"]}),
-                    dict(name='a', attrs={'class':["arnaranja12b","trbris11","arazul12rel","trrojo10"]}),
-                    dict(name='img', src = "/img/icono_imprimir.gif"),
-                    dict(name='img', src = "/img/icono_enviar_mail.gif"),
-                    dict(name='img', src = "/img/icono_fuente_g.gif"),
-                    dict(name='img', src = "/img/icono_fuente_m.gif"),
-                    dict(name='img', src = "/img/icono_fuente_c.gif"),
-                    dict(name='img', src = "/img/icono_compartir.gif"),
-                    dict(name='img', src = "/img/icono_enviar_coment.gif"),
-                     dict(name='img', src = "http://www.eluniversal.com.mx/n_img/bot-notasrel.gif"),
-                    dict(name='img', src = "http://www.eluniversal.com.mx/n_img/fr.gif"),
-                    dict(name='img', src = "/img/espiral2.gif"),
-                    dict(name='img', src = "http://www.eluniversal.com.mx/n_img/b"),
-                    dict(name='img', src = "/img/icono_enviar_coment.gifot-notasrel.gif"),
-                    dict(name='img', src = "/n_img/icono_tipo3.gif"),
-                    dict(name='img', src = "/n_img/icono_tipo2.gif"),
-                    dict(name='img', src = "/n_img/icono_print.gif"),
-                    dict(name='img', src = "/n_img/icono_mail2.gif"),
-                    dict(name='img', src = "/n_img/im-comentarios-2a.gif"),
-                    dict(name='img', src = "/n_img/im-comentarios-1a.gif"),
-                    dict(name='img', src = "/img/icono_coment.gif"),
-                     dict(name='img', src = "http://www.eluniversal.com.mx/n_img/bot-sitiosrel.gif"),
-                    dict(name='img', src = "/n_img/icono_tipomenos.gif"),
-                    dict(name='img', src = "/img/futbol/19.jpg"),
-                    dict(name='img', alt = "Facebook"),
-                    dict(name='img', alt = "Twitter"),
-                    dict(name='img', alt = "Google"),
-                    dict(name='img', alt = "LinkedIn"),
-                    dict(name='img', alt = "Viadeo"),
-                    dict(name='img', alt = "Digg"),
-                    dict(name='img', alt = "Delicious"),
-                    dict(name='img', alt = "Meneame"),
-                    dict(name='img', alt = "Yahoo"),
-                    dict(name='img', alt = "Technorati"),
-                    dict(name='a',text =["Compartir","Facebook","Twitter","Google","LinkedIn","Viadeo","Digg","Delicious","Meneame","Yahoo","Technorati"]),
-                    dict(name='select'),
-                    dict(name='a', attrs={'class':"tbgriscompartir"}),
-                    ]
 
     feeds = [
               (u'Minuto por Minuto', u'http://www.eluniversal.com.mx/rss/universalmxm.xml' )
@@ -101,25 +49,3 @@ class ElUniversal(BasicNewsRecipe):
              ,(u'Computacion'      , u'http://www.eluniversal.com.mx/rss/computo.xml'      )
              ,(u'Sociedad'         , u'http://www.eluniversal.com.mx/rss/sociedad.xml'     )
             ]
-
-   # def print_version(self, url):
-   #     return url.replace('/notas/','/notas/vi_')
-
-    def preprocess_html(self, soup):
-       mtag = '<meta http-equiv="Content-Language" content="es-MX"/><meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
-       soup.head.insert(0,mtag)
-       for tag in soup.findAll(name='td',attrs={'class': 'arazul50'}):
-                 tag.insert(0,"<h1>")
-                 tag.insert(2,"</h1>")
-
-       return soup
-
-    def postprocess_html(self, soup,first):
-
-         for tag in soup.findAll(name=['table', 'span','i']):
-                tag.name = 'div'
-         for item in soup.findAll(align = "right"):
-                del item['align']
-
-         return soup
-

resources/recipes/elpais_impreso.recipe

@@ -1,86 +1,95 @@
-# -*- coding: utf-8 -*-
 __license__   = 'GPL v3'
 __copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
-www.elpais.com/diario/
+www.elpais.com
 '''
 
-from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class ElPaisImpresa(BasicNewsRecipe):
+class ElPais_RSS(BasicNewsRecipe):
-    title                 = u'El Pa\xeds - edicion impresa'
+    title                 = 'El Pais'
     __author__            = 'Darko Miletic'
-    description           = u'el periodico global en Espa\xf1ol'
+    description           = 'el periodico global en Castellano'
     publisher             = 'EDICIONES EL PAIS, S.L.'
-    category              = 'news, politics,Spain,actualidad,noticias,informacion,videos,fotografias,audios,graficos,nacional,internacional,deportes,economia,tecnologia,cultura,gente,television,sociedad,opinion,blogs,foros,chats,encuestas,entrevistas,participacion'
+    category              = 'news, politics, finances, world, spain'
+    oldest_article        = 2
+    max_articles_per_feed = 200
     no_stylesheets        = True
-    encoding              = 'latin1'
+    encoding              = 'cp1252'
     use_embedded_content  = False
-    language              = 'es'
+    language              = 'es_ES'
+    remove_empty_feeds    = True
     publication_type      = 'newspaper'
-    masthead_url          = 'http://www.elpais.com/im/tit_logo_global.gif'
+    masthead_url          = 'http://www.elpais.com/im/tit_logo.gif'
-    index                 = 'http://www.elpais.com/diario/'
+    extra_css             = """
-    extra_css             = ' p{text-align: justify} body{ text-align: left; font-family: Georgia,"Times New Roman",Times,serif } h2{font-family: Arial,Helvetica,sans-serif} img{margin-bottom: 0.4em} '
+                               body{font-family: Georgia,"Times New Roman",Times,serif }
+                               h3{font-family: Arial,Helvetica,sans-serif}
+                               img{margin-bottom: 0.4em; display:block}
+                            """
 
     conversion_options = {
-                          'comment'      : description
+                          'comment'   : description
-                        , 'tags'         : category
+                        , 'tags'      : category
-                        , 'publisher'    : publisher
+                        , 'publisher' : publisher
-                        , 'language'     : language
+                        , 'language'  : language
                         }
 
-    feeds          = [
+    keep_only_tags    = [dict(attrs={'class':['cabecera_noticia estirar','cabecera_noticia','','contenido_noticia']})]
-                        (u'Internacional'     , index + u'internacional/'     )
+    remove_tags       = [
-                       ,(u'Espa\xf1a'            , index + u'espana/'            )
+                           dict(name=['meta','link','base','iframe','embed','object'])
-                       ,(u'Economia'          , index + u'economia/'          )
+                          ,dict(attrs={'class':['info_complementa','estructura_2col_der','votos estirar','votos']})
-                       ,(u'Opinion'           , index + u'opinion/'           )
+                          ,dict(attrs={'id':'utilidades'})
-                       ,(u'Vi\xf1etas'           , index + u'vineta/'            )
+                        ]
-                       ,(u'Sociedad'          , index + u'sociedad/'          )
+    remove_tags_after = dict(attrs={'id':'utilidades'})
-                       ,(u'Cultura'           , index + u'cultura/'           )
+    remove_attributes = ['lang','border','width','height']
-                       ,(u'Tendencias'        , index + u'tendencias/'        )
-                       ,(u'Gente'             , index + u'gente/'             )
-                       ,(u'Obituarios'        , index + u'obituarios/'        )
-                       ,(u'Deportes'          , index + u'deportes/'          )
-                       ,(u'Pantallas'         , index + u'radioytv/'          )
-                       ,(u'Ultima'            , index + u'ultima/'            )
-                       ,(u'Educacion'         , index + u'educacion/'         )
-                       ,(u'Saludo'            , index + u'salud/'             )
-                       ,(u'Ciberpais'         , index + u'ciberpais/'         )
-                       ,(u'EP3'               , index + u'ep3/'               )
-                       ,(u'Cine'              , index + u'cine/'              )
-                       ,(u'Babelia'           , index + u'babelia/'           )
-                       ,(u'El viajero'        , index + u'viajero/'           )
-                       ,(u'Negocios'          , index + u'negocios/'          )
-                       ,(u'Domingo'           , index + u'domingo/'           )
-                       ,(u'El Pais semanal'   , index + u'eps/'               )
-                       ,(u'Quadern Catalunya' , index + u'quadern-catalunya/' )
-                     ]
 
-    keep_only_tags=[dict(attrs={'class':['cabecera_noticia','contenido_noticia']})]
+    feeds = [
-    remove_attributes=['width','height']
+              (u'Lo ultimo'            , u'http://www.elpais.com/rss/feed.html?feedId=17046')
-    remove_tags=[dict(name='link')]
+             ,(u'America Latina'       , u'http://www.elpais.com/rss/feed.html?feedId=17041')
-
+             ,(u'Mexico'               , u'http://www.elpais.com/rss/feed.html?feedId=17042')
-    def parse_index(self):
+             ,(u'Europa'               , u'http://www.elpais.com/rss/feed.html?feedId=17043')
-        totalfeeds = []
+             ,(u'Estados Unidos'       , u'http://www.elpais.com/rss/feed.html?feedId=17044')
-        lfeeds = self.get_feeds()
+             ,(u'Oriente proximo'      , u'http://www.elpais.com/rss/feed.html?feedId=17045')
-        for feedobj in lfeeds:
+             ,(u'Espana'               , u'http://www.elpais.com/rss/feed.html?feedId=1002' )
-            feedtitle, feedurl = feedobj
+             ,(u'Andalucia'            , u'http://www.elpais.com/rss/feed.html?feedId=17057')
-            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+             ,(u'Catalunia'            , u'http://www.elpais.com/rss/feed.html?feedId=17059')
-            articles = []
+             ,(u'Comunidad Valenciana' , u'http://www.elpais.com/rss/feed.html?feedId=17061')
-            soup = self.index_to_soup(feedurl)
+             ,(u'Madrid'               , u'http://www.elpais.com/rss/feed.html?feedId=1016' )
-            for item in soup.findAll('a',attrs={'class':['g19r003','g19i003','g17r003','g17i003']}):
+             ,(u'Pais Vasco'           , u'http://www.elpais.com/rss/feed.html?feedId=17062')
-                url   = 'http://www.elpais.com' + item['href'].rpartition('/')[0]
+             ,(u'Galicia'              , u'http://www.elpais.com/rss/feed.html?feedId=17063')
-                title = self.tag_to_string(item)
+             ,(u'Opinion'              , u'http://www.elpais.com/rss/feed.html?feedId=1003' )             
-                date  = strftime(self.timefmt)
+             ,(u'Sociedad'             , u'http://www.elpais.com/rss/feed.html?feedId=1004' )             
-                articles.append({
+             ,(u'Deportes'             , u'http://www.elpais.com/rss/feed.html?feedId=1007' )
-                                      'title'      :title
+             ,(u'Cultura'              , u'http://www.elpais.com/rss/feed.html?feedId=1008' )
-                                     ,'date'       :date
+             ,(u'Cine'                 , u'http://www.elpais.com/rss/feed.html?feedId=17052')
-                                     ,'url'        :url
+             ,(u'Literatura'           , u'http://www.elpais.com/rss/feed.html?feedId=17053')
-                                     ,'description':''
+             ,(u'Musica'               , u'http://www.elpais.com/rss/feed.html?feedId=17051')
-                                    })
+             ,(u'Arte'                 , u'http://www.elpais.com/rss/feed.html?feedId=17060')             
-            totalfeeds.append((feedtitle, articles))
+             ,(u'Tecnologia'           , u'http://www.elpais.com/rss/feed.html?feedId=1005' )
-        return totalfeeds
+             ,(u'Economia'             , u'http://www.elpais.com/rss/feed.html?feedId=1006' )
+             ,(u'Ciencia'              , u'http://www.elpais.com/rss/feed.html?feedId=17068')
+             ,(u'Salud'                , u'http://www.elpais.com/rss/feed.html?feedId=17074')
+             ,(u'Ocio'                 , u'http://www.elpais.com/rss/feed.html?feedId=17075')
+             ,(u'Justicia y Leyes'     , u'http://www.elpais.com/rss/feed.html?feedId=17069')
+             ,(u'Guerras y conflictos' , u'http://www.elpais.com/rss/feed.html?feedId=17070')
+             ,(u'Politica'             , u'http://www.elpais.com/rss/feed.html?feedId=17073')
+            ]
 
     def print_version(self, url):
         return url + '?print=1'
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            if item.string is not None:
+               tstr = item.string
+               item.replaceWith(tstr)
+            else:
+               item.name='span'
+               for atrs in ['href','target','alt','title']:
+                   if item.has_key(atrs):
+                      del item[atrs]
+        for item in soup.findAll('img',alt=False):
+            item['alt'] = 'image'
+        return soup

resources/recipes/heraldo.recipe

@@ -1,50 +1,65 @@
 #!/usr/bin/env  python
-__license__    = 'GPL v3'
+__license__     = 'GPL v3'
-__author__     = 'Lorenzo Vigentini'
+__copyright__   = '04 December 2010, desUBIKado'
-__copyright__   = '2009, Lorenzo Vigentini <l.vigentini at gmail.com>'
+__author__      = 'desUBIKado'
 __description__ = 'Daily newspaper from Aragon'
-__version__     = 'v1.01'
+__version__     = 'v0.03'
-__date__        = '30, January 2010'
+__date__        = '11, December 2010'
-
 '''
-http://www.heraldo.es/
+[url]http://www.heraldo.es/[/url]
 '''
 
+import time
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class heraldo(BasicNewsRecipe):
-    author        = 'Lorenzo Vigentini'
+    __author__        = 'desUBIKado'
     description   = 'Daily newspaper from Aragon'
-
-    cover_url      = 'http://www.heraldo.es/MODULOS/global/publico/interfaces/img/logo.gif'
     title          = u'Heraldo de Aragon'
     publisher      = 'OJD Nielsen'
     category       = 'News, politics, culture, economy, general interest'
-
     language       = 'es'
     timefmt        = '[%a, %d %b, %Y]'
-
     oldest_article = 1
-    max_articles_per_feed = 25
+    max_articles_per_feed = 100
-
     use_embedded_content  = False
-    recursion             = 10
-
     remove_javascript = True
     no_stylesheets = True
-
+    recursion      = 10
-    keep_only_tags = [
-                        dict(name='div', attrs={'class':['titularNoticiaNN','textoGrisVerdanaContenidos']})
-                    ]
 
     feeds          = [
-                        (u'Portadas ', u'http://www.heraldo.es/index.php/mod.portadas/mem.rss')
+                        (u'Portadas', u'http://www.heraldo.es/index.php/mod.portadas/mem.rss')
-                    ]
+                     ]
+
+
+
+    keep_only_tags     = [dict(name='div', attrs={'id':['dts','com']})]
+
+    remove_tags        = [dict(name='a', attrs={'class':['com flo-r','enl-if','enl-df']}),
+                          dict(name='div', attrs={'class':['brb-b-s con marg-btt','cnt-rel con']}),
+                          dict(name='form', attrs={'class':'form'})]
+
+    remove_tags_before = dict(name='div' , attrs={'id':'dts'})
+    remove_tags_after  = dict(name='div' , attrs={'id':'com'})
+
+    def get_cover_url(self):
+        cover = None
+        st = time.localtime()
+        year = str(st.tm_year)
+        month = "%.2d" % st.tm_mon
+        day = "%.2d" % st.tm_mday
+		#[url]http://oldorigin-www.heraldo.es/20101211/primeras/portada_aragon.pdf[/url]
+        cover='http://oldorigin-www.heraldo.es/'+ year +  month + day +'/primeras/portada_aragon.pdf'
+        br = BasicNewsRecipe.get_browser()
+        try:
+            br.open(cover)
+        except:
+            self.log("\nPortada no disponible")
+            cover ='http://www.heraldo.es/MODULOS/global/publico/interfaces/img/logo-Heraldo.png'
+        return cover
+
+
+
     extra_css = '''
-                    .articledate {color: gray;font-family: monospace;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:xx-large;}
-                    .articledescription {display: block;font-family: sans;font-size: 0.7em; text-indent: 0;}
+		'''
-                    .firma {color: #666;display: block;font-family: verdana, arial, helvetica;font-size: 1em;margin-bottom: 8px;}
-                    .textoGrisVerdanaContenidos {color: #56595c;display: block;font-family: Verdana;font-size: 1.28571em;padding-bottom: 10px}
-                    .titularNoticiaNN {display: block;padding-bottom: 10px;padding-left: 0;padding-right: 0;padding-top: 4px}
-                    .titulo {color: #003066;font-family: Tahoma;font-size: 1.92857em;font-weight: bold;line-height: 1.2em}
-                    '''

resources/recipes/karlsruhe.recipe

@@ -0,0 +1,52 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class KANewsRecipe(BasicNewsRecipe):
+    title = u'KA-News.de'
+    description = u'Nachrichten aus Karlsruhe, Deutschland und der Welt.'
+    __author__ = 'tfeld'
+    lang='de'
+    no_stylesheets = True
+
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    feeds = [
+                (u'News aus Karlsruhe', 'http://www.ka-news.de/storage/rss/rss/karlsruhe.xml'),
+                (u'Kulturnachrichten aus Karlsruhe', 'http://www.ka-news.de/storage/rss/rss/kultur.xml'),
+                (u'Durlach: News aus Durlach', 'http://www.ka-news.de/storage/rss/rss/durlach.xml'),
+                (u'Stutensee: News aus Stutensee Blankenloch, Büchig, Friedrichstal, Staffort, Spöck', 'http://www.ka-news.de/storage/rss/rss/stutensee.xml'),
+                (u'Bruchsal: News aus Bruchsal', 'http://www.ka-news.de/storage/rss/rss/bruchsal.xml'),
+                (u'Wirtschaftsnews aus Karlsruhe', 'http://www.ka-news.de/storage/rss/rss/wirtschaft.xml'),
+                (u'ka-news.de - Sport', 'http://www.ka-news.de/storage/rss/rss/sport.xml'),
+                (u'KSC-News - News rund um den KSC', 'http://www.ka-news.de/storage/rss/rss/ksc.xml'),
+                (u'ka-news.de - BG Karlsruhe', 'http://www.ka-news.de/storage/rss/rss/basketball.xml')
+            ]
+
+    preprocess_regexps = [
+        (re.compile(r'width:[0-9]*?px', re.DOTALL|re.IGNORECASE), lambda match: ''),
+    ]
+
+    remove_tags_before = dict(id='artdetail_ueberschrift')
+    remove_tags_after = dict(id='artdetail_unterzeile')
+    remove_tags = [dict(name=['div'], attrs={'class': 'lbx_table'}),
+         dict(name=['div'], attrs={'class': 'lk_zumthema'}),
+         dict(name=['div'], attrs={'class': 'lk_thumb'}),
+         dict(name=['div'], attrs={'class': 'lk_trenner'}),
+         dict(name=['div'], attrs={'class': 'lupen_container'}),
+         dict(name=['script']),
+         dict(name=['span'], attrs={'style': 'display:none;'}),
+         dict(name=['span'], attrs={'class': 'comm_info'}),
+         dict(name=['h3'], attrs={'id': 'artdetail_unterzeile'})]
+
+    # removing style attribute _after_ removing specifig tags above
+    remove_attributes = ['width','height','style']
+
+    extra_css = '''
+         h1{ font-size:large; font-weight:bold;  }
+         h2{ font-size:medium; font-weight:bold; }
+    '''
+
+    def get_cover_url(self):
+        return 'http://www.ka-news.de/storage/scl/techkanews/logos/434447_m1t1w250q75s1v29681_ka-news-Logo_mit_Schatten_transparent.png'
+

resources/recipes/red_aragon.recipe

@@ -0,0 +1,47 @@
+#!/usr/bin/env  python
+__license__     = 'GPL v3'
+__copyright__   = '11 December 2010, desUBIKado'
+__author__      = 'desUBIKado'
+__description__ = 'Entertainment guide from Aragon'
+__version__     = 'v0.01'
+__date__        = '11, December 2010'
+'''
+[url]http://www.redaragon.es/[/url]
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class heraldo(BasicNewsRecipe):
+    __author__         = 'desUBIKado'
+    description    = u'Guia de ocio desde Aragon'
+    title          = u'RedAragon'
+    publisher      = 'Grupo Z'
+    category       = 'Concerts, Movies, Entertainment news'
+    cover_url      = 'http://www.redaragon.com/2008_img/logotipo.gif'
+    language       = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+    oldest_article = 15
+    max_articles_per_feed = 100
+    encoding       = 'iso-8859-1'
+    use_embedded_content  = False
+    remove_javascript = True
+    no_stylesheets = True
+
+    feeds              = [(u'Conciertos', u'http://redaragon.com/rss/agenda.asp?tid=1'),
+                          (u'Exposiciones', u'http://redaragon.com/rss/agenda.asp?tid=5'),
+                          (u'Teatro', u'http://redaragon.com/rss/agenda.asp?tid=10'),
+                          (u'Conferencias', u'http://redaragon.com/rss/agenda.asp?tid=2'),
+                          (u'Ferias', u'http://redaragon.com/rss/agenda.asp?tid=6'),
+                          (u'Filmotecas/Cineclubs', u'http://redaragon.com/rss/agenda.asp?tid=7'),
+                          (u'Presentaciones', u'http://redaragon.com/rss/agenda.asp?tid=9'),
+                          (u'Fiestas', u'http://redaragon.com/rss/agenda.asp?tid=11'),
+                          (u'Infantil', u'http://redaragon.com/rss/agenda.asp?tid=13'),
+                          (u'Otros', u'http://redaragon.com/rss/agenda.asp?tid=8')]
+
+    keep_only_tags     = [dict(name='div', attrs={'id':'FichaEventoAgenda'})]
+
+    remove_tags        = [dict(name='div', attrs={'class':['Comparte','CajaAgenda','Caja','Cintillo']})]
+
+    remove_tags_before = dict(name='div' , attrs={'id':'FichaEventoAgenda'})
+
+    remove_tags_after  = dict(name='div' , attrs={'class':'Cintillo'})

resources/recipes/salon.recipe

@@ -25,22 +25,20 @@ class Salon_com(BasicNewsRecipe):
 
     feeds = [
         ('News & Politics', 'http://feeds.salon.com/salon/news'),
-        ('War Room', 'http://feeds.salon.com/salon/war_room'),
+        ('War Room', 'http://feeds.feedburner.com/salon/war_room'),
-        ('Arts & Entertainment', 'http://feeds.salon.com/salon/ent'),
+        ('Joan Walsh', 'http://feeds.feedburner.com/Salon_Joan_Walsh'),
-        ('I Like to Watch', 'http://feeds.salon.com/salon/iltw'),
+        ('Glenn Greenwald', 'http://feeds.feedburner.com/salon/greenwald'),
-        ('Beyond Multiplex', 'http://feeds.salon.com/salon/btm'),
-        ('Book Reviews', 'http://feeds.salon.com/salon/books'),
-        ('All Life', 'http://feeds.salon.com/salon/mwt'),
-        ('All Opinion', 'http://feeds.salon.com/salon/opinion'),
-        ('Glenn Greenwald', 'http://feeds.salon.com/salon/greenwald'),
-        ('Garrison Keillor', 'http://dir.salon.com/topics/garrison_keillor/index.rss'),
-        ('Joan Walsh', 'http://www.salon.com/rss/walsh.rss'),
-        ('All Sports', 'http://feeds.salon.com/salon/sports'),
         ('Tech & Business', 'http://feeds.salon.com/salon/tech'),
-        ('How World Works', 'http://feeds.salon.com/salon/htww')
+        ('Ask the Pilot', 'http://feeds.feedburner.com/salon/ask_the_pilot'),
+        ('How World Works', 'http://feeds.feedburner.com/salon/htww'),
+        ('Life', 'http://feeds.feedburner.com/salon/mwt'),
+        ('Broadsheet', 'http://feeds.feedburner.com/salon/broadsheet'),
+        ('Movie Reviews', 'http://feeds.feedburner.com/salon/movie_reviews'),
+        ('Film Salon', 'http://feeds.feedburner.com/Salon/Film_Salon'),
+        ('TV', 'http://feeds.feedburner.com/salon/tv'),
+        ('Books', 'http://feeds.feedburner.com/salon/books')
             ]
 
     def print_version(self, url):
         return url.replace('/index.html', '/print.html')
 
-

resources/recipes/the_week_magazine_free.recipe

@@ -1,17 +1,19 @@
-
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010, JOlo'
 '''
 www.theweek.com
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
+import re
 
-class TheWeekFree(BasicNewsRecipe):
+class TheWeek(BasicNewsRecipe):
-    title                 = 'The Week Magazine - Free content'
+    title                 = 'The Week Magazine'
-    __author__            = 'Darko Miletic'
+    __author__            = 'Jim Olo'
     description           = "The best of the US and international media.  Daily coverage of commentary and analysis of the day's events, as well as arts, entertainment, people and gossip, and political cartoons."
     publisher             = 'The Week Publications, Inc.'
+    masthead_url           = 'http://test.theweek.com/images/logo_theweek.gif'
+    cover_url              = masthead_url
     category              = 'news, politics, USA'
     oldest_article        = 7
     max_articles_per_feed = 100
@@ -19,31 +21,27 @@ class TheWeekFree(BasicNewsRecipe):
     encoding              = 'utf-8'
     use_embedded_content  = False
     language              = 'en'
+    preprocess_regexps      = [(re.compile(r'<h3><a href=.*</body>', re.DOTALL), lambda match: '</body>')]
+    remove_tags_before    = dict(name='h1')
+    remove_tags_after     = dict(name='div', attrs={'class':'articleSubscribe4free'})
+    remove_tags           = [
+                              dict(name='div', attrs={'class':['floatLeft','imageCaption','slideshowImageAttribution','postDate','utilities','cartoonInfo','left','middle','col300','articleSubscribe4free',' articleFlyout','articleFlyout floatRight','fourFreeBar']})
+                             ,dict(name='div', attrs={'id':['cartoonThumbs','rightColumn','header','partners']})
+                             ,dict(name='ul', attrs={'class':['slideshowNav','hotTopicsList topicList']})
+                              ]
+    remove_attributes     = ['width','height', 'style', 'font', 'color']
+    extra_css = '''
+                h1{font-family:Geneva, Arial, Helvetica, sans-serif;color:#154B7A;}
+                h3{font-size: 14px;color:#999999; font-family:Geneva, Arial, Helvetica, sans-serif;font-weight: bold;}
+                h2{color:#666666; font-family:Geneva, Arial, Helvetica, sans-serif;font-size:small;}
+                p {font-family:Arial,Helvetica,sans-serif;}
+                '''
+    filter_regexps = [r'www\.palmcoastdata\.com']
 
-    conversion_options = {
+    feeds          =  [
-                          'comment'   : description
+               (u'News-Opinion', u'http://theweek.com/section/index/news_opinion.rss'),
-                        , 'tags'      : category
+               (u'Business', u'http://theweek.com/section/index/business.rss'),
-                        , 'publisher' : publisher
+               (u'Arts-Life', u'http://theweek.com/section/index/arts_life.rss'),
-                        , 'language'  : language
+               (u'Cartoons', u'http://theweek.com/section/index/cartoon_wit/0/all-cartoons.rss')
-                        }
+                      ]
-
-    keep_only_tags = [
-                        dict(name=['h1','h2'])
-                      , dict(name='div', attrs={'class':'basefont'})
-                      , dict(name='div', attrs={'id':'slideshowLoader'})
-                     ]
-
-    remove_tags = [
-                     dict(name='div', attrs={'id':['digg_dugg','articleRight','dateHeader']})
-                    ,dict(name=['object','embed','iframe'])
-                  ]
-
-
-    feeds = [
-              (u'News & Opinions'       , u'http://www.theweek.com/section/index/news_opinion.rss')
-             ,(u'Arts & Leisure'        , u'http://www.theweek.com/section/index/arts_leisure.rss')
-             ,(u'Business'              , u'http://www.theweek.com/section/index/business.rss'    )
-             ,(u'Cartoon & Short takes' , u'http://www.theweek.com/section/index/cartoons_wit.rss')
-            ]
-
 

resources/recipes/wired.recipe

@@ -38,12 +38,12 @@ class Wired(BasicNewsRecipe):
     keep_only_tags = [dict(name='div', attrs={'class':'post'})]
     remove_tags_after = dict(name='div', attrs={'class':'tweetmeme_button'})
     remove_tags = [
-                     dict(name=['object','embed','iframe','link'])
+                     dict(name=['object','embed','iframe','link','meta','base'])
                     ,dict(name='div', attrs={'class':['podcast_storyboard','tweetmeme_button']})
                     ,dict(attrs={'id':'ff_bottom_nav'})
                     ,dict(name='a',attrs={'href':'http://www.wired.com/app'})
                   ]
-    remove_attributes = ['height','width']
+    remove_attributes = ['height','width','lang','border','clear']
 
 
     def parse_index(self):
@@ -78,7 +78,9 @@ class Wired(BasicNewsRecipe):
                divurl = item.find('div',attrs={'class':'feature-header'})
                if divurl:
                    divdesc = item.find('div',attrs={'class':'feature-text'})
-                   url   = 'http://www.wired.com' + divurl.a['href']
+                   url = divurl.a['href']
+                   if not divurl.a['href'].startswith('http://www.wired.com'):
+                      url   = 'http://www.wired.com' + divurl.a['href']
                    title = self.tag_to_string(divurl.a)
                    description = self.tag_to_string(divdesc)
                    date  = strftime(self.timefmt)
@@ -127,5 +129,17 @@ class Wired(BasicNewsRecipe):
     def preprocess_html(self, soup):
         for item in soup.findAll(style=True):
             del item['style']
+        for item in soup.findAll('a'):
+            if item.string is not None:
+               tstr = item.string
+               item.replaceWith(tstr)
+            else:
+               item.name='span'
+               for atrs in ['href','target','alt','title','name','id']:
+                   if item.has_key(atrs):
+                      del item[atrs]
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'            
         return soup
 

src/calibre/customize/builtins.py

@@ -478,7 +478,7 @@ from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
 from calibre.devices.sne.driver import SNE
 from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, KOGAN, \
         GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, Q600, LUMIREAD, ALURATEK_COLOR, \
-        TREKSTOR
+        TREKSTOR, EEEREADER, NEXTBOOK
 from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 from calibre.devices.kobo.driver import KOBO
 from calibre.devices.bambook.driver import BAMBOOK
@@ -605,6 +605,8 @@ plugins += [
     ALURATEK_COLOR,
     BAMBOOK,
     TREKSTOR,
+    EEEREADER,
+    NEXTBOOK,
     ITUNES,
 ]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \

src/calibre/customize/profiles.py

@@ -439,6 +439,13 @@ class TabletOutput(iPadOutput):
     screen_size = (sys.maxint, sys.maxint)
     comic_screen_size = (sys.maxint, sys.maxint)
 
+class SamsungGalaxy(TabletOutput):
+    name = 'Samsung Galaxy'
+    shortname = 'galaxy'
+    description = _('Intended for the Samsung Galaxy and similar tablet devices with '
+            'a resolution of 600x1280')
+    screen_size = comic_screen_size = (600, 1280)
+
 class SonyReaderOutput(OutputProfile):
 
     name        = 'Sony Reader'
@@ -709,7 +716,7 @@ class BambookOutput(OutputProfile):
 output_profiles = [OutputProfile, SonyReaderOutput, SonyReader300Output,
         SonyReader900Output, MSReaderOutput, MobipocketOutput, HanlinV3Output,
         HanlinV5Output, CybookG3Output, CybookOpusOutput, KindleOutput,
-        iPadOutput, KoboReaderOutput, TabletOutput,
+        iPadOutput, KoboReaderOutput, TabletOutput, SamsungGalaxy,
         SonyReaderLandscapeOutput, KindleDXOutput, IlliadOutput,
         IRexDR1000Output, IRexDR800Output, JetBook5Output, NookOutput,
         BambookOutput, NookColorOutput]

src/calibre/devices/android/driver.py

@@ -64,7 +64,8 @@ class ANDROID(USBMS):
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
-            'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000']
+            'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
+            'SGH-T849']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD']
 

src/calibre/devices/bambook/driver.py

@@ -29,12 +29,16 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
     booklist_class = BookList
     book_class = Book
 
+    ip = None
+
     FORMATS = [ "snb" ]
     VENDOR_ID = 0x230b
     PRODUCT_ID = 0x0001
     BCD = None
     CAN_SET_METADATA = False
     THUMBNAIL_HEIGHT = 155
+    EXTRA_CUSTOMIZATION_MESSAGE = \
+        _("Device IP Address (restart calibre after changing)")
 
     icon = I("devices/bambook.png")
 #    OPEN_FEEDBACK_MESSAGE = _(
@@ -47,6 +51,10 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
     METADATA_FILE_GUID = 'calibremetadata.snb'
 
     bambook = None
+    is_connected = False
+
+    def __init__(self, ip):
+        self.ip = ip
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None) :
@@ -60,15 +68,23 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
         self.eject()
         # Connect
         self.bambook = Bambook()
-        self.bambook.Connect()
+        self.bambook.Connect(ip = self.ip, timeout = 10000)
         if self.bambook.GetState() != CONN_CONNECTED:
             self.bambook = None
-            raise Exception(_("Unable to connect to Bambook."))
+            raise OpenFeedback(_("Unable to connect to Bambook. \n"
+                                 "If you are trying to connect via Wi-Fi, "
+                                 "please make sure the IP address of Bambook has been correctly configured."))
+        self.is_connected = True
+        return True
+
+    def unmount_device(self):
+        self.eject()
 
     def eject(self):
         if self.bambook:
             self.bambook.Disconnect()
             self.bambook = None
+            self.is_connected = False
 
     def post_yank_cleanup(self):
         self.eject()
@@ -475,3 +491,8 @@ class BAMBOOK(DeviceConfig, DevicePlugin):
     def get_guid(uuid):
         guid = hashlib.md5(uuid).hexdigest()[0:15] + ".snb"
         return guid
+
+class BAMBOOKWifi(BAMBOOK):
+    def is_usb_connected(self, devices_on_system, debug=False,
+                         only_presence=False):
+        return self.is_connected, self

src/calibre/devices/bambook/libbambookcore.py

@@ -329,6 +329,8 @@ class Bambook:
         self.handle = None
 
     def Connect(self, ip = DEFAULT_BAMBOOK_IP, timeout = 10000):
+        if ip == None or ip == '':
+            ip = DEFAULT_BAMBOOK_IP
         self.handle = BambookConnect(ip, timeout)
         if self.handle and self.handle != 0:
             return True

src/calibre/devices/eb600/driver.py

@@ -230,7 +230,7 @@ class POCKETBOOK301(USBMS):
 class POCKETBOOK602(USBMS):
 
     name = 'PocketBook Pro 602/902 Device Interface'
-    description    = _('Communicate with the PocketBook 602 reader.')
+    description    = _('Communicate with the PocketBook 602/603/902/903 reader.')
     author         = 'Kovid Goyal'
     supported_platforms = ['windows', 'osx', 'linux']
     FORMATS = ['epub', 'fb2', 'prc', 'mobi', 'pdf', 'djvu', 'rtf', 'chm',
@@ -244,7 +244,7 @@ class POCKETBOOK602(USBMS):
     BCD         = [0x0324]
 
     VENDOR_NAME = ''
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['PB602', 'PB902']
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['PB602', 'PB603', 'PB902', 'PB903']
 
 class POCKETBOOK701(USBMS):
 

src/calibre/devices/folder_device/driver.py

@@ -18,9 +18,9 @@ class FOLDER_DEVICE_FOR_CONFIG(USBMS):
     supported_platforms = ['windows', 'osx', 'linux']
     FORMATS     = ['epub', 'fb2', 'mobi', 'azw', 'lrf', 'tcr', 'pmlz', 'lit',
                    'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb', 'prc']
-    VENDOR_ID   = 0xffff
+    VENDOR_ID   = [0xffff]
-    PRODUCT_ID  = 0xffff
+    PRODUCT_ID  = [0xffff]
-    BCD         = 0xffff
+    BCD         = [0xffff]
     DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
 
 
@@ -34,9 +34,9 @@ class FOLDER_DEVICE(USBMS):
     supported_platforms = ['windows', 'osx', 'linux']
     FORMATS     = FOLDER_DEVICE_FOR_CONFIG.FORMATS
 
-    VENDOR_ID   = 0xffff
+    VENDOR_ID   = [0xffff]
-    PRODUCT_ID  = 0xffff
+    PRODUCT_ID  = [0xffff]
-    BCD         = 0xffff
+    BCD         = [0xffff]
     DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
 
     THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device

src/calibre/devices/iriver/driver.py

@@ -20,11 +20,11 @@ class IRIVER_STORY(USBMS):
     FORMATS     = ['epub', 'fb2', 'pdf', 'djvu', 'txt']
 
     VENDOR_ID   = [0x1006]
-    PRODUCT_ID  = [0x4023, 0x4025]
+    PRODUCT_ID  = [0x4023, 0x4024, 0x4025]
     BCD         = [0x0323]
 
     VENDOR_NAME = 'IRIVER'
-    WINDOWS_MAIN_MEM = ['STORY', 'STORY_EB05']
+    WINDOWS_MAIN_MEM = ['STORY', 'STORY_EB05', 'STORY_WI-FI']
     WINDOWS_CARD_A_MEM = ['STORY', 'STORY_SD']
 
     #OSX_MAIN_MEM = 'Kindle Internal Storage Media'

src/calibre/devices/misc.py

@@ -244,3 +244,43 @@ class TREKSTOR(USBMS):
     VENDOR_NAME = 'TREKSTOR'
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_PLAYER_7'
 
+class EEEREADER(USBMS):
+
+    name           = 'Asus EEE Reader device interface'
+    gui_name       = 'EEE Reader'
+    description    = _('Communicate with the EEE Reader')
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['epub', 'fb2', 'txt', 'pdf']
+
+    VENDOR_ID   = [0x0b05]
+    PRODUCT_ID  = [0x178f]
+    BCD         = [0x0319]
+
+    EBOOK_DIR_MAIN = 'Books'
+
+    VENDOR_NAME = 'LINUX'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET'
+
+class NEXTBOOK(USBMS):
+
+    name           = 'Nextbook device interface'
+    gui_name       = 'Nextbook'
+    description    = _('Communicate with the Nextbook Reader')
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['epub', 'fb2', 'txt', 'pdf']
+
+    VENDOR_ID   = [0x05e3]
+    PRODUCT_ID  = [0x0726]
+    BCD         = [0x021a]
+
+    EBOOK_DIR_MAIN = ''
+
+    VENDOR_NAME = 'NEXT2'
+    WINDOWS_MAIN_MEM = '1.0.14'
+

src/calibre/devices/usbms/books.py

@@ -14,6 +14,7 @@ from calibre.constants import preferred_encoding
 from calibre import isbytestring, force_unicode
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.icu import strcmp
+from calibre.utils.formatter import eval_formatter
 
 class Book(Metadata):
     def __init__(self, prefix, lpath, size=None, other=None):
@@ -107,23 +108,25 @@ class CollectionsBookList(BookList):
                 return sortattr
         return None
 
-    def compute_category_name(self, attr, category, field_meta):
+    def compute_category_name(self, field_key, field_value, field_meta):
         renames = tweaks['sony_collection_renaming_rules']
-        attr_name = renames.get(attr, None)
+        field_name = renames.get(field_key, None)
-        if attr_name is None:
+        if field_name is None:
             if field_meta['is_custom']:
-                attr_name = '(%s)'%field_meta['name']
+                field_name = field_meta['name']
             else:
-                attr_name = ''
+                field_name = ''
-        elif attr_name != '':
+        cat_name = eval_formatter.safe_format(
-            attr_name = '(%s)'%attr_name
+                        fmt=tweaks['sony_collection_name_template'],
-        cat_name = '%s %s'%(category, attr_name)
+                        kwargs={'category':field_name, 'value':field_value},
+                        error_value='GET_CATEGORY', book=None)
         return cat_name.strip()
 
     def get_collections(self, collection_attributes):
         from calibre.devices.usbms.driver import debug_print
         debug_print('Starting get_collections:', prefs['manage_device_metadata'])
         debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules'])
+        debug_print('Formatting template:', tweaks['sony_collection_name_template'])
         debug_print('Sorting rules:', tweaks['sony_collection_sorting_rules'])
 
         # Complexity: we can use renaming rules only when using automatic

src/calibre/ebooks/fb2/input.py

@@ -41,9 +41,12 @@ class FB2Input(InputFormatPlugin):
         from calibre.ebooks.metadata.opf2 import OPFCreator
         from calibre.ebooks.metadata.meta import get_metadata
         from calibre.ebooks.oeb.base import XLINK_NS, XHTML_NS, RECOVER_PARSER
+        from calibre.ebooks.chardet import xml_to_unicode
         NAMESPACES = {'f':FB2NS, 'l':XLINK_NS}
         log.debug('Parsing XML...')
         raw = stream.read().replace('\0', '')
+        raw = xml_to_unicode(raw, strip_encoding_pats=True,
+            assume_utf8=True)[0]
         try:
             doc = etree.fromstring(raw)
         except etree.XMLSyntaxError:

src/calibre/ebooks/metadata/book/base.py

@@ -159,6 +159,11 @@ class Metadata(object):
         try:
             return self.__getattribute__(field)
         except AttributeError:
+            if field.startswith('#') and field.endswith('_index'):
+                try:
+                    return self.get_extra(field[:-6])
+                except:
+                    pass
             return default
 
     def get_extra(self, field):

src/calibre/ebooks/metadata/fb2.py

@@ -9,6 +9,7 @@ import mimetypes, os
 from base64 import b64decode
 from lxml import etree
 from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.chardet import xml_to_unicode
 
 XLINK_NS     = 'http://www.w3.org/1999/xlink'
 def XLINK(name):
@@ -23,7 +24,10 @@ def get_metadata(stream):
     tostring = lambda x : etree.tostring(x, method='text',
             encoding=unicode).strip()
     parser = etree.XMLParser(recover=True, no_network=True)
-    root = etree.fromstring(stream.read(), parser=parser)
+    raw = stream.read()
+    raw = xml_to_unicode(raw, strip_encoding_pats=True,
+            assume_utf8=True)[0]
+    root = etree.fromstring(raw, parser=parser)
     authors, author_sort = [], None
     for au in XPath('//fb2:author')(root):
         fname = lname = author = None

src/calibre/ebooks/mobi/mobiml.py

@@ -468,8 +468,9 @@ class MobiMLizer(object):
                         vtag.append(child)
                     else:
                         break
-                for child in vbstate.para:
+                if vbstate.para is not None:
-                    vtag.append(child)
+                    for child in vbstate.para:
+                        vtag.append(child)
                 return
 
         if text or tag in CONTENT_TAGS or tag in NESTABLE_TAGS:

src/calibre/ebooks/mobi/reader.py

@@ -513,11 +513,14 @@ class MobiReader(object):
         mobi_version = self.book_header.mobi_version
         for x in root.xpath('//ncx'):
             x.getparent().remove(x)
+        svg_tags = []
         for i, tag in enumerate(root.iter(etree.Element)):
             tag.attrib.pop('xmlns', '')
             for x in tag.attrib:
                 if ':' in x:
                     del tag.attrib[x]
+            if tag.tag and barename(tag.tag) == 'svg':
+                svg_tags.append(tag)
             if tag.tag and barename(tag.tag.lower()) in \
                 ('country-region', 'place', 'placetype', 'placename',
                     'state', 'city', 'street', 'address', 'content', 'form'):
@@ -628,6 +631,11 @@ class MobiReader(object):
                 cls = cls + (' ' if cls else '') + ncls
                 attrib['class'] = cls
 
+        for tag in svg_tags:
+            p = tag.getparent()
+            if hasattr(p, 'remove'):
+                p.remove(tag)
+
     def create_opf(self, htmlfile, guide=None, root=None):
         mi = getattr(self.book_header.exth, 'mi', self.embedded_mi)
         if mi is None:

src/calibre/ebooks/pdf/fonts.cpp

@@ -1,6 +1,6 @@
 /**
  * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
- * License: GNU GPL v3
+ * License: GNU GPL v2+
  */
 
 

src/calibre/ebooks/pdf/fonts.h

@@ -1,6 +1,6 @@
 /**
  * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
- * License: GNU GPL v3
+ * License: GNU GPL v2+
  */
 
 

src/calibre/ebooks/pdf/images.cpp

@@ -1,3 +1,10 @@
+/**
+ * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
+ * License: GNU GPL v2+
+ */
+
+
+
 #include <stdio.h>
 #include <errno.h>
 #include <sstream>

src/calibre/ebooks/pdf/images.h

@@ -1,3 +1,10 @@
+/**
+ * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
+ * License: GNU GPL v2+
+ */
+
+
+
 #pragma once
 
 #include <vector>

src/calibre/ebooks/pdf/links.cpp

@@ -1,6 +1,6 @@
 /**
  * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
- * License: GNU GPL v3
+ * License: GNU GPL v2+
  */
 
 

src/calibre/ebooks/pdf/links.h

@@ -1,6 +1,6 @@
 /**
  * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
- * License: GNU GPL v3
+ * License: GNU GPL v2+
  */
 
 

src/calibre/ebooks/pdf/main.cpp

@@ -1,3 +1,10 @@
+/**
+ * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
+ * License: GNU GPL v2+
+ */
+
+
+
 #ifndef PDF2XML
 #define UNICODE
 #define PY_SSIZE_T_CLEAN

src/calibre/ebooks/pdf/reflow.cpp

@@ -1,6 +1,6 @@
 /**
  * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
- * License: GNU GPL v3
+ * License: GNU GPL v2+
  */
 
 #include <Outline.h>

src/calibre/ebooks/pdf/reflow.h

@@ -1,6 +1,6 @@
 /**
  * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
- * License: GNU GPL v3
+ * License: GNU GPL v2+
  * Based on pdftohtml from the poppler project.
  */
 

src/calibre/ebooks/pdf/utils.h

@@ -1,6 +1,6 @@
 /**
  * Copyright 2009 Kovid Goyal <kovid@kovidgoyal.net>
- * License: GNU GPL v3
+ * License: GNU GPL v2+
  */
 
 

src/calibre/ebooks/pdf/writer.py

@@ -25,10 +25,6 @@ from PyQt4.QtWebKit import QWebView
 
 from pyPdf import PdfFileWriter, PdfFileReader
 
-def get_pdf_printer():
-    return QPrinter(QPrinter.HighResolution)
-
-
 def get_custom_size(opts):
     custom_size = None
     if opts.custom_size != None:
@@ -42,12 +38,12 @@ def get_custom_size(opts):
                 custom_size = None
     return custom_size
 
-def setup_printer(opts, for_comic=False):
+def get_pdf_printer(opts, for_comic=False):
     from calibre.gui2 import is_ok_to_use_qt
     if not is_ok_to_use_qt():
         raise Exception('Not OK to use Qt')
 
-    printer = get_pdf_printer()
+    printer = QPrinter(QPrinter.HighResolution)
     custom_size = get_custom_size(opts)
 
     if opts.output_profile.short_name == 'default':
@@ -61,15 +57,22 @@ def setup_printer(opts, for_comic=False):
         h = opts.output_profile.comic_screen_size[1] if for_comic else \
                 opts.output_profile.height
         dpi = opts.output_profile.dpi
-        printer.setPaperSize(QSizeF(float(w) / dpi, float(h)/dpi), QPrinter.Inch)
+        printer.setPaperSize(QSizeF(float(w) / dpi, float(h) / dpi), QPrinter.Inch)
 
-    printer.setPageMargins(0, 0, 0, 0, QPrinter.Point)
+    if for_comic:
+        # Comic pages typically have their own margins, or their background
+        # color is not white, in which case the margin looks bad
+        printer.setPageMargins(0, 0, 0, 0, QPrinter.Point)
+    else:
+        printer.setPageMargins(opts.margin_left, opts.margin_top,
+                opts.margin_right, opts.margin_bottom, QPrinter.Point)
     printer.setOrientation(orientation(opts.orientation))
     printer.setOutputFormat(QPrinter.PdfFormat)
+    printer.setFullPage(True)
     return printer
 
 def get_printer_page_size(opts, for_comic=False):
-    printer = setup_printer(opts, for_comic=for_comic)
+    printer = get_pdf_printer(opts, for_comic=for_comic)
     size =  printer.paperSize(QPrinter.Millimeter)
     return size.width() / 10., size.height() / 10.
 
@@ -154,24 +157,11 @@ class PDFWriter(QObject): # {{{
 
         self.view.load(QUrl.fromLocalFile(item))
 
-    def get_printer(self, set_horz_margins=False):
-        printer = get_pdf_printer()
-        printer.setPaperSize(QSizeF(self.size[0] * 10, self.size[1] * 10), QPrinter.Millimeter)
-        if set_horz_margins:
-            printer.setPageMargins(0., self.opts.margin_top, 0.,
-                    self.opts.margin_bottom, QPrinter.Point)
-        else:
-            printer.setPageMargins(0, 0, 0, 0, QPrinter.Point)
-        printer.setOrientation(orientation(self.opts.orientation))
-        printer.setOutputFormat(QPrinter.PdfFormat)
-        printer.setFullPage(not set_horz_margins)
-        return printer
-
     def _render_html(self, ok):
         if ok:
             item_path = os.path.join(self.tmp_path, '%i.pdf' % len(self.combine_queue))
-            self.logger.debug('\tRendering item %s as %i' % (os.path.basename(str(self.view.url().toLocalFile())), len(self.combine_queue)))
+            self.logger.debug('\tRendering item %s as %i.pdf' % (os.path.basename(str(self.view.url().toLocalFile())), len(self.combine_queue)))
-            printer = self.get_printer(set_horz_margins=True)
+            printer = get_pdf_printer(self.opts)
             printer.setOutputFileName(item_path)
             self.view.print_(printer)
         self._render_book()
@@ -233,16 +223,11 @@ class ImagePDFWriter(object):
             os.remove(f.name)
 
     def render_images(self, outpath, mi, items):
-        printer = get_pdf_printer()
+        printer = get_pdf_printer(self.opts, for_comic=True)
-        printer.setPaperSize(QSizeF(self.size[0] * 10, self.size[1] * 10), QPrinter.Millimeter)
-        printer.setPageMargins(0, 0, 0, 0, QPrinter.Point)
-        printer.setOrientation(orientation(self.opts.orientation))
-        printer.setOutputFormat(QPrinter.PdfFormat)
         printer.setOutputFileName(outpath)
         printer.setDocName(mi.title)
         printer.setCreator(u'%s [%s]'%(__appname__, __version__))
         # Seems to be no way to set author
-        printer.setFullPage(True)
 
         painter = QPainter(printer)
         painter.setRenderHints(QPainter.Antialiasing|QPainter.SmoothPixmapTransform)

src/calibre/ebooks/snb/input.py

@@ -46,14 +46,27 @@ class SNBInput(InputFormatPlugin):
         meta = snbFile.GetFileStream('snbf/book.snbf')
         if meta != None:
             meta = etree.fromstring(meta)
-            oeb.metadata.add('title', meta.find('.//head/name').text)
+            l = { 'title'    : './/head/name',
-            oeb.metadata.add('creator', meta.find('.//head/author').text, attrib={'role':'aut'})
+                  'creator'  : './/head/author',
-            oeb.metadata.add('language', meta.find('.//head/language').text.lower().replace('_', '-'))
+                  'language' : './/head/language',
-            oeb.metadata.add('creator', meta.find('.//head/generator').text)
+                  'generator': './/head/generator',
-            oeb.metadata.add('publisher', meta.find('.//head/publisher').text)
+                  'publisher': './/head/publisher',
-            cover = meta.find('.//head/cover')
+                  'cover'    : './/head/cover', }
-            if cover != None and cover.text != None:
+            d = {}
-                oeb.guide.add('cover', 'Cover', cover.text)
+            for item in l:
+                node = meta.find(l[item])
+                if node != None:
+                    d[item] = node.text if node.text != None else ''
+                else:
+                    d[item] = ''
+
+            oeb.metadata.add('title', d['title'])
+            oeb.metadata.add('creator', d['creator'], attrib={'role':'aut'})
+            oeb.metadata.add('language', d['language'].lower().replace('_', '-'))
+            oeb.metadata.add('generator', d['generator'])
+            oeb.metadata.add('publisher', d['publisher'])
+            if d['cover'] != '':
+                oeb.guide.add('cover', 'Cover', d['cover'])
 
         bookid = str(uuid.uuid4())
         oeb.metadata.add('identifier', bookid, id='uuid_id', scheme='uuid')

src/calibre/ebooks/txt/output.py

@@ -29,8 +29,7 @@ class TXTOutput(OutputFormatPlugin):
         OptionRecommendation(name='output_encoding', recommended_value='utf-8',
             level=OptionRecommendation.LOW,
             help=_('Specify the character encoding of the output document. ' \
-            'The default is utf-8. Note: This option is not honored by all ' \
+            'The default is utf-8.')),
-            'formats.')),
         OptionRecommendation(name='inline_toc',
             recommended_value=False, level=OptionRecommendation.LOW,
             help=_('Add Table of Contents to beginning of the book.')),

src/calibre/gui2/actions/device.py

@@ -12,11 +12,15 @@ from PyQt4.Qt import QToolButton, QMenu, pyqtSignal, QIcon
 from calibre.gui2.actions import InterfaceAction
 from calibre.utils.smtp import config as email_config
 from calibre.constants import iswindows, isosx
+from calibre.customize.ui import is_disabled
+from calibre.devices.bambook.driver import BAMBOOK
 
 class ShareConnMenu(QMenu): # {{{
 
     connect_to_folder = pyqtSignal()
     connect_to_itunes = pyqtSignal()
+    connect_to_bambook = pyqtSignal()
+
     config_email = pyqtSignal()
     toggle_server = pyqtSignal()
     dont_add_to = frozenset(['toolbar-device', 'context-menu-device'])
@@ -34,6 +38,17 @@ class ShareConnMenu(QMenu): # {{{
         self.connect_to_itunes_action = mitem
         if not (iswindows or isosx):
             mitem.setVisible(False)
+        mitem = self.addAction(QIcon(I('devices/bambook.png')), _('Connect to Bambook'))
+        mitem.setEnabled(True)
+        mitem.triggered.connect(lambda x : self.connect_to_bambook.emit())
+        self.connect_to_bambook_action = mitem
+        bambook_visible = False
+        if not is_disabled(BAMBOOK):
+            device_ip = BAMBOOK.settings().extra_customization
+            if device_ip:
+                bambook_visible = True
+        self.connect_to_bambook_action.setVisible(bambook_visible)
+
         self.addSeparator()
         self.toggle_server_action = \
             self.addAction(QIcon(I('network-server.png')),
@@ -88,6 +103,7 @@ class ShareConnMenu(QMenu): # {{{
     def set_state(self, device_connected):
         self.connect_to_folder_action.setEnabled(not device_connected)
         self.connect_to_itunes_action.setEnabled(not device_connected)
+        self.connect_to_bambook_action.setEnabled(not device_connected)
 
 
 # }}}
@@ -126,6 +142,7 @@ class ConnectShareAction(InterfaceAction):
         self.qaction.setMenu(self.share_conn_menu)
         self.share_conn_menu.connect_to_folder.connect(self.gui.connect_to_folder)
         self.share_conn_menu.connect_to_itunes.connect(self.gui.connect_to_itunes)
+        self.share_conn_menu.connect_to_bambook.connect(self.gui.connect_to_bambook)
 
     def location_selected(self, loc):
         enabled = loc == 'library'

src/calibre/gui2/book_details.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 import os, collections, sys
 from Queue import Queue
 
-from PyQt4.Qt import QPixmap, QSize, QWidget, Qt, pyqtSignal, \
+from PyQt4.Qt import QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, \
     QPropertyAnimation, QEasingCurve, QThread, QApplication, QFontInfo, \
     QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette
 from PyQt4.QtWebKit import QWebView
@@ -18,7 +18,7 @@ from calibre.gui2.widgets import IMAGE_EXTENSIONS
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.constants import preferred_encoding
 from calibre.library.comments import comments_to_html
-from calibre.gui2 import config, open_local_file
+from calibre.gui2 import config, open_local_file, open_url
 from calibre.utils.icu import sort_key
 
 # render_rows(data) {{{
@@ -44,7 +44,10 @@ def render_rows(data):
             key = key.decode(preferred_encoding, 'replace')
         if isinstance(txt, str):
             txt = txt.decode(preferred_encoding, 'replace')
-        if '</font>' not in txt:
+        if key.endswith(u':html'):
+            key = key[:-5]
+            txt = comments_to_html(txt)
+        elif '</font>' not in txt:
             txt = prepare_string_for_xml(txt)
         if 'id' in data:
             if key == _('Path'):
@@ -409,6 +412,12 @@ class BookDetails(QWidget): # {{{
             self.view_specific_format.emit(int(id_), fmt)
         elif typ == 'devpath':
             open_local_file(val)
+        else:
+            try:
+                open_url(QUrl(link, QUrl.TolerantMode))
+            except:
+                import traceback
+                traceback.print_exc()
 
 
     def mouseDoubleClickEvent(self, ev):

src/calibre/gui2/comments_editor.py

@@ -259,6 +259,19 @@ class EditorWidget(QWebView): # {{{
 
         return property(fget=fget, fset=fset)
 
+    def keyPressEvent(self, ev):
+        if ev.key() in (Qt.Key_Tab, Qt.Key_Escape, Qt.Key_Backtab):
+            ev.ignore()
+        else:
+            return QWebView.keyPressed(self, ev)
+
+    def keyReleaseEvent(self, ev):
+        if ev.key() in (Qt.Key_Tab, Qt.Key_Escape, Qt.Key_Backtab):
+            ev.ignore()
+        else:
+            return QWebView.keyReleased(self, ev)
+
+
 # }}}
 
 # Highlighter {{{
@@ -479,6 +492,7 @@ class Editor(QWidget): # {{{
         QWidget.__init__(self, parent)
         self.toolbar1 = QToolBar(self)
         self.toolbar2 = QToolBar(self)
+        self.toolbar3 = QToolBar(self)
         self.editor = EditorWidget(self)
         self.tabs = QTabWidget(self)
         self.tabs.setTabPosition(self.tabs.South)
@@ -493,6 +507,7 @@ class Editor(QWidget): # {{{
         l.setContentsMargins(0, 0, 0, 0)
         l.addWidget(self.toolbar1)
         l.addWidget(self.toolbar2)
+        l.addWidget(self.toolbar3)
         l.addWidget(self.editor)
         self._layout.addWidget(self.tabs)
         self.tabs.addTab(self.wyswyg, _('Normal view'))
@@ -500,19 +515,7 @@ class Editor(QWidget): # {{{
         self.tabs.currentChanged[int].connect(self.change_tab)
         self.highlighter = Highlighter(self.code_edit.document())
 
-        for x in ('bold', 'italic', 'underline', 'strikethrough',
+        # toolbar1 {{{
-                'superscript', 'subscript', 'indent', 'outdent'):
-            ac = getattr(self.editor, 'action_'+x)
-            if x in ('superscript', 'indent'):
-                self.toolbar2.addSeparator()
-            self.toolbar2.addAction(ac)
-        self.toolbar2.addSeparator()
-
-        for x in ('left', 'center', 'right', 'justified'):
-            ac = getattr(self.editor, 'action_align_'+x)
-            self.toolbar2.addAction(ac)
-        self.toolbar2.addSeparator()
-
         self.toolbar1.addAction(self.editor.action_undo)
         self.toolbar1.addAction(self.editor.action_redo)
         self.toolbar1.addAction(self.editor.action_select_all)
@@ -523,21 +526,39 @@ class Editor(QWidget): # {{{
         for x in ('copy', 'cut', 'paste'):
             ac = getattr(self.editor, 'action_'+x)
             self.toolbar1.addAction(ac)
-        self.toolbar1.addSeparator()
 
+        self.toolbar1.addSeparator()
+        self.toolbar1.addAction(self.editor.action_background)
+        # }}}
+
+        # toolbar2 {{{
         for x in ('', 'un'):
             ac = getattr(self.editor, 'action_%sordered_list'%x)
-            self.toolbar1.addAction(ac)
+            self.toolbar2.addAction(ac)
-        self.toolbar1.addSeparator()
+        self.toolbar2.addSeparator()
+        for x in ('superscript', 'subscript', 'indent', 'outdent'):
+            self.toolbar2.addAction(getattr(self.editor, 'action_' + x))
+            if x in ('subscript', 'outdent'):
+                self.toolbar2.addSeparator()
 
-        self.toolbar1.addAction(self.editor.action_color)
+        self.toolbar2.addAction(self.editor.action_block_style)
-        self.toolbar1.addAction(self.editor.action_background)
+        w = self.toolbar2.widgetForAction(self.editor.action_block_style)
-        self.toolbar1.addSeparator()
-
-        self.toolbar1.addAction(self.editor.action_block_style)
-        w = self.toolbar1.widgetForAction(self.editor.action_block_style)
         w.setPopupMode(w.InstantPopup)
         self.toolbar2.addAction(self.editor.action_insert_link)
+        # }}}
+
+        # toolbar3 {{{
+        for x in ('bold', 'italic', 'underline', 'strikethrough'):
+            ac = getattr(self.editor, 'action_'+x)
+            self.toolbar3.addAction(ac)
+        self.toolbar3.addSeparator()
+
+        for x in ('left', 'center', 'right', 'justified'):
+            ac = getattr(self.editor, 'action_align_'+x)
+            self.toolbar3.addAction(ac)
+        self.toolbar3.addSeparator()
+        self.toolbar3.addAction(self.editor.action_color)
+        # }}}
 
         self.code_edit.textChanged.connect(self.code_dirtied)
         self.editor.page().contentsChanged.connect(self.wyswyg_dirtied)

src/calibre/gui2/convert/pdb_output.py

@@ -19,7 +19,7 @@ class PluginWidget(Widget, Ui_Form):
     ICON = I('mimetypes/unknown.png')
 
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
-        Widget.__init__(self, parent, ['format', 'inline_toc'])
+        Widget.__init__(self, parent, ['format', 'inline_toc', 'output_encoding'])
         self.db, self.book_id = db, book_id
         self.initialize_options(get_option, get_help, db, book_id)
 

src/calibre/gui2/convert/pdb_output.ui

@@ -27,7 +27,7 @@
    <item row="0" column="1">
     <widget class="QComboBox" name="opt_format"/>
    </item>
-   <item row="2" column="0">
+   <item row="3" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -40,13 +40,23 @@
      </property>
     </spacer>
    </item>
-   <item row="1" column="0">
+   <item row="2" column="0">
     <widget class="QCheckBox" name="opt_inline_toc">
      <property name="text">
       <string>&amp;Inline TOC</string>
      </property>
     </widget>
    </item>
+   <item row="1" column="0">
+    <widget class="QLabel" name="label_2">
+     <property name="text">
+      <string>Output Encoding:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="1">
+    <widget class="QLineEdit" name="opt_output_encoding"/>
+   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/convert/pml_output.py

@@ -17,6 +17,7 @@ class PluginWidget(Widget, Ui_Form):
     ICON = I('mimetypes/unknown.png')
 
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
-        Widget.__init__(self, parent, ['inline_toc', 'full_image_depth'])
+        Widget.__init__(self, parent, ['inline_toc', 'full_image_depth',
+            'output_encoding'])
         self.db, self.book_id = db, book_id
         self.initialize_options(get_option, get_help, db, book_id)

src/calibre/gui2/convert/pmlz_output.ui

@@ -14,7 +14,7 @@
    <string>Form</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
-   <item row="2" column="0">
+   <item row="3" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -27,20 +27,30 @@
      </property>
     </spacer>
    </item>
-   <item row="0" column="0">
+   <item row="1" column="0">
     <widget class="QCheckBox" name="opt_inline_toc">
      <property name="text">
       <string>&amp;Inline TOC</string>
      </property>
     </widget>
    </item>
-   <item row="1" column="0">
+   <item row="2" column="0">
     <widget class="QCheckBox" name="opt_full_image_depth">
      <property name="text">
       <string>Do not reduce image size and depth</string>
      </property>
     </widget>
    </item>
+   <item row="0" column="0">
+    <widget class="QLabel" name="label">
+     <property name="text">
+      <string>Output Encoding:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="0" column="1">
+    <widget class="QLineEdit" name="opt_output_encoding"/>
+   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/convert/txt_output.py

@@ -21,7 +21,8 @@ class PluginWidget(Widget, Ui_Form):
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
         Widget.__init__(self, parent,
         ['newline', 'max_line_length', 'force_max_line_length',
-        'inline_toc', 'markdown_format', 'keep_links', 'keep_image_references'])
+        'inline_toc', 'markdown_format', 'keep_links', 'keep_image_references',
+        'output_encoding'])
         self.db, self.book_id = db, book_id
         self.initialize_options(get_option, get_help, db, book_id)
 

src/calibre/gui2/convert/txt_output.ui

@@ -27,7 +27,7 @@
    <item row="0" column="1">
     <widget class="QComboBox" name="opt_newline"/>
    </item>
-   <item row="7" column="0">
+   <item row="8" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -40,7 +40,7 @@
      </property>
     </spacer>
    </item>
-   <item row="3" column="0" colspan="2">
+   <item row="4" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_inline_toc">
      <property name="text">
       <string>&amp;Inline TOC</string>
@@ -60,34 +60,44 @@
      </property>
     </widget>
    </item>
-   <item row="2" column="0" colspan="2">
+   <item row="3" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_force_max_line_length">
      <property name="text">
       <string>Force maximum line length</string>
      </property>
     </widget>
    </item>
-   <item row="4" column="0">
+   <item row="5" column="0">
     <widget class="QCheckBox" name="opt_markdown_format">
      <property name="text">
       <string>Apply Markdown formatting to text</string>
      </property>
     </widget>
    </item>
-   <item row="5" column="0">
+   <item row="6" column="0">
     <widget class="QCheckBox" name="opt_keep_links">
      <property name="text">
       <string>Do not remove links (&lt;a&gt; tags) before processing</string>
      </property>
     </widget>
    </item>
-   <item row="6" column="0">
+   <item row="7" column="0">
     <widget class="QCheckBox" name="opt_keep_image_references">
      <property name="text">
       <string>Do not remove image references before processing</string>
      </property>
     </widget>
    </item>
+   <item row="2" column="0">
+    <widget class="QLabel" name="label_3">
+     <property name="text">
+      <string>Output Encoding:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1">
+    <widget class="QLineEdit" name="opt_output_encoding"/>
+   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/custom_column_widgets.py

@@ -9,15 +9,17 @@ import sys
 from functools import partial
 
 from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \
-        QDate, QGroupBox, QVBoxLayout, QPlainTextEdit, QSizePolicy, \
+        QDate, QGroupBox, QVBoxLayout, QSizePolicy, \
         QSpacerItem, QIcon, QCheckBox, QWidget, QHBoxLayout, SIGNAL, \
         QPushButton
 
 from calibre.utils.date import qt_to_dt, now
 from calibre.gui2.widgets import TagsLineEdit, EnComboBox
+from calibre.gui2.comments_editor import Editor as CommentsEditor
 from calibre.gui2 import UNDEFINED_QDATE, error_dialog
 from calibre.utils.config import tweaks
 from calibre.utils.icu import sort_key
+from calibre.library.comments import comments_to_html
 
 class Base(object):
 
@@ -186,9 +188,9 @@ class Comments(Base):
         self._box = QGroupBox(parent)
         self._box.setTitle('&'+self.col_metadata['name'])
         self._layout = QVBoxLayout()
-        self._tb = QPlainTextEdit(self._box)
+        self._tb = CommentsEditor(self._box)
         self._tb.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Minimum)
-        self._tb.setTabChangesFocus(True)
+        #self._tb.setTabChangesFocus(True)
         self._layout.addWidget(self._tb)
         self._box.setLayout(self._layout)
         self.widgets = [self._box]
@@ -196,10 +198,10 @@ class Comments(Base):
     def setter(self, val):
         if val is None:
             val = ''
-        self._tb.setPlainText(val)
+        self._tb.html = comments_to_html(val)
 
     def getter(self):
-        val = unicode(self._tb.toPlainText()).strip()
+        val = unicode(self._tb.html).strip()
         if not val:
             val = None
         return val

src/calibre/gui2/device.py

@@ -24,6 +24,7 @@ from calibre.utils.filenames import ascii_filename
 from calibre.devices.errors import FreeSpaceError
 from calibre.devices.apple.driver import ITUNES_ASYNC
 from calibre.devices.folder_device.driver import FOLDER_DEVICE
+from calibre.devices.bambook.driver import BAMBOOK, BAMBOOKWifi
 from calibre.ebooks.metadata.meta import set_metadata
 from calibre.constants import DEBUG
 from calibre.utils.config import prefs, tweaks
@@ -635,6 +636,10 @@ class DeviceMixin(object): # {{{
         if dir is not None:
             self.device_manager.mount_device(kls=FOLDER_DEVICE, kind='folder', path=dir)
 
+    def connect_to_bambook(self):
+        self.device_manager.mount_device(kls=BAMBOOKWifi, kind='bambook', 
+                                         path=BAMBOOK.settings().extra_customization)
+
     def connect_to_itunes(self):
         self.device_manager.mount_device(kls=ITUNES_ASYNC, kind='itunes', path=None)
 

src/calibre/gui2/dialogs/book_info.py

@@ -9,9 +9,10 @@ from PyQt4.Qt import QCoreApplication, SIGNAL, QModelIndex, QTimer, Qt, \
     QDialog, QPixmap, QGraphicsScene, QIcon, QSize
 
 from calibre.gui2.dialogs.book_info_ui import Ui_BookInfo
-from calibre.gui2 import dynamic, open_local_file
+from calibre.gui2 import dynamic, open_local_file, open_url
 from calibre import fit_image
 from calibre.library.comments import comments_to_html
+from calibre.utils.icu import sort_key
 
 class BookInfo(QDialog, Ui_BookInfo):
 
@@ -21,6 +22,8 @@ class BookInfo(QDialog, Ui_BookInfo):
         self.setupUi(self)
         self.cover_pixmap = None
         self.comments.sizeHint = self.comments_size_hint
+        self.comments.page().setLinkDelegationPolicy(self.comments.page().DelegateAllLinks)
+        self.comments.linkClicked(self.link_clicked)
         self.view_func = view_func
 
 
@@ -40,6 +43,8 @@ class BookInfo(QDialog, Ui_BookInfo):
         screen_height = desktop.availableGeometry().height() - 100
         self.resize(self.size().width(), screen_height)
 
+    def link_clicked(self, url):
+        open_url(url)
 
     def comments_size_hint(self):
         return QSize(350, 250)
@@ -114,6 +119,7 @@ class BookInfo(QDialog, Ui_BookInfo):
             lines = [x if x.strip() else '<br><br>' for x in lines]
             comments = '\n'.join(lines)
         self.comments.setHtml('<div>%s</div>' % comments)
+        self.comments.page().setLinkDelegationPolicy(self.comments.page().DelegateAllLinks)
         cdata = info.pop('cover', '')
         self.cover_pixmap = QPixmap.fromImage(cdata)
         self.resize_cover()
@@ -130,9 +136,12 @@ class BookInfo(QDialog, Ui_BookInfo):
             for f in formats:
                 f = f.strip()
                 info[_('Formats')] += '<a href="%s">%s</a>, '%(f,f)
-        for key in info.keys():
+        for key in sorted(info.keys(), key=sort_key):
             if key == 'id': continue
             txt  = info[key]
+            if key.endswith(':html'):
+                key = key[:-5]
+                txt = comments_to_html(txt)
             if key != _('Path'):
                 txt  = u'<br />\n'.join(textwrap.wrap(txt, 120))
             rows += u'<tr><td><b>%s:</b></td><td>%s</td></tr>'%(key, txt)

src/calibre/gui2/dialogs/comments_dialog.py

@@ -5,6 +5,7 @@ __license__   = 'GPL v3'
 
 from PyQt4.Qt import Qt, QDialog, QDialogButtonBox
 from calibre.gui2.dialogs.comments_dialog_ui import Ui_CommentsDialog
+from calibre.library.comments import comments_to_html
 
 class CommentsDialog(QDialog, Ui_CommentsDialog):
 
@@ -18,8 +19,8 @@ class CommentsDialog(QDialog, Ui_CommentsDialog):
         self.setWindowIcon(icon)
 
         if text is not None:
-            self.textbox.setPlainText(text)
+            self.textbox.html = comments_to_html(text)
-        self.textbox.setTabChangesFocus(True)
+        # self.textbox.setTabChangesFocus(True)
         self.buttonBox.button(QDialogButtonBox.Ok).setText(_('&OK'))
         self.buttonBox.button(QDialogButtonBox.Cancel).setText(_('&Cancel'))
 

src/calibre/gui2/dialogs/comments_dialog.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>336</width>
+    <width>400</width>
-    <height>235</height>
+    <height>400</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -19,22 +19,29 @@
   <property name="windowTitle">
    <string>Edit Comments</string>
   </property>
-   <layout class="QVBoxLayout" name="verticalLayout">
+  <layout class="QVBoxLayout" name="verticalLayout">
-    <item>
+   <item>
-     <widget class="QPlainTextEdit" name="textbox"/>
+    <widget class="Editor" name="textbox" native="true"/>
-    </item>
+   </item>
-    <item>
+   <item>
-     <widget class="QDialogButtonBox" name="buttonBox">
+    <widget class="QDialogButtonBox" name="buttonBox">
-      <property name="orientation">
+     <property name="orientation">
-       <enum>Qt::Horizontal</enum>
+      <enum>Qt::Horizontal</enum>
-      </property>
+     </property>
-      <property name="standardButtons">
+     <property name="standardButtons">
-       <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
-      </property>
+     </property>
-     </widget>
+    </widget>
-    </item>
+   </item>
-   </layout>
+  </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>Editor</class>
+   <extends>QWidget</extends>
+   <header>calibre/gui2/comments_editor.h</header>
+  </customwidget>
+ </customwidgets>
  <resources/>
  <connections>
   <connection>

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -414,6 +414,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         self.s_r_template.completer().setCaseSensitivity(Qt.CaseSensitive)
 
         self.s_r_search_mode_changed(self.search_mode.currentIndex())
+        self.multiple_separator.setFixedWidth(30)
+        self.multiple_separator.setText(' ::: ')
+        self.multiple_separator.textChanged.connect(self.s_r_separator_changed)
+        self.results_count.valueChanged[int].connect(self.s_r_display_bounds_changed)
+        self.starting_from.valueChanged[int].connect(self.s_r_display_bounds_changed)
 
     def s_r_get_field(self, mi, field):
         if field:
@@ -436,6 +441,9 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             val = []
         return val
 
+    def s_r_display_bounds_changed(self, i):
+        self.s_r_search_field_changed(self.search_field.currentIndex())
+
     def s_r_template_changed(self):
         self.s_r_search_field_changed(self.search_field.currentIndex())
 
@@ -451,21 +459,23 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             mi = self.db.get_metadata(self.ids[i], index_is_id=True)
             src = unicode(self.search_field.currentText())
             t = self.s_r_get_field(mi, src)
-            w.setText(''.join(t[0:1]))
+            if len(t) > 1:
+                t = t[self.starting_from.value()-1:
+                      self.starting_from.value()-1 + self.results_count.value()]
+            w.setText(unicode(self.multiple_separator.text()).join(t))
 
         if self.search_mode.currentIndex() == 0:
             self.destination_field.setCurrentIndex(idx)
         else:
+            self.s_r_destination_field_changed(self.destination_field.currentText())
             self.s_r_paint_results(None)
 
     def s_r_destination_field_changed(self, txt):
         txt = unicode(txt)
-        self.comma_separated.setEnabled(True)
+        if not txt:
-        if txt:
+            txt = unicode(self.search_field.currentText())
-            fm = self.db.metadata_for_field(txt)
+        if txt and txt in self.writable_fields:
-            if fm['is_multiple']:
+            self.destination_field_fm = self.db.metadata_for_field(txt)
-                self.comma_separated.setEnabled(False)
-                self.comma_separated.setChecked(True)
         self.s_r_paint_results(None)
 
     def s_r_search_mode_changed(self, val):
@@ -493,6 +503,9 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             self.s_r_heading.setText('<p>'+self.main_heading + self.regexp_heading)
         self.s_r_paint_results(None)
 
+    def s_r_separator_changed(self, txt):
+        self.s_r_search_field_changed(self.search_field.currentIndex())
+
     def s_r_set_colors(self):
         if self.s_r_error is not None:
             col = 'rgb(255, 0, 0, 20%)'
@@ -533,6 +546,22 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             dest = src
         dest_mode = self.replace_mode.currentIndex()
 
+        if self.destination_field_fm['is_multiple']:
+            if self.comma_separated.isChecked():
+                if dest == 'authors':
+                    splitter = ' & '
+                else:
+                    splitter = ','
+
+                res = []
+                for v in val:
+                    for x in v.split(splitter):
+                        if x.strip():
+                            res.append(x.strip())
+                val = res
+            else:
+                val = [v.replace(',', '') for v in val]
+
         if dest_mode != 0:
             dest_val = mi.get(dest, '')
             if dest_val is None:
@@ -592,8 +621,13 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             wr = getattr(self, 'book_%d_result'%(i+1))
             try:
                 result = self.s_r_do_regexp(mi)
-                t = self.s_r_do_destination(mi, result[0:1])
+                t = self.s_r_do_destination(mi, result)
-                t = self.s_r_replace_mode_separator().join(t)
+                if len(t) > 1 and self.destination_field_fm['is_multiple']:
+                    t = t[self.starting_from.value()-1:
+                          self.starting_from.value()-1 + self.results_count.value()]
+                    t = unicode(self.multiple_separator.text()).join(t)
+                else:
+                    t = self.s_r_replace_mode_separator().join(t)
                 wr.setText(t)
             except Exception as e:
                 self.s_r_error = e

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -478,7 +478,7 @@ Future conversion of these books will use the default settings.</string>
            <item>
             <widget class="QLabel" name="xlabel_24">
              <property name="text">
-              <string>Search mode:</string>
+              <string>Search &amp;mode:</string>
              </property>
              <property name="buddy">
               <cstring>search_mode</cstring>
@@ -559,7 +559,7 @@ Future conversion of these books will use the default settings.</string>
             <string>Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored</string>
            </property>
            <property name="text">
-            <string>Case sensitive</string>
+            <string>Cas&amp;e sensitive</string>
            </property>
            <property name="checked">
             <bool>true</bool>
@@ -588,7 +588,7 @@ Future conversion of these books will use the default settings.</string>
            <item>
             <widget class="QLabel" name="label_41">
              <property name="text">
-              <string>Apply function after replace:</string>
+              <string>&amp;Apply function after replace:</string>
              </property>
              <property name="buddy">
               <cstring>replace_func</cstring>
@@ -641,7 +641,7 @@ If blank, the source field is used if the field is modifiable</string>
            <item>
             <widget class="QLabel" name="replace_mode_label">
              <property name="text">
-              <string>Mode:</string>
+              <string>M&amp;ode:</string>
              </property>
              <property name="buddy">
               <cstring>replace_mode</cstring>
@@ -658,11 +658,12 @@ If blank, the source field is used if the field is modifiable</string>
            <item>
             <widget class="QCheckBox" name="comma_separated">
              <property name="toolTip">
-              <string>If the replace mode is prepend or append, then this box indicates whether a comma or
+              <string>Specifies whether result items should be split into multiple values or
-nothing should be put between the original text and the inserted text</string>
+left as single values. This option has the most effect when the source field is
+not multiple and the destination field is multiple</string>
              </property>
              <property name="text">
-              <string>use comma</string>
+              <string>Split &amp;result</string>
              </property>
              <property name="checked">
               <bool>true</bool>
@@ -684,25 +685,91 @@ nothing should be put between the original text and the inserted text</string>
            </item>
           </layout>
          </item>
-         <item row="8" column="1">
+         <item row="8" column="1" colspan="2">
-          <widget class="QLabel" name="xlabel_3">
+          <layout class="QHBoxLayout" name="horizontalLayout_21">
-           <property name="text">
+           <item>
-            <string>Test &amp;text</string>
+            <spacer name="HSpacer_347">
-           </property>
+             <property name="orientation">
-           <property name="buddy">
+              <enum>Qt::Horizontal</enum>
-            <cstring>test_text</cstring>
+             </property>
-           </property>
+             <property name="sizeHint" stdset="0">
-          </widget>
+              <size>
-         </item>
+               <width>20</width>
-         <item row="8" column="2">
+               <height>0</height>
-          <widget class="QLabel" name="label_51">
+              </size>
-           <property name="text">
+             </property>
-            <string>Test re&amp;sult</string>
+            </spacer>
-           </property>
+           </item>
-           <property name="buddy">
+           <item>
-            <cstring>test_result</cstring>
+            <widget class="QLabel" name="xlabel_412">
-           </property>
+             <property name="text">
-          </widget>
+              <string>For multiple-valued fields, sho&amp;w</string>
+             </property>
+             <property name="buddy">
+              <cstring>results_count</cstring>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QSpinBox" name="results_count">
+             <property name="enabled">
+              <bool>true</bool>
+             </property>
+             <property name="minimum">
+              <number>1</number>
+             </property>
+             <property name="maximum">
+              <number>999</number>
+             </property>
+             <property name="value">
+              <number>999</number>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QLabel" name="xlabel_412">
+             <property name="text">
+              <string>values starting a&amp;t</string>
+             </property>
+             <property name="buddy">
+              <cstring>starting_from</cstring>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QSpinBox" name="starting_from">
+             <property name="enabled">
+              <bool>true</bool>
+             </property>
+             <property name="minimum">
+              <number>1</number>
+             </property>
+             <property name="maximum">
+              <number>999</number>
+             </property>
+             <property name="value">
+              <number>1</number>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QLabel" name="xlabel_41">
+             <property name="text">
+              <string>with values separated b&amp;y</string>
+             </property>
+             <property name="buddy">
+              <cstring>multiple_separator</cstring>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QLineEdit" name="multiple_separator">
+             <property name="toolTip">
+              <string>Used when displaying test results to separate values in multiple-valued fields</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
          </item>
          <item row="9" column="0" colspan="4">
           <widget class="QScrollArea" name="scrollArea11">
@@ -722,6 +789,20 @@ nothing should be put between the original text and the inserted text</string>
              </rect>
             </property>
             <layout class="QGridLayout" name="testgrid">
+             <item row="7" column="1">
+              <widget class="QLabel" name="xlabel_3">
+               <property name="text">
+                <string>Test text</string>
+               </property>
+              </widget>
+             </item>
+             <item row="7" column="2">
+              <widget class="QLabel" name="xlabel_3">
+               <property name="text">
+                <string>Test result</string>
+               </property>
+              </widget>
+             </item>
              <item row="8" column="0">
               <widget class="QLabel" name="label_31">
                <property name="text">
@@ -823,7 +904,9 @@ nothing should be put between the original text and the inserted text</string>
   <tabstop>destination_field</tabstop>
   <tabstop>replace_mode</tabstop>
   <tabstop>comma_separated</tabstop>
-  <tabstop>scrollArea11</tabstop>
+  <tabstop>results_count</tabstop>
+  <tabstop>starting_from</tabstop>
+  <tabstop>multiple_separator</tabstop>
   <tabstop>test_text</tabstop>
   <tabstop>test_result</tabstop>
  </tabstops>

src/calibre/gui2/dialogs/template_dialog.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+__license__   = 'GPL v3'
+
+from PyQt4.Qt import Qt, QDialog, QDialogButtonBox
+from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog
+
+class TemplateDialog(QDialog, Ui_TemplateDialog):
+
+    def __init__(self, parent, text):
+        QDialog.__init__(self, parent)
+        Ui_TemplateDialog.__init__(self)
+        self.setupUi(self)
+        # Remove help icon on title bar
+        icon = self.windowIcon()
+        self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint))
+        self.setWindowIcon(icon)
+
+        if text is not None:
+            self.textbox.setPlainText(text)
+        self.textbox.setTabStopWidth(50)
+        self.buttonBox.button(QDialogButtonBox.Ok).setText(_('&OK'))
+        self.buttonBox.button(QDialogButtonBox.Cancel).setText(_('&Cancel'))
+

src/calibre/gui2/dialogs/template_dialog.ui

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>TemplateDialog</class>
+ <widget class="QDialog" name="TemplateDialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>500</width>
+    <height>235</height>
+   </rect>
+  </property>
+  <property name="sizePolicy">
+   <sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
+    <horstretch>0</horstretch>
+    <verstretch>0</verstretch>
+   </sizepolicy>
+  </property>
+  <property name="windowTitle">
+   <string>Edit Comments</string>
+  </property>
+   <layout class="QVBoxLayout" name="verticalLayout">
+    <item>
+     <widget class="QPlainTextEdit" name="textbox"/>
+    </item>
+    <item>
+     <widget class="QDialogButtonBox" name="buttonBox">
+      <property name="orientation">
+       <enum>Qt::Horizontal</enum>
+      </property>
+      <property name="standardButtons">
+       <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+      </property>
+     </widget>
+    </item>
+   </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>TemplateDialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>229</x>
+     <y>211</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>234</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>TemplateDialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>297</x>
+     <y>217</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>234</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/email.py

@@ -22,6 +22,7 @@ from calibre.customize.ui import available_input_formats, available_output_forma
 from calibre.ebooks.metadata import authors_to_string
 from calibre.constants import preferred_encoding
 from calibre.gui2 import config, Dispatcher, warning_dialog
+from calibre.utils.config import tweaks
 
 class EmailJob(BaseJob): # {{{
 
@@ -83,7 +84,7 @@ class Emailer(Thread): # {{{
         rh = opts.relay_host
         if rh and (
             'gmail.com' in rh or 'live.com' in rh):
-            self.rate_limit = 301
+            self.rate_limit = tweaks['public_smtp_relay_delay']
 
     def stop(self):
         self._run = False

src/calibre/gui2/library/__init__.py

@@ -5,6 +5,4 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import Qt
+DEFAULT_SORT = ('timestamp', False)
-
-DEFAULT_SORT = ('timestamp', Qt.DescendingOrder)

src/calibre/gui2/library/delegates.py

@@ -13,7 +13,7 @@ from PyQt4.Qt import QColor, Qt, QModelIndex, QSize, \
                      QPen, QStyle, QPainter, QStyleOptionViewItemV4, \
                      QIcon,  QDoubleSpinBox, QVariant, QSpinBox, \
                      QStyledItemDelegate, QCompleter, \
-                     QComboBox
+                     QComboBox, QTextDocument
 
 from calibre.gui2 import UNDEFINED_QDATE, error_dialog
 from calibre.gui2.widgets import EnLineEdit, TagsLineEdit
@@ -22,6 +22,8 @@ from calibre.utils.config import tweaks
 from calibre.utils.formatter import validation_formatter
 from calibre.utils.icu import sort_key
 from calibre.gui2.dialogs.comments_dialog import CommentsDialog
+from calibre.gui2.dialogs.template_dialog import TemplateDialog
+
 
 class RatingDelegate(QStyledItemDelegate): # {{{
     COLOR    = QColor("blue")
@@ -294,6 +296,24 @@ class CcCommentsDelegate(QStyledItemDelegate): # {{{
     Delegate for comments data.
     '''
 
+    def __init__(self, parent):
+        QStyledItemDelegate.__init__(self, parent)
+        self.document = QTextDocument()
+
+    def paint(self, painter, option, index):
+        style = self.parent().style()
+        self.document.setHtml(index.data(Qt.DisplayRole).toString())
+        painter.save()
+        if hasattr(QStyle, 'CE_ItemViewItem'):
+            style.drawControl(QStyle.CE_ItemViewItem, option,
+                    painter, self.parent())
+        elif option.state & QStyle.State_Selected:
+            painter.fillRect(option.rect, option.palette.highlight())
+        painter.setClipRect(option.rect)
+        painter.translate(option.rect.topLeft())
+        self.document.drawContents(painter)
+        painter.restore()
+
     def createEditor(self, parent, option, index):
         m = index.model()
         col = m.column_map[index.column()]
@@ -301,11 +321,11 @@ class CcCommentsDelegate(QStyledItemDelegate): # {{{
         editor = CommentsDialog(parent, text)
         d = editor.exec_()
         if d:
-            m.setData(index, QVariant(editor.textbox.toPlainText()), Qt.EditRole)
+            m.setData(index, QVariant(editor.textbox.html), Qt.EditRole)
         return None
 
     def setModelData(self, editor, model, index):
-        model.setData(index, QVariant(editor.textbox.toPlainText()), Qt.EditRole)
+        model.setData(index, QVariant(editor.textbox.html), Qt.EditRole)
 # }}}
 
 class CcBoolDelegate(QStyledItemDelegate): # {{{
@@ -351,7 +371,7 @@ class CcTemplateDelegate(QStyledItemDelegate): # {{{
     def createEditor(self, parent, option, index):
         m = index.model()
         text = m.custom_columns[m.column_map[index.column()]]['display']['composite_template']
-        editor = CommentsDialog(parent, text)
+        editor = TemplateDialog(parent, text)
         editor.setWindowTitle(_("Edit template"))
         editor.textbox.setTabChangesFocus(False)
         editor.textbox.setTabStopWidth(20)

src/calibre/gui2/library/models.py

@@ -247,9 +247,10 @@ class BooksModel(QAbstractTableModel): # {{{
         if not self.db:
             return
         self.about_to_be_sorted.emit(self.db.id)
-        ascending = order == Qt.AscendingOrder
+        if not isinstance(order, bool):
+            order = order == Qt.AscendingOrder
         label = self.column_map[col]
-        self.db.sort(label, ascending)
+        self.db.sort(label, order)
         if reset:
             self.reset()
         self.sorted_on = (label, order)
@@ -334,6 +335,8 @@ class BooksModel(QAbstractTableModel): # {{{
             if key not in cf_to_display:
                 continue
             name, val = mi.format_field(key)
+            if mi.metadata_for_field(key)['datatype'] == 'comments':
+                name += ':html'
             if val:
                 data[name] = val
         return data

src/calibre/gui2/library/views.py

@@ -57,6 +57,11 @@ class BooksView(QTableView): # {{{
         elif tweaks['doubleclick_on_library_view'] == 'open_viewer':
             self.setEditTriggers(self.SelectedClicked|self.editTriggers())
             self.doubleClicked.connect(parent.iactions['View'].view_triggered)
+        elif tweaks['doubleclick_on_library_view'] == 'edit_metadata':
+            # Must not enable single-click to edit, or the field will remain
+            # open in edit mode underneath the edit metadata dialog
+            self.doubleClicked.connect(
+                        partial(parent.iactions['Edit Metadata'].edit_metadata, checked=False))
 
         self.drag_allowed = True
         self.setDragEnabled(True)
@@ -160,7 +165,7 @@ class BooksView(QTableView): # {{{
                     partial(self.column_header_context_handler,
                         action='descending', column=col))
             if self._model.sorted_on[0] == col:
-                ac = a if self._model.sorted_on[1] == Qt.AscendingOrder else d
+                ac = a if self._model.sorted_on[1] else d
                 ac.setCheckable(True)
                 ac.setChecked(True)
             if col not in ('ondevice', 'rating', 'inlibrary') and \
@@ -277,17 +282,21 @@ class BooksView(QTableView): # {{{
     def cleanup_sort_history(self, sort_history):
         history = []
         for col, order in sort_history:
+            if not isinstance(order, bool):
+                continue
             if col == 'date':
                 col = 'timestamp'
-            if col in self.column_map and (not history or history[0][0] != col):
+            if col in self.column_map:
-                history.append([col, order])
+                if (not history or history[-1][0] != col):
+                    history.append([col, order])
         return history
 
     def apply_sort_history(self, saved_history):
         if not saved_history:
             return
         for col, order in reversed(self.cleanup_sort_history(saved_history)[:3]):
-            self.sortByColumn(self.column_map.index(col), order)
+            self.sortByColumn(self.column_map.index(col),
+                              Qt.AscendingOrder if order else Qt.DescendingOrder)
 
     def apply_state(self, state):
         h = self.column_header

src/calibre/gui2/tag_view.py

@@ -10,22 +10,25 @@ Browsing book collection by tags.
 from itertools import izip
 from functools import partial
 
-from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \
+from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, QFont, QSize, \
-                     QFont, QSize, QIcon, QPoint, QVBoxLayout, QComboBox, \
+                     QIcon, QPoint, QVBoxLayout, QHBoxLayout, QComboBox,\
                      QAbstractItemModel, QVariant, QModelIndex, QMenu, \
-                     QPushButton, QWidget, QItemDelegate
+                     QPushButton, QWidget, QItemDelegate, QString
 
 from calibre.ebooks.metadata import title_sort
 from calibre.gui2 import config, NONE
 from calibre.library.field_metadata import TagsIcons, category_icon_map
+from calibre.library.database2 import Tag
 from calibre.utils.config import tweaks
-from calibre.utils.icu import sort_key
+from calibre.utils.icu import sort_key, upper, lower
 from calibre.utils.search_query_parser import saved_searches
-from calibre.gui2 import error_dialog
+from calibre.utils.formatter import eval_formatter
+from calibre.gui2 import error_dialog, warning_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.tag_categories import TagCategories
 from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.dialogs.edit_authors_dialog import EditAuthorsDialog
+from calibre.gui2.widgets import HistoryLineEdit
 
 class TagDelegate(QItemDelegate): # {{{
 
@@ -52,6 +55,8 @@ class TagDelegate(QItemDelegate): # {{{
             painter.setClipRect(r)
 
         # Paint the text
+        if item.boxed:
+            painter.drawRoundedRect(r.adjusted(1,1,-1,-1), 5, 5)
         r.setLeft(r.left()+r.height()+3)
         painter.drawText(r, Qt.AlignLeft|Qt.AlignVCenter,
                         model.data(index, Qt.DisplayRole).toString())
@@ -331,12 +336,13 @@ class TagsView(QTreeView): # {{{
     # If the number of user categories changed,  if custom columns have come or
     # gone, or if columns have been hidden or restored, we must rebuild the
     # model. Reason: it is much easier than reconstructing the browser tree.
-    def set_new_model(self):
+    def set_new_model(self, filter_categories_by=None):
         try:
             self._model = TagsModel(self.db, parent=self,
                                     hidden_categories=self.hidden_categories,
                                     search_restriction=self.search_restriction,
-                                    drag_drop_finished=self.drag_drop_finished)
+                                    drag_drop_finished=self.drag_drop_finished,
+                                    filter_categories_by=filter_categories_by)
             self.setModel(self._model)
         except:
             # The DB must be gone. Set the model to None and hope that someone
@@ -355,6 +361,7 @@ class TagTreeItem(object): # {{{
                  parent=None, tooltip=None, category_key=None):
         self.parent = parent
         self.children = []
+        self.boxed = False
         if self.parent is not None:
             self.parent.append(self)
         if data is None:
@@ -400,7 +407,7 @@ class TagTreeItem(object): # {{{
 
     def category_data(self, role):
         if role == Qt.DisplayRole:
-            return QVariant(self.py_name + ' [%d]'%len(self.children))
+            return QVariant(self.py_name + ' [%d]'%len(self.child_tags()))
         if role == Qt.DecorationRole:
             return self.icon
         if role == Qt.FontRole:
@@ -441,12 +448,22 @@ class TagTreeItem(object): # {{{
         if self.type == self.TAG:
             self.tag.state = (self.tag.state + 1)%3
 
+    def child_tags(self):
+        res = []
+        for t in self.children:
+            if t.type == TagTreeItem.CATEGORY:
+                for c in t.children:
+                    res.append(c)
+            else:
+                res.append(t)
+        return res
     # }}}
 
 class TagsModel(QAbstractItemModel): # {{{
 
     def __init__(self, db, parent, hidden_categories=None,
-            search_restriction=None, drag_drop_finished=None):
+            search_restriction=None, drag_drop_finished=None,
+            filter_categories_by=None):
         QAbstractItemModel.__init__(self, parent)
 
         # must do this here because 'QPixmap: Must construct a QApplication
@@ -466,6 +483,7 @@ class TagsModel(QAbstractItemModel): # {{{
         self.hidden_categories = hidden_categories
         self.search_restriction = search_restriction
         self.row_map = []
+        self.filter_categories_by = filter_categories_by
 
         # get_node_tree cannot return None here, because row_map is empty
         data = self.get_node_tree(config['sort_tags_by'])
@@ -477,19 +495,11 @@ class TagsModel(QAbstractItemModel): # {{{
                 tt = _('The lookup/search name is "{0}"').format(r)
             else:
                 tt = ''
-            c = TagTreeItem(parent=self.root_item,
+            TagTreeItem(parent=self.root_item,
                     data=self.categories[i],
                     category_icon=self.category_icon_map[r],
                     tooltip=tt, category_key=r)
-            # This duplicates code in refresh(). Having it here as well
+        self.refresh(data=data)
-            # can save seconds during startup, because we avoid a second
-            # call to get_node_tree.
-            for tag in data[r]:
-                if r not in self.categories_with_ratings and \
-                            not self.db.field_metadata[r]['is_custom'] and \
-                            not self.db.field_metadata[r]['kind'] == 'user':
-                    tag.avg_rating = None
-                TagTreeItem(parent=c, data=tag, icon_map=self.icon_state_map)
 
     def mimeTypes(self):
         return ["application/calibre+from_library"]
@@ -641,6 +651,11 @@ class TagsModel(QAbstractItemModel): # {{{
         else:
             data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map)
 
+        if self.filter_categories_by:
+            for category in data.keys():
+                data[category] = [t for t in data[category]
+                        if lower(t.name).find(self.filter_categories_by) >= 0]
+
         tb_categories = self.db.field_metadata
         for category in tb_categories:
             if category in data: # The search category can come and go
@@ -652,35 +667,85 @@ class TagsModel(QAbstractItemModel): # {{{
             return None
         return data
 
-    def refresh(self):
+    def refresh(self, data=None):
-        data = self.get_node_tree(config['sort_tags_by']) # get category data
+        sort_by = config['sort_tags_by']
+        if data is None:
+            data = self.get_node_tree(sort_by) # get category data
         if data is None:
             return False
         row_index = -1
+        empty_tag = Tag('')
+        collapse = tweaks['categories_collapse_more_than']
+        collapse_model = tweaks['categories_collapse_model']
+        if sort_by == 'name':
+            collapse_template = tweaks['categories_collapsed_name_template']
+        elif sort_by == 'rating':
+            collapse_model = 'partition'
+            collapse_template = tweaks['categories_collapsed_rating_template']
+        else:
+            collapse_model = 'partition'
+            collapse_template = tweaks['categories_collapsed_popularity_template']
+        collapse_letter = None
+
         for i, r in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
             row_index += 1
             category = self.root_item.children[row_index]
-            names = [t.tag.name for t in category.children]
+            names = []
-            states = [t.tag.state for t in category.children]
+            states = []
+            children = category.child_tags()
+            states = [t.tag.state for t in children]
+            names = [t.tag.name for names in children]
             state_map = dict(izip(names, states))
             category_index = self.index(row_index, 0, QModelIndex())
+            category_node = category_index.internalPointer()
             if len(category.children) > 0:
                 self.beginRemoveRows(category_index, 0,
                         len(category.children)-1)
                 category.children = []
                 self.endRemoveRows()
-            if len(data[r]) > 0:
+            cat_len = len(data[r])
-                self.beginInsertRows(category_index, 0, len(data[r])-1)
+            if cat_len <= 0:
-                for tag in data[r]:
+                continue
-                    if r not in self.categories_with_ratings and \
+
+            self.beginInsertRows(category_index, 0, len(data[r])-1)
+            clear_rating = True if r not in self.categories_with_ratings and \
                                 not self.db.field_metadata[r]['is_custom'] and \
-                                not self.db.field_metadata[r]['kind'] == 'user':
+                                not self.db.field_metadata[r]['kind'] == 'user' \
-                        tag.avg_rating = None
+                            else False
-                    tag.state = state_map.get(tag.name, 0)
+            for idx,tag in enumerate(data[r]):
+                if clear_rating:
+                    tag.avg_rating = None
+                tag.state = state_map.get(tag.name, 0)
+
+                if collapse > 0 and cat_len > collapse:
+                    if collapse_model == 'partition':
+                        if (idx % collapse) == 0:
+                            d = {'first': tag}
+                            if cat_len > idx + collapse:
+                                d['last'] = data[r][idx+collapse-1]
+                            else:
+                                d['last'] = empty_tag
+                            name = eval_formatter.safe_format(collapse_template,
+                                                              d, 'TAG_VIEW', None)
+                            sub_cat = TagTreeItem(parent=category,
+                                     data = name, tooltip = None,
+                                     category_icon = category_node.icon,
+                                     category_key=category_node.category_key)
+                    else:
+                        if upper(tag.sort[0]) != collapse_letter:
+                            collapse_letter = upper(tag.name[0])
+                            sub_cat = TagTreeItem(parent=category,
+                                     data = collapse_letter,
+                                     category_icon = category_node.icon,
+                                     tooltip = None,
+                                     category_key=category_node.category_key)
+                    t = TagTreeItem(parent=sub_cat, data=tag,
+                                        icon_map=self.icon_state_map)
+                else:
                     t = TagTreeItem(parent=category, data=tag, icon_map=self.icon_state_map)
-                self.endInsertRows()
+            self.endInsertRows()
         return True
 
     def columnCount(self, parent):
@@ -824,19 +889,27 @@ class TagsModel(QAbstractItemModel): # {{{
 
     def reset_all_states(self, except_=None):
         update_list = []
-        for i in xrange(self.rowCount(QModelIndex())):
+        def process_tag(tag_index, tag_item):
-            category_index = self.index(i, 0, QModelIndex())
+            tag = tag_item.tag
+            if tag is except_:
+                self.dataChanged.emit(tag_index, tag_index)
+                return
+            if tag.state != 0 or tag in update_list:
+                tag.state = 0
+                update_list.append(tag)
+                self.dataChanged.emit(tag_index, tag_index)
+
+        def process_level(category_index):
             for j in xrange(self.rowCount(category_index)):
                 tag_index = self.index(j, 0, category_index)
                 tag_item = tag_index.internalPointer()
-                tag = tag_item.tag
+                if tag_item.type == TagTreeItem.CATEGORY:
-                if tag is except_:
+                    process_level(tag_index)
-                    self.dataChanged.emit(tag_index, tag_index)
+                else:
-                    continue
+                    process_tag(tag_index, tag_item)
-                if tag.state != 0 or tag in update_list:
+
-                    tag.state = 0
+        for i in xrange(self.rowCount(QModelIndex())):
-                    update_list.append(tag)
+            process_level(self.index(i, 0, QModelIndex()))
-                    self.dataChanged.emit(tag_index, tag_index)
 
     def clear_state(self):
         self.reset_all_states()
@@ -856,6 +929,7 @@ class TagsModel(QAbstractItemModel): # {{{
         ans = []
         tags_seen = set()
         row_index = -1
+
         for i, key in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
@@ -863,7 +937,7 @@ class TagsModel(QAbstractItemModel): # {{{
             if key.endswith(':'): # User category, so skip it. The tag will be marked in its real category
                 continue
             category_item = self.root_item.children[row_index]
-            for tag_item in category_item.children:
+            for tag_item in category_item.child_tags():
                 tag = tag_item.tag
                 if tag.state > 0:
                     prefix = ' not ' if tag.state == 2 else ''
@@ -878,6 +952,82 @@ class TagsModel(QAbstractItemModel): # {{{
                         ans.append('%s%s:"=%s"'%(prefix, category, tag.name))
         return ans
 
+    def find_node(self, key, txt, start_index):
+        if not txt:
+            return None
+        txt = lower(txt)
+        if start_index is None or not start_index.isValid():
+            start_index = QModelIndex()
+        self.node_found = None
+
+        def process_tag(depth, tag_index, tag_item, start_path):
+            path = self.path_for_index(tag_index)
+            if depth < len(start_path) and path[depth] <= start_path[depth]:
+                return False
+            tag = tag_item.tag
+            if tag is None:
+                return False
+            if lower(tag.name).find(txt) >= 0:
+                self.node_found = tag_index
+                return True
+            return False
+
+        def process_level(depth, category_index, start_path):
+            path = self.path_for_index(category_index)
+            if depth < len(start_path):
+                if path[depth] < start_path[depth]:
+                    return False
+                if path[depth] > start_path[depth]:
+                    start_path = path
+            if key and category_index.internalPointer().category_key != key:
+                return False
+            for j in xrange(self.rowCount(category_index)):
+                tag_index = self.index(j, 0, category_index)
+                tag_item = tag_index.internalPointer()
+                if tag_item.type == TagTreeItem.CATEGORY:
+                    if process_level(depth+1, tag_index, start_path):
+                        return True
+                else:
+                    if process_tag(depth+1, tag_index, tag_item, start_path):
+                        return True
+            return False
+
+        for i in xrange(self.rowCount(QModelIndex())):
+            if process_level(0, self.index(i, 0, QModelIndex()),
+                                  self.path_for_index(start_index)):
+                break
+        return self.node_found
+
+    def show_item_at_index(self, idx, box=False):
+        if idx.isValid():
+            tag_item = idx.internalPointer()
+            self.tags_view.setCurrentIndex(idx)
+            self.tags_view.scrollTo(idx, QTreeView.PositionAtCenter)
+            if box:
+                tag_item.boxed = True
+                self.dataChanged.emit(idx, idx)
+
+    def clear_boxed(self):
+        def process_tag(tag_index, tag_item):
+            if tag_item.boxed:
+                tag_item.boxed = False
+                self.dataChanged.emit(tag_index, tag_index)
+
+        def process_level(category_index):
+            for j in xrange(self.rowCount(category_index)):
+                tag_index = self.index(j, 0, category_index)
+                tag_item = tag_index.internalPointer()
+                if tag_item.type == TagTreeItem.CATEGORY:
+                    process_level(tag_index)
+                else:
+                    process_tag(tag_index, tag_item)
+
+        for i in xrange(self.rowCount(QModelIndex())):
+            process_level(self.index(i, 0, QModelIndex()))
+
+    def get_filter_categories_by(self):
+        return self.filter_categories_by
+
     # }}}
 
 class TagBrowserMixin(object): # {{{
@@ -993,10 +1143,40 @@ class TagBrowserWidget(QWidget): # {{{
 
     def __init__(self, parent):
         QWidget.__init__(self, parent)
+        self.parent = parent
         self._layout = QVBoxLayout()
         self.setLayout(self._layout)
         self._layout.setContentsMargins(0,0,0,0)
 
+        search_layout = QHBoxLayout()
+        self._layout.addLayout(search_layout)
+        self.item_search = HistoryLineEdit(parent)
+        try:
+            self.item_search.lineEdit().setPlaceholderText(_('Find item in tag browser'))
+        except:
+            # Using Qt < 4.7
+            pass
+        self.item_search.setToolTip(_(
+        'Search for items. This is a "contains" search; items containing the\n'
+        'text anywhere in the name will be found. You can limit the search\n'
+        'to particular categories using syntax similar to search. For example,\n'
+        'tags:foo will find foo in any tag, but not in authors etc. Entering\n'
+        '*foo will filter all categories at once, showing only those items\n'
+        'containing the text "foo"'))
+        search_layout.addWidget(self.item_search)
+        self.search_button = QPushButton()
+        self.search_button.setText(_('&Find'))
+        self.search_button.setToolTip(_('Find the first/next matching item'))
+        self.search_button.setFixedWidth(40)
+        search_layout.addWidget(self.search_button)
+        self.current_position = None
+        self.search_button.clicked.connect(self.find)
+        self.item_search.initialize('tag_browser_search')
+        self.item_search.lineEdit().returnPressed.connect(self.do_find)
+        self.item_search.lineEdit().textEdited.connect(self.find_text_changed)
+        self.item_search.activated[QString].connect(self.do_find)
+        self.item_search.completer().setCaseSensitivity(Qt.CaseSensitive)
+
         parent.tags_view = TagsView(parent)
         self.tags_view = parent.tags_view
         self._layout.addWidget(parent.tags_view)
@@ -1031,6 +1211,59 @@ class TagBrowserWidget(QWidget): # {{{
     def set_pane_is_visible(self, to_what):
         self.tags_view.set_pane_is_visible(to_what)
 
+    def find_text_changed(self, str):
+        self.current_position = None
+
+    def do_find(self, str=None):
+        self.current_position = None
+        self.find()
+
+    def find(self):
+        model = self.tags_view.model()
+        model.clear_boxed()
+        txt = unicode(self.item_search.currentText()).strip()
+
+        if txt.startswith('*'):
+            self.tags_view.set_new_model(filter_categories_by=txt[1:])
+            self.current_position = None
+            return
+        if model.get_filter_categories_by():
+            self.tags_view.set_new_model(filter_categories_by=None)
+            self.current_position = None
+            model = self.tags_view.model()
+
+        if not txt:
+            return
+
+        self.item_search.blockSignals(True)
+        self.item_search.lineEdit().blockSignals(True)
+        self.search_button.setFocus(True)
+        idx = self.item_search.findText(txt, Qt.MatchFixedString)
+        if idx < 0:
+            self.item_search.insertItem(0, txt)
+        else:
+            t = self.item_search.itemText(idx)
+            self.item_search.removeItem(idx)
+            self.item_search.insertItem(0, t)
+        self.item_search.setCurrentIndex(0)
+        self.item_search.blockSignals(False)
+        self.item_search.lineEdit().blockSignals(False)
+
+        colon = txt.find(':')
+        key = None
+        if colon > 0:
+            key = self.parent.library_view.model().db.\
+                        field_metadata.search_term_to_field_key(txt[:colon])
+            txt = txt[colon+1:]
+
+        self.current_position = model.find_node(key, txt, self.current_position)
+        if self.current_position:
+            model.show_item_at_index(self.current_position, box=True)
+        elif self.item_search.text():
+            warning_dialog(self.tags_view, _('No item found'),
+                        _('No (more) matches for that search')).exec_()
+
+
 
 # }}}
 

src/calibre/gui2/viewer/documentview.py

@@ -769,7 +769,7 @@ class DocumentView(QWebView): # {{{
                 self.to_bottom = True
                 if epf:
                     self.flipper.initialize(self.current_page_image(), False)
-                    self.manager.previous_document()
+                self.manager.previous_document()
         else:
             opos = self.document.ypos
             upper_limit = opos - delta_y
@@ -783,8 +783,8 @@ class DocumentView(QWebView): # {{{
                 if epf:
                     self.flipper(self.current_page_image(),
                             duration=self.document.page_flip_duration)
-            if self.manager is not None:
+                if self.manager is not None:
-                self.manager.scrolled(self.scroll_fraction)
+                    self.manager.scrolled(self.scroll_fraction)
 
     def next_page(self):
         if self.flipper.running and not self.is_auto_repeat_event:

src/calibre/library/caches.py

@@ -669,6 +669,9 @@ class ResultCache(SearchQueryParser): # {{{
             fields = [('timestamp', False)]
 
         keyg = SortKeyGenerator(fields, self.field_metadata, self._data)
+        # For efficiency, the key generator returns a plain value if only one
+        # field is in the sort field list. Because the normal cmp function will
+        # always assume asc, we must deal with asc/desc here.
         if len(fields) == 1:
             self._map.sort(key=keyg, reverse=not fields[0][1])
         else:
@@ -697,7 +700,7 @@ class SortKeyGenerator(object):
     def __init__(self, fields, field_metadata, data):
         from calibre.utils.icu import sort_key
         self.field_metadata = field_metadata
-        self.orders = [-1 if x[1] else 1 for x in fields]
+        self.orders = [1 if x[1] else -1 for x in fields]
         self.entries = [(x[0], field_metadata[x[0]]) for x in fields]
         self.library_order = tweaks['title_series_sorting'] == 'library_order'
         self.data = data

src/calibre/library/database2.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 '''
 The database used to store ebook metadata
 '''
-import os, sys, shutil, cStringIO, glob, time, functools, traceback, re
+import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, json
 from itertools import repeat
 from math import ceil
 from Queue import Queue
@@ -32,7 +32,7 @@ from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
-from calibre.utils.config import prefs, tweaks
+from calibre.utils.config import prefs, tweaks, from_json, to_json
 from calibre.utils.icu import sort_key
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
@@ -2700,6 +2700,34 @@ books_series_link      feeds
 
         return duplicates
 
+    def add_custom_book_data(self, book_id, name, val):
+        x = self.conn.get('SELECT id FROM books WHERE ID=?', (book_id,), all=False)
+        if x is None:
+            raise ValueError('add_custom_book_data: no such book_id %d'%book_id)
+        # Do the json encode first, in case it throws an exception
+        s = json.dumps(val, default=to_json)
+        self.conn.execute('DELETE FROM books_plugin_data WHERE book=? AND name=?',
+                          (book_id, name))
+        self.conn.execute('''INSERT INTO books_plugin_data(book, name, val)
+                             VALUES(?, ?, ?)''', (book_id, name, s))
+        self.commit()
+
+    def get_custom_book_data(self, book_id, name, default=None):
+        try:
+            s = self.conn.get('''select val FROM books_plugin_data
+                    WHERE book=? AND name=?''', (book_id, name), all=False)
+            if s is None:
+                return default
+            return json.loads(s, object_hook=from_json)
+        except:
+            pass
+        return default
+
+    def delete_custom_book_data(self, book_id, name):
+        self.conn.execute('DELETE FROM books_plugin_data WHERE book=? AND name=?',
+                          (book_id, name))
+        self.commit()
+
     def get_custom_recipes(self):
         for id, title, script in self.conn.get('SELECT id,title,script FROM feeds'):
             yield id, title, script

src/calibre/library/schema_upgrades.py

@@ -441,3 +441,31 @@ class SchemaUpgrade(object):
                          WHERE id=NEW.id AND OLD.title <> NEW.title;
             END;
         ''')
+
+    def upgrade_version_17(self):
+        'custom book data table (for plugins)'
+        script = '''
+        DROP TABLE IF EXISTS books_plugin_data;
+        CREATE TABLE books_plugin_data(id INTEGER PRIMARY KEY,
+                                     book INTEGER NON NULL,
+                                     name TEXT NON NULL,
+                                     val TEXT NON NULL,
+                                     UNIQUE(book,name));
+        DROP TRIGGER IF EXISTS books_delete_trg;
+        CREATE TRIGGER books_delete_trg
+            AFTER DELETE ON books
+            BEGIN
+                DELETE FROM books_authors_link WHERE book=OLD.id;
+                DELETE FROM books_publishers_link WHERE book=OLD.id;
+                DELETE FROM books_ratings_link WHERE book=OLD.id;
+                DELETE FROM books_series_link WHERE book=OLD.id;
+                DELETE FROM books_tags_link WHERE book=OLD.id;
+                DELETE FROM data WHERE book=OLD.id;
+                DELETE FROM comments WHERE book=OLD.id;
+                DELETE FROM conversion_options WHERE book=OLD.id;
+                DELETE FROM books_plugin_data WHERE book=OLD.id;
+        END;
+        '''
+        self.conn.executescript(script)
+
+

src/calibre/library/server/browse.py

@@ -556,18 +556,19 @@ class BrowseServer(object):
                 ids = self.search_cache('search:"%s"'%which)
             except:
                 raise cherrypy.HTTPError(404, 'Search: %r not understood'%which)
-        all_ids = self.search_cache('')
-        if category == 'newest':
-            ids = all_ids
-            hide_sort = 'true'
-        elif category == 'allbooks':
-            ids = all_ids
         else:
-            q = category
+            all_ids = self.search_cache('')
-            if q == 'news':
+            if category == 'newest':
-                q = 'tags'
+                ids = all_ids
-            ids = self.db.get_books_for_category(q, cid)
+                hide_sort = 'true'
-            ids = [x for x in ids if x in all_ids]
+            elif category == 'allbooks':
+                ids = all_ids
+            else:
+                q = category
+                if q == 'news':
+                    q = 'tags'
+                ids = self.db.get_books_for_category(q, cid)
+                ids = [x for x in ids if x in all_ids]
 
         items = [self.db.data._data[x] for x in ids]
         if category == 'newest':

src/calibre/library/server/opds.py

@@ -173,6 +173,8 @@ def ACQUISITION_ENTRY(item, version, db, updated, CFM, CKEYS, prefix):
                 extra.append('%s: %s<br />'%(xml(name), xml(format_tag_string(val, ',',
                                                            ignore_max=True,
                                                            no_tag_count=True))))
+            elif datatype == 'comments':
+                extra.append('%s: %s<br />'%(xml(name), comments_to_html(unicode(val))))
             else:
                 extra.append('%s: %s<br />'%(xml(name), xml(unicode(val))))
     comments = item[FM['comments']]

src/calibre/manual/customize.rst

@@ -21,6 +21,7 @@ Environment variables
 -----------------------
 
     * ``CALIBRE_CONFIG_DIRECTORY`` - sets the directory where configuration files are stored/read.
+    * ``CALIBRE_TEMP_DIR`` - sets the temporary directory used by calibre
     * ``CALIBRE_OVERRIDE_DATABASE_PATH`` - allows you to specify the full path to metadata.db. Using this variable you can have metadata.db be in a location other than the library folder. Useful if your library folder is on a networked drive that does not support file locking.
     * ``CALIBRE_DEVELOP_FROM`` - Used to run from a calibre development environment. See :ref:`develop`.
     * ``CALIBRE_OVERRIDE_LANG`` - Used to force the language used by the interface (ISO 639 language code)

src/calibre/manual/template_lang.rst

@@ -137,8 +137,8 @@ Note that you can use the prefix and suffix as well. If you want the number to a
     {#myint:0>3s:ifempty(0)|[|]}
     
 
-Using functions in templates - program mode
+Using functions in templates - template program mode
--------------------------------------------
+----------------------------------------------------
 
 The template language program mode differs from single-function mode in that it permits you to write template expressions that refer to other metadata fields, modify values, and do arithmetic. It is a reasonably complete programming language.
 
@@ -161,10 +161,13 @@ The syntax of the language is shown by the following grammar::
     constant   ::= " string " | ' string ' | number
     identifier ::= sequence of letters or ``_`` characters
     function   ::= identifier ( statement [ , statement ]* )
-    expression ::= identifier | constant | function
+    expression ::= identifier | constant | function | assignment
+    assignment ::= identifier '=' expression
     statement  ::= expression [ ; expression ]*
     program    ::= statement
 
+Comments are lines with a '#' character at the beginning of the line.
+
 An ``expression`` always has a value, either the value of the constant, the value contained in the identifier, or the value returned by a function. The value of a ``statement`` is the value of the last expression in the sequence of statements. As such, the value of the program (statement)::
 
     1; 2; 'foobar'; 3
@@ -208,13 +211,102 @@ The following functions are available in addition to those described in single-f
     * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers.
     * ``field(name)`` -- returns the metadata field named by ``name``.
+    * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables.
     * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers.
+    * ``print(a, b, ...)`` -- prints the arguments to standard output. Unless you start calibre from the command line (``calibre-debug -g``), the output will go to a black hole.
     * ``strcat(a, b, ...)`` -- can take any number of arguments. Returns a string formed by concatenating all the arguments.
     * ``strcmp(x, y, lt, eq, gt)`` -- does a case-insensitive comparison x and y as strings. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
     * ``substr(str, start, end)`` -- returns the ``start``'th through the ``end``'th characters of ``str``. The first character in ``str`` is the zero'th character. If end is negative, then it indicates that many characters counting from the right. If end is zero, then it indicates the last character. For example, ``substr('12345', 1, 0)`` returns ``'2345'``, and ``substr('12345', 1, -1)`` returns ``'234'``.
     * ``subtract(x, y)`` -- returns x - y. Throws an exception if either x or y are not numbers.
     * ``template(x)`` -- evaluates x as a template. The evaluation is done in its own context, meaning that variables are not shared between the caller and the template evaluation. Because the `{` and `}` characters are special, you must use `[[` for the `{` character and `]]` for the '}' character; they are converted automatically. For example, ``template('[[title_sort]]') will evaluate the template ``{title_sort}`` and return its value.
-    
+
+Using general program mode
+-----------------------------------
+
+For more complicated template programs, it is sometimes easier to avoid template syntax (all the `{` and `}` characters), instead writing a more classical-looking program. You can do this in |app| by beginning the template with `program:`. In this case, no template processing is done. The special variable `$` is not set. It is up to your program to produce the correct results.
+
+One advantage of `program:` mode is that the brackets are no longer special. For example, it is not necessary to use `[[` and `]]` when using the `template()` function.
+
+The following example is a `program:` mode implementation of a recipe on the MobileRead forum: "Put series into the title, using either initials or a shortened form. Strip leading articles from the series name (any)." For example, for the book The Two Towers in the Lord of the Rings series, the recipe gives `LotR [02] The Two Towers`. Using standard templates, the recipe requires three custom columns and a plugboard, as explained in the following:
+
+The solution requires creating three composite columns. The first column is used to remove the leading articles. The second is used to compute the 'shorten' form. The third is to compute the 'initials' form. Once you have these columns, the plugboard selects between them. You can hide any or all of the three columns on the library view.
+
+    First column:
+    Name: #stripped_series. 
+    Template: {series:re(^(A|The|An)\s+,)||}
+
+    Second column (the shortened form):
+    Name: #shortened. 
+    Template: {#stripped_series:shorten(4,-,4)}
+
+    Third column (the initials form):
+    Name: #initials. 
+    Template: {#stripped_series:re(([^\s])[^\s]+(\s|$),\1)}
+
+    Plugboard expression:
+    Template:{#stripped_series:lookup(.\s,#initials,.,#shortened,series)}{series_index:0>2.0f| [|] }{title}
+    Destination field: title
+
+    This set of fields and plugboard produces:
+    Series: The Lord of the Rings
+    Series index: 2
+    Title: The Two Towers
+    Output: LotR [02] The Two Towers
+
+    Series: Dahak
+    Series index: 1
+    Title: Mutineers Moon
+    Output: Dahak [01] Mutineers Moon
+
+    Series: Berserkers
+    Series Index: 4
+    Title: Berserker Throne
+    Output: Bers-kers [04] Berserker Throne
+
+    Series: Meg Langslow Mysteries
+    Series Index: 3
+    Title: Revenge of the Wrought-Iron Flamingos
+    Output: MLM [03] Revenge of the Wrought-Iron Flamingos
+
+The following program produces the same results as the original recipe, using only one custom column to hold the results of a program that computes the special title value::
+
+    Custom column: 
+    Name: #special_title
+    Template: (the following with all leading spaces removed)
+        program:
+        #	compute the equivalent of the composite fields and store them in local variables
+            stripped = re(field('series'), '^(A|The|An)\s+', '');
+            shortened = shorten(stripped, 4, '-' ,4);
+            initials = re(stripped, '[^\w]*(\w?)[^\s]+(\s|$)', '\1');
+
+        #	Format the series index. Ends up as empty if there is no series index.
+        #	Note that leading and trailing spaces will be removed by the formatter,
+        # 	so we cannot add them here. We will do that in the strcat below.
+        #	Also note that because we are in 'program' mode, we can freely use
+        #	curly brackets in strings, something we cannot do in template mode.
+            s_index = template('{series_index:0>2.0f}');
+
+        #	print(stripped, shortened, initials, s_index);
+
+        #	Now concatenate all the bits together. The switch picks between 
+        # 	initials and shortened, depending on whether there is a space
+        #	in stripped. We then add the brackets around s_index if it is
+        #	not empty. Finally, add the title. As this is the last function in
+        # 	the program, its value will be returned.
+            strcat(
+                switch(	stripped, 
+                        '.\s', initials, 
+                        '.', shortened,
+                        field('series')),
+                test(s_index, strcat(' [', s_index, '] '), ''),
+                field('title'));
+
+    Plugboard expression:
+    Template:{#special_title}
+    Destination field: title
+
+It would be possible to do the above with no custom columns by putting the program into the template box of the plugboard. However, to do so, all comments must be removed because the plugboard text box does not support multi-line editing. It is debatable whether the gain of not having the custom column is worth the vast increase in difficulty caused by the program being one giant line.
+
 Special notes for save/send templates
 -------------------------------------
 
@@ -257,4 +349,4 @@ You might find the following tips useful.
     * Templates can use other templates by referencing a composite custom column.
     * In a plugboard, you can set a field to empty (or whatever is equivalent to empty) by using the special template ``{null}``. This template will always evaluate to an empty string.
     * The technique described above to show numbers even if they have a zero value works with the standard field series_index.
-    
+    

src/calibre/ptempfile.py

@@ -40,7 +40,7 @@ def base_dir():
             _base_dir = td
         else:
             _base_dir = tempfile.mkdtemp(prefix='%s_%s_tmp_'%(__appname__,
-                __version__))
+                __version__), dir=os.environ.get('CALIBRE_TEMP_DIR', None))
             atexit.register(remove_dir, _base_dir)
     return _base_dir
 

src/calibre/utils/formatter.py

@@ -36,7 +36,7 @@ class _Parser(object):
         return gt
 
     def _assign(self, target, value):
-        setattr(self, target, value)
+        self.variables[target] = value
         return value
 
     def _concat(self, *args):
@@ -55,20 +55,30 @@ class _Parser(object):
             }
         x = float(x if x else 0)
         y = float(y if y else 0)
-        return ops[op](x, y)
+        return unicode(ops[op](x, y))
 
     def _template(self, template):
         template = template.replace('[[', '{').replace(']]', '}')
         return self.parent.safe_format(template, self.parent.kwargs, 'TEMPLATE',
                                        self.parent.book)
 
+    def _eval(self, template):
+        template = template.replace('[[', '{').replace(']]', '}')
+        return eval_formatter.safe_format(template, self.variables, 'EVAL', None)
+
+    def _print(self, *args):
+        print args
+        return None
+
     local_functions = {
             'add'      : (2, partial(_math, op='+')),
             'assign'   : (2, _assign),
             'cmp'      : (5, _cmp),
             'divide'   : (2, partial(_math, op='/')),
+            'eval'     : (1, _eval),
             'field'    : (1, lambda s, x: s.parent.get_value(x, [], s.parent.kwargs)),
             'multiply' : (2, partial(_math, op='*')),
+            'print'    : (-1, _print),
             'strcat'   : (-1, _concat),
             'strcmp'   : (5, _strcmp),
             'substr'   : (3, lambda s, x, y, z: x[int(y): len(x) if int(z) == 0 else int(z)]),
@@ -82,7 +92,7 @@ class _Parser(object):
         if prog[1] != '':
             self.error(_('failed to scan program. Invalid input {0}').format(prog[1]))
         self.parent = parent
-        setattr(self, '$', val)
+        self.variables = {'$':val}
 
     def error(self, message):
         m = 'Formatter: ' + message + _(' near ')
@@ -138,13 +148,19 @@ class _Parser(object):
             if not self.token_op_is_a(';'):
                 return val
             self.consume()
+            if self.token_is_eof():
+                return val
 
     def expr(self):
         if self.token_is_id():
             # We have an identifier. Determine if it is a function
             id = self.token()
             if not self.token_op_is_a('('):
-                return getattr(self, id, _('unknown id ') + id)
+                if self.token_op_is_a('='):
+                    # classic assignment statement
+                    self.consume()
+                    return self._assign(id, self.expr())
+                return self.variables.get(id, _('unknown id ') + id)
             # We have a function.
             # Check if it is a known one. We do this here so error reporting is
             # better, as it can identify the tokens near the problem.
@@ -334,6 +350,7 @@ class TemplateFormatter(string.Formatter):
                 (r'\w+',                lambda x,t: (2, t)),
                 (r'".*?((?<!\\)")',     lambda x,t: (3, t[1:-1])),
                 (r'\'.*?((?<!\\)\')',   lambda x,t: (3, t[1:-1])),
+                (r'\n#.*?(?=\n)',       None),
                 (r'\s',                 None)
         ])
 
@@ -418,14 +435,17 @@ class TemplateFormatter(string.Formatter):
         self.book = book
         self.composite_values = {}
         try:
-            ans = self.vformat(fmt, [], kwargs).strip()
+            if fmt.startswith('program:'):
+                ans = self._eval_program(None, fmt[8:])
+            else:
+                ans = self.vformat(fmt, [], kwargs).strip()
         except Exception, e:
             if DEBUG:
                 traceback.print_exc()
             ans = error_value + ' ' + e.message
         return ans
 
-class ValidateFormat(TemplateFormatter):
+class ValidateFormatter(TemplateFormatter):
     '''
     Provides a format function that substitutes '' for any missing value
     '''
@@ -435,6 +455,14 @@ class ValidateFormat(TemplateFormatter):
     def validate(self, x):
         return self.vformat(x, [], {})
 
-validation_formatter = ValidateFormat()
+validation_formatter = ValidateFormatter()
 
+class EvalFormatter(TemplateFormatter):
+    '''
+    A template formatter that uses a simple dict instead of an mi instance
+    '''
+    def get_value(self, key, args, kwargs):
+        return kwargs.get(key, _('No such variable ') + key)
+
+eval_formatter = EvalFormatter()