From 1df789da2b15516dc31305f7e1fb3cffbb33cdc0 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 25 Nov 2011 07:44:13 +0530
Subject: [PATCH 01/21] Fix #894540 (Updated recipe for El Mundo)

---
 recipes/elmundo.recipe | 74 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 65 insertions(+), 9 deletions(-)

diff --git a/recipes/elmundo.recipe b/recipes/elmundo.recipe
index 76ce785ba3..4f04f68575 100644
--- a/recipes/elmundo.recipe
+++ b/recipes/elmundo.recipe
@@ -4,7 +4,8 @@ __copyright__ = '2009-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 elmundo.es
 '''
-
+import re
+import time
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class ElMundo(BasicNewsRecipe):
@@ -18,12 +19,15 @@ class ElMundo(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'iso8859_15'
+    remove_javascript     = True
+    remove_empty_feeds    = True
     language              = 'es'
     masthead_url          = 'http://estaticos03.elmundo.es/elmundo/iconos/v4.x/v4.01/bg_h1.png'
     publication_type      = 'newspaper'
     extra_css             = """
                                body{font-family: Arial,Helvetica,sans-serif}
                                .metadata_noticia{font-size: small}
+                               .pestana_GDP{font-size: small; font-weight:bold}
                                h1,h2,h3,h4,h5,h6,.subtitulo {color: #3F5974}
                                .hora{color: red}
                                .update{color: gray}
@@ -41,22 +45,43 @@ class ElMundo(BasicNewsRecipe):
     remove_tags_after  = dict(name='div' , attrs={'id':['desarrollo_noticia','tamano']})
     remove_attributes  = ['lang','border']
     remove_tags = [
-                     dict(name='div', attrs={'class':['herramientas','publicidad_google']})
-                    ,dict(name='div', attrs={'id':'modulo_multimedia' })
+                     dict(name='div', attrs={'class':['herramientas','publicidad_google','comenta','col col-2b','apoyos','no-te-pierdas']})
+                    ,dict(name='div', attrs={'class':['publicidad publicidad_cuerpo_noticia','comentarios_nav','mensaje_privado','interact']})
+                    ,dict(name='div', attrs={'class':['num_comentarios estirar']})
+                    ,dict(name='span', attrs={'class':['links_comentar']})
+                    ,dict(name='div', attrs={'id':['comentar']})
                     ,dict(name='ul', attrs={'class':'herramientas' })
                     ,dict(name=['object','link','embed','iframe','base','meta'])
                   ]
 
     feeds = [
-              (u'Portada'         , u'http://estaticos.elmundo.es/elmundo/rss/portada.xml'       )
+              (u'Portada'         , u'http://estaticos.elmundo.es/elmundo/rss/portada.xml'       )                                      
              ,(u'Deportes'        , u'http://estaticos.elmundo.es/elmundodeporte/rss/portada.xml')
-             ,(u'Economia'        , u'http://estaticos.elmundo.es/elmundo/rss/economia.xml'      )
-             ,(u'Espana'          , u'http://estaticos.elmundo.es/elmundo/rss/espana.xml'        )
+             ,(u'Econom\xeda'     , u'http://estaticos.elmundo.es/elmundo/rss/economia.xml'      )
+             ,(u'Espa\xf1a'       , u'http://estaticos.elmundo.es/elmundo/rss/espana.xml'        )
              ,(u'Internacional'   , u'http://estaticos.elmundo.es/elmundo/rss/internacional.xml' )
              ,(u'Cultura'         , u'http://estaticos.elmundo.es/elmundo/rss/cultura.xml'       )
-             ,(u'Ciencia/Ecologia', u'http://estaticos.elmundo.es/elmundo/rss/ciencia.xml'       )
-             ,(u'Comunicacion'    , u'http://estaticos.elmundo.es/elmundo/rss/comunicacion.xml'  )
-             ,(u'Television'      , u'http://estaticos.elmundo.es/elmundo/rss/television.xml'    )
+             ,(u'Ciencia/Ecolog\xeda', u'http://estaticos.elmundo.es/elmundo/rss/ciencia.xml'    )
+             ,(u'Comunicaci\xf3n' , u'http://estaticos.elmundo.es/elmundo/rss/comunicacion.xml'  )
+             ,(u'Televisi\xf3n'   , u'http://estaticos.elmundo.es/elmundo/rss/television.xml'    )
+
+             ,(u'Salud'           , u'http://estaticos.elmundo.es/elmundosalud/rss/portada.xml'  )
+             ,(u'Solidaridad'     , u'http://estaticos.elmundo.es/elmundo/rss/solidaridad.xml'   )
+             ,(u'Su vivienda'     , u'http://estaticos.elmundo.es/elmundo/rss/suvivienda.xml'    )             
+             ,(u'Motor'           , u'http://estaticos.elmundo.es/elmundomotor/rss/portada.xml'  )             
+             
+             ,(u'Madrid'          , u'http://estaticos.elmundo.es/elmundo/rss/madrid.xml'        )
+             ,(u'Barcelona'       , u'http://estaticos.elmundo.es/elmundo/rss/barcelona.xml'     )
+             ,(u'Pa\xeds Vasco'   , u'http://estaticos.elmundo.es/elmundo/rss/paisvasco.xml'     )	     
+             ,(u'Baleares'        , u'http://estaticos.elmundo.es/elmundo/rss/baleares.xml'      )
+	     ,(u'Castilla y Le\xf3n' , u'http://estaticos.elmundo.es/elmundo/rss/castillayleon.xml' )	     
+	     ,(u'Valladolid'      , u'http://estaticos.elmundo.es/elmundo/rss/valladolid.xml'    )
+	     ,(u'Valencia'        , u'http://estaticos.elmundo.es/elmundo/rss/valencia.xml'      )
+	     ,(u'Alicante'        , u'http://estaticos.elmundo.es/elmundo/rss/alicante.xml'      )
+	     ,(u'Castell\xf3n'    , u'http://estaticos.elmundo.es/elmundo/rss/castellon.xml'     )	
+	     ,(u'Andaluc\xeda'    , u'http://estaticos.elmundo.es/elmundo/rss/andalucia.xml'     )
+	     ,(u'Sevilla'         , u'http://estaticos.elmundo.es/elmundo/rss/andalucia_sevilla.xml'  )
+	     ,(u'M\xe1laga'       , u'http://estaticos.elmundo.es/elmundo/rss/andalucia_malaga.xml'   )
             ]
 
     def preprocess_html(self, soup):
@@ -67,3 +92,34 @@ class ElMundo(BasicNewsRecipe):
     def get_article_url(self, article):
         return article.get('guid',  None)
 
+
+    preprocess_regexps = [     
+                           # Para presentar la imagen de los videos incrustados                           
+
+                           (re.compile(r'var imagen', re.DOTALL|re.IGNORECASE), lambda match: '--></script><img src'),
+                           (re.compile(r'.jpg";', re.DOTALL|re.IGNORECASE), lambda match: '.jpg">'),
+                           (re.compile(r'var video=', re.DOTALL|re.IGNORECASE), lambda match: '<script language="Javascript" type="text/javascript"><!--'),
+
+                           # Para que no salga la numeración de comentarios: 1, 2, 3 ...
+
+                           (re.compile(r'<ol>\n<li style="z-index:', re.DOTALL|re.IGNORECASE), lambda match: '<ul><li style="z-index:'),
+                           (re.compile(r'</ol>\n<div class="num_comentarios estirar">', re.DOTALL|re.IGNORECASE), lambda match: '</ul><div class="num_comentarios estirar">'),
+                         ]
+
+    # Obtener la imagen de portada
+
+    def get_cover_url(self):
+       cover = None
+       st = time.localtime()
+       year = str(st.tm_year)
+       month = "%.2d" % st.tm_mon
+       day = "%.2d" % st.tm_mday
+		#http://img.kiosko.net/2011/11/19/es/elmundo.750.jpg
+       cover='http://img.kiosko.net/'+ year + '/' +  month + '/' + day +'/es/elmundo.750.jpg'
+       br = BasicNewsRecipe.get_browser()
+       try:
+           br.open(cover)
+       except:
+           self.log("\nPortada no disponible")
+           cover ='http://estaticos03.elmundo.es/elmundo/iconos/v4.x/v4.01/bg_h1.png'
+       return cover 

From fe0744431f20b82fad79db62849d92fb9d1e2353 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 25 Nov 2011 08:59:27 +0530
Subject: [PATCH 02/21] version 0.8.28

---
 Changelog.yaml           | 57 ++++++++++++++++++++++++++++++++++++++++
 src/calibre/constants.py |  2 +-
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/Changelog.yaml b/Changelog.yaml
index 478e6a311b..21b92493a7 100644
--- a/Changelog.yaml
+++ b/Changelog.yaml
@@ -19,6 +19,63 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.28
+  date: 2011-11-25
+
+  new features:
+    - title: "Get Books: Add litres.ru store"
+
+    - title: "Change the algorithm that generates title sort strings to strip leading articles from both english and the current language set for the calibre user interface. In addition, in the edit metadata dialog, calibre will use the book's language when calculating the sort string. This behavior can be adjusted via Preferences->Tweaks."
+      tickets: [886763]
+
+    - title: "Driver for Cybook Odyssey."
+      tickets: [893457]
+
+    - title: "Irex driver: Put books into the top level directory instead of into /ebooks or /Books."
+      tickets: [883616]
+  
+  bug fixes:
+    - title: "Have downloaded periodicals recognized when transferred via USB to the Kindle Fire"
+
+    - title: "MOBI Output: Fix underline and strikethrough properties declared on parents not being rendered on child tags."
+      tickets: [894245]
+
+    - title: "Template language: Fix regression that broke ordering of items when formatting a list"
+
+    - title: "Conversion pipeline: When removing obsolete <font> tags convert them to <div> instead of <span> if they contain block level tags."
+      tickets: [892525] 
+
+    - title: "When downloading metadata, fix the case normalization of double-barelled author names."
+      tickets: [893257]
+
+    - title: "Template language: Fix regression that broke using general program mode in save to disk templates"
+
+    - title: "calibredb: Fix use of ranges when specifying ids for the remove command"
+
+    - title: "Apple driver: Add ids for iPhone 4S. More robust against iTunes automation errors when adding artwork."
+      tickets: [892468]
+
+    - title: "Fix encoding of comments incorrectly detected when downloading metadata from ozon.ru"
+
+    - title: "Fix calibre not getting list of books on the Kindle Fire"
+
+  improved recipes:
+    - El Mundo
+    - BBC
+    - NIN Online
+    - ABC Australia
+    - Salon.com
+    - Expansion (Spanish)
+    - The Week
+    - Heise Online
+
+  new recipes:
+    - title: Give me something to read and Let's get Critical
+      author: Barty
+
+    - title: Worldcrunch
+      author: Krittika Goyal
+
 - version: 0.8.27
   date: 2011-11-18
 
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index b4b4741293..5538be0ea6 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 27)
+numeric_version = (0, 8, 28)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

From f2ce27fae9772a1c51c39518161703c0dc850741 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 25 Nov 2011 11:37:58 +0530
Subject: [PATCH 03/21] When uploading installers, try repeatedly in case of
 failure

---
 setup/upload.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/setup/upload.py b/setup/upload.py
index 4d0fc44712..1d0c05282c 100644
--- a/setup/upload.py
+++ b/setup/upload.py
@@ -141,8 +141,17 @@ class UploadToGoogleCode(Command): # {{{
                 'dmg':'OSX','bz2':'Linux','gz':'All'}[ext]
         desc = installer_description(fname)
         start = time.time()
-        path = self.upload(os.path.abspath(fname), desc,
-                labels=[typ, op, 'Featured'])
+        for i in range(5):
+            try:
+                path = self.upload(os.path.abspath(fname), desc,
+                    labels=[typ, op, 'Featured'])
+            except:
+                import traceback
+                traceback.print_exc()
+                print ('\nUpload failed, trying again in 30 secs')
+                time.sleep(30)
+            else:
+                break
         self.info('Uploaded to:', path, 'in', int(time.time() - start),
                 'seconds')
         return path
@@ -312,9 +321,16 @@ class UploadToSourceForge(Command): # {{{
             if not os.path.exists(x): continue
             start = time.time()
             self.info('Uploading', x)
-            check_call(['rsync', '-z', '--progress', '-e', 'ssh -x', x,
-                '%s,%s@frs.sourceforge.net:%s'%(self.USERNAME, self.PROJECT,
-                    self.rdir+'/')])
+            for i in range(5):
+                try:
+                    check_call(['rsync', '-z', '--progress', '-e', 'ssh -x', x,
+                    '%s,%s@frs.sourceforge.net:%s'%(self.USERNAME, self.PROJECT,
+                        self.rdir+'/')])
+                except:
+                    print ('\nUpload failed, trying again in 30 seconds')
+                    time.sleep(30)
+                else:
+                    break
             print 'Uploaded in', int(time.time() - start), 'seconds'
             print ('\n')
 

From 5fd4bf6e036c00f808a3d76d2f8958c7d2beeabd Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 25 Nov 2011 11:44:47 +0530
Subject: [PATCH 04/21] More granularity in the publish stages

---
 setup/commands.py |  5 +++--
 setup/publish.py  | 15 ++++++++-------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/setup/commands.py b/setup/commands.py
index febc684c08..c7aa3ab307 100644
--- a/setup/commands.py
+++ b/setup/commands.py
@@ -21,7 +21,7 @@ __all__ = [
         'linux32', 'linux64', 'linux', 'linux_freeze',
         'osx32_freeze', 'osx', 'rsync', 'push',
         'win32_freeze', 'win32', 'win',
-        'stage1', 'stage2', 'stage3', 'stage4', 'publish'
+        'stage1', 'stage2', 'stage3', 'stage4', 'stage5', 'publish'
         ]
 
 
@@ -54,13 +54,14 @@ resources = Resources()
 kakasi = Kakasi()
 
 from setup.publish import Manual, TagRelease, Stage1, Stage2, \
-        Stage3, Stage4, Publish
+        Stage3, Stage4, Stage5, Publish
 manual = Manual()
 tag_release = TagRelease()
 stage1 = Stage1()
 stage2 = Stage2()
 stage3 = Stage3()
 stage4 = Stage4()
+stage5 = Stage5()
 publish = Publish()
 
 from setup.upload import UploadUserManual, UploadInstallers, UploadDemo, \
diff --git a/setup/publish.py b/setup/publish.py
index 5c6b5563e9..26769f271e 100644
--- a/setup/publish.py
+++ b/setup/publish.py
@@ -37,19 +37,20 @@ class Stage2(Command):
        if os.path.exists(build):
            shutil.rmtree(build)
 
-
-
 class Stage3(Command):
 
    description = 'Stage 3 of the publish process'
-   sub_commands = ['upload_user_manual', 'upload_demo', 'sdist',
-            'upload_to_sourceforge', 'upload_to_google_code',
-            'tag_release', 'upload_to_server',
-   ]
+   sub_commands = ['upload_user_manual', 'upload_demo', 'sdist']
 
 class Stage4(Command):
 
     description = 'Stage 4 of the publish process'
+    sub_commands = ['upload_to_sourceforge', 'upload_to_google_code']
+
+class Stage5(Command):
+
+    description = 'Stage 5 of the publish process'
+    sub_commands = ['tag_release', 'upload_to_server']
 
     def run(self, opts):
         subprocess.check_call('rm -rf build/* dist/*', shell=True)
@@ -57,7 +58,7 @@ class Stage4(Command):
 class Publish(Command):
 
     description = 'Publish a new calibre release'
-    sub_commands = ['stage1', 'stage2', 'stage3', 'stage4']
+    sub_commands = ['stage1', 'stage2', 'stage3', 'stage4', 'stage5', ]
 
 class Manual(Command):
 

From fc0a33c7eee6419b4890d85209bd8a701daf8c0f Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 25 Nov 2011 13:21:58 +0530
Subject: [PATCH 05/21] IGN:Tag release

---
 src/calibre/translations/calibre.pot | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/calibre/translations/calibre.pot b/src/calibre/translations/calibre.pot
index 512d72d64d..b1a1d4e3e5 100644
--- a/src/calibre/translations/calibre.pot
+++ b/src/calibre/translations/calibre.pot
@@ -4,9 +4,9 @@
 #
 msgid ""
 msgstr ""
-"Project-Id-Version: calibre 0.8.27\n"
-"POT-Creation-Date: 2011-11-23 13:00+IST\n"
-"PO-Revision-Date: 2011-11-23 13:00+IST\n"
+"Project-Id-Version: calibre 0.8.28\n"
+"POT-Creation-Date: 2011-11-25 09:00+IST\n"
+"PO-Revision-Date: 2011-11-25 09:00+IST\n"
 "Last-Translator: Automatically generated\n"
 "Language-Team: LANGUAGE\n"
 "MIME-Version: 1.0\n"
@@ -24,8 +24,8 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/db/cache.py:105
 #: /home/kovid/work/calibre/src/calibre/db/cache.py:108
 #: /home/kovid/work/calibre/src/calibre/db/cache.py:119
-#: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:265
 #: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:266
+#: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:267
 #: /home/kovid/work/calibre/src/calibre/devices/hanvon/driver.py:99
 #: /home/kovid/work/calibre/src/calibre/devices/hanvon/driver.py:100
 #: /home/kovid/work/calibre/src/calibre/devices/jetbook/driver.py:74
@@ -889,15 +889,15 @@ msgstr ""
 msgid "Communicate with Android phones."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:148
+#: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:149
 msgid "Comma separated list of directories to send e-books to on the device. The first one that exists will be used"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:205
+#: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:206
 msgid "Communicate with S60 phones."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:224
+#: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:225
 msgid "Communicate with WebOS tablets."
 msgstr ""
 
@@ -16284,23 +16284,23 @@ msgstr ""
 msgid "list_union(list1, list2, separator) -- return a list made by merging the items in list1 and list2, removing duplicate items using a case-insensitive compare. If items differ in case, the one in list1 is used. The items in list1 and list2 are separated by separator, as are the items in the returned list."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:936
+#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:935
 msgid "list_difference(list1, list2, separator) -- return a list made by removing from list1 any item found in list2, using a case-insensitive compare. The items in list1 and list2 are separated by separator, as are the items in the returned list."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:957
+#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:956
 msgid "list_intersection(list1, list2, separator) -- return a list made by removing from list1 any item not found in list2, using a case-insensitive compare. The items in list1 and list2 are separated by separator, as are the items in the returned list."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:978
+#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:977
 msgid "list_sort(list, direction, separator) -- return list sorted using a case-insensitive sort. If direction is zero, the list is sorted ascending, otherwise descending. The list items are separated by separator, as are the items in the returned list."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:993
+#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:992
 msgid "list_equals(list1, sep1, list2, sep2, yes_val, no_val) -- return yes_val if list1 and list2 contain the same items, otherwise return no_val. The items are determined by splitting each list using the appropriate separator character (sep1 or sep2). The order of items in the lists is not relevant. The compare is case insensitive."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:1011
+#: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:1010
 msgid "list_re(src_list, separator, search_re, opt_replace) -- Construct a list by first separating src_list into items using the separator character. For each item in the list, check if it matches search_re. If it does, then add it to the list to be returned. If opt_replace is not the empty string, then apply the replacement before adding the item to the returned list."
 msgstr ""
 

From 7316bcad840152475908cc584718436f27f2e730 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 25 Nov 2011 19:21:33 +0530
Subject: [PATCH 06/21] ...

---
 recipes/guardian.recipe | 40 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/recipes/guardian.recipe b/recipes/guardian.recipe
index f063934b3d..840e8302af 100644
--- a/recipes/guardian.recipe
+++ b/recipes/guardian.recipe
@@ -9,6 +9,7 @@ www.guardian.co.uk
 from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 from datetime import date
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
 
 class Guardian(BasicNewsRecipe):
 
@@ -16,16 +17,19 @@ class Guardian(BasicNewsRecipe):
     if date.today().weekday() == 6:
         base_url = "http://www.guardian.co.uk/theobserver"
         cover_pic = 'Observer digital edition'
+        masthead_url = 'http://static.guim.co.uk/sys-images/Guardian/Pix/site_furniture/2010/10/19/1287478087992/The-Observer-001.gif'
     else:
         base_url = "http://www.guardian.co.uk/theguardian"
         cover_pic = 'Guardian digital edition'
+        masthead_url = 'http://static.guim.co.uk/static/f76b43f9dcfd761f0ecf7099a127b603b2922118/common/images/logos/the-guardian/titlepiece.gif'
 
     __author__ = 'Seabound and Sujata Raman'
     language = 'en_GB'
 
-    oldest_article = 7
-    max_articles_per_feed = 100
-    remove_javascript = True
+    oldest_article              = 7
+    max_articles_per_feed       = 100
+    remove_javascript           = True
+    encoding                    = 'utf-8'
 
     # List of section titles to ignore
     # For example: ['Sport']
@@ -41,6 +45,16 @@ class Guardian(BasicNewsRecipe):
                         dict(name='div', attrs={'class':["guardian-tickets promo-component",]}),
                         dict(name='ul', attrs={'class':["pagination"]}),
                         dict(name='ul', attrs={'id':["content-actions"]}),
+                        # article history link
+                        dict(name='a', attrs={'class':["rollover history-link"]}),
+                        # "a version of this article ..." speil
+                        dict(name='div' , attrs = { 'class' : ['section']}),
+                        # "about this article" js dialog
+                        dict(name='div', attrs={'class':["share-top",]}),
+                        # author picture
+                        dict(name='img', attrs={'class':["contributor-pic-small"]}),
+                        # embedded videos/captions
+                        dict(name='span',attrs={'class' : ['inline embed embed-media']}),
                         #dict(name='img'),
                         ]
     use_embedded_content    = False
@@ -67,6 +81,13 @@ class Guardian(BasicNewsRecipe):
 
     def preprocess_html(self, soup):
 
+          # multiple html sections in soup, useful stuff in the first
+          html = soup.find('html')
+          soup2 = BeautifulSoup()
+          soup2.insert(0,html) 
+          
+          soup = soup2  
+          
           for item in soup.findAll(style=True):
               del item['style']
 
@@ -74,7 +95,18 @@ class Guardian(BasicNewsRecipe):
               del item['face']
           for tag in soup.findAll(name=['ul','li']):
                 tag.name = 'div'
-
+         
+         # removes number next to rating stars
+          items_to_remove = []
+          rating_container = soup.find('div', attrs = {'class': ['rating-container']})
+          if rating_container:
+            for item in rating_container:
+                if isinstance(item, Tag) and str(item.name) == 'span':
+                    items_to_remove.append(item)
+          
+          for item in items_to_remove:
+            item.extract()
+          
           return soup
 
     def find_sections(self):

From 94ed1632d5cacbd85f9442b76146d924e656a8d6 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 25 Nov 2011 19:23:41 +0530
Subject: [PATCH 07/21] ...

---
 setup/iso_639/iso_639_3.pot | 2 +-
 setup/iso_639/iso_639_3.xml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/iso_639/iso_639_3.pot b/setup/iso_639/iso_639_3.pot
index 7e5f955cea..ec33b56d1a 100644
--- a/setup/iso_639/iso_639_3.pot
+++ b/setup/iso_639/iso_639_3.pot
@@ -17949,7 +17949,7 @@ msgid "Ndoola"
 msgstr ""
 
 #. name for nds
-msgid "Saxon; Low"
+msgid "German; Low"
 msgstr ""
 
 #. name for ndt
diff --git a/setup/iso_639/iso_639_3.xml b/setup/iso_639/iso_639_3.xml
index d06630f8f0..6b94a3850b 100644
--- a/setup/iso_639/iso_639_3.xml
+++ b/setup/iso_639/iso_639_3.xml
@@ -22836,7 +22836,7 @@ Source: <http://www.sil.org/iso639-3/>
 		part2_code="nds"
 		scope="I"
 		type="L"
-		name="Saxon; Low" />
+		name="German; Low" />
 	<iso_639_3_entry
 		id="ndt"
 		scope="I"

From 00163d73929531c3ae15a07710c024464ad742c5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 25 Nov 2011 19:46:29 +0530
Subject: [PATCH 08/21] Fix Buffalo News

---
 recipes/buffalo_news.recipe | 66 ++++++++++++++++---------------------
 1 file changed, 28 insertions(+), 38 deletions(-)

diff --git a/recipes/buffalo_news.recipe b/recipes/buffalo_news.recipe
index 51985a3c51..ae84e4433e 100644
--- a/recipes/buffalo_news.recipe
+++ b/recipes/buffalo_news.recipe
@@ -10,49 +10,39 @@ http://www.buffalonews.com/RSS/
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class AdvancedUserRecipe1298680852(BasicNewsRecipe):
+class BuffaloNews(BasicNewsRecipe):
     title          = u'Buffalo News'
     oldest_article = 2
     language = 'en'
-    __author__ = 'ChappyOnIce'
+    __author__ = 'ChappyOnIce, Krittika Goyal'
     max_articles_per_feed = 20
     encoding = 'utf-8'
     masthead_url = 'http://www.buffalonews.com/buffalonews/skins/buffalonews/images/masthead/the_buffalo_news_logo.png'
-    remove_javascript = True
-    extra_css = 'body {text-align: justify;}\n  \
-       p {text-indent: 20px;}'
+    auto_cleanup = True
+    remove_empty_feeds = True
 
-    keep_only_tags    = [
-                       dict(name='div', attrs={'class':['main-content-left']})
-                        ]
-
-    remove_tags = [
-                       dict(name='div', attrs={'id':['commentCount']}),
-       dict(name='div', attrs={'class':['story-list-links']})
-                        ]
-
-    remove_tags_after  = dict(name='div', attrs={'class':['body storyContent']})
-
-    feeds          = [(u'City of Buffalo', u'http://www.buffalonews.com/city/communities/buffalo/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Southern Erie County', u'http://www.buffalonews.com/city/communities/southern-erie/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Eastern Erie County', u'http://www.buffalonews.com/city/communities/eastern-erie/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Southern Tier', u'http://www.buffalonews.com/city/communities/southern-tier/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Niagara County', u'http://www.buffalonews.com/city/communities/niagara-county/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Business', u'http://www.buffalonews.com/business/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'MoneySmart', u'http://www.buffalonews.com/business/moneysmart/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Bills & NFL', u'http://www.buffalonews.com/sports/bills-nfl/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Sabres & NHL', u'http://www.buffalonews.com/sports/sabres-nhl/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Bob DiCesare', u'http://www.buffalonews.com/sports/columns/bob-dicesare/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Bucky Gleason', u'http://www.buffalonews.com/sports/columns/bucky-gleason/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Mark Gaughan', u'http://www.buffalonews.com/sports/bills-nfl/inside-the-nfl/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Mike Harrington', u'http://www.buffalonews.com/sports/columns/mike-harrington/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Jerry Sullivan', u'http://www.buffalonews.com/sports/columns/jerry-sullivan/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Other Sports Columns', u'http://www.buffalonews.com/sports/columns/other-sports-columns/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Life', u'http://www.buffalonews.com/life/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Bruce Andriatch', u'http://www.buffalonews.com/city/columns/bruce-andriatch/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Donn Esmonde', u'http://www.buffalonews.com/city/columns/donn-esmonde/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Rod Watson', u'http://www.buffalonews.com/city/columns/rod-watson/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Entertainment', u'http://www.buffalonews.com/entertainment/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Off Main Street', u'http://www.buffalonews.com/city/columns/off-main-street/?widget=rssfeed&view=feed&contentId=77944'),
-         (u'Editorials', u'http://www.buffalonews.com/editorial-page/buffalo-news-editorials/?widget=rssfeed&view=feed&contentId=77944')
+    feeds          = [
+            (u'City of Buffalo', u'http://www.buffalonews.com/city/communities/buffalo/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Southern Erie County', u'http://www.buffalonews.com/city/communities/southern-erie/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Eastern Erie County', u'http://www.buffalonews.com/city/communities/eastern-erie/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Southern Tier', u'http://www.buffalonews.com/city/communities/southern-tier/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Niagara County', u'http://www.buffalonews.com/city/communities/niagara-county/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Business', u'http://www.buffalonews.com/business/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'MoneySmart', u'http://www.buffalonews.com/business/moneysmart/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Bills & NFL', u'http://www.buffalonews.com/sports/bills-nfl/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Sabres & NHL', u'http://www.buffalonews.com/sports/sabres-nhl/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Bob DiCesare', u'http://www.buffalonews.com/sports/columns/bob-dicesare/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Bucky Gleason', u'http://www.buffalonews.com/sports/columns/bucky-gleason/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Mark Gaughan', u'http://www.buffalonews.com/sports/bills-nfl/inside-the-nfl/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Mike Harrington', u'http://www.buffalonews.com/sports/columns/mike-harrington/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Jerry Sullivan', u'http://www.buffalonews.com/sports/columns/jerry-sullivan/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Other Sports Columns', u'http://www.buffalonews.com/sports/columns/other-sports-columns/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Life', u'http://www.buffalonews.com/life/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Bruce Andriatch', u'http://www.buffalonews.com/city/columns/bruce-andriatch/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Donn Esmonde', u'http://www.buffalonews.com/city/columns/donn-esmonde/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Rod Watson', u'http://www.buffalonews.com/city/columns/rod-watson/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Entertainment', u'http://www.buffalonews.com/entertainment/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Off Main Street', u'http://www.buffalonews.com/city/columns/off-main-street/?widget=rssfeed&view=feed&contentId=77944'),
+            (u'Editorials', u'http://www.buffalonews.com/editorial-page/buffalo-news-editorials/?widget=rssfeed&view=feed&contentId=77944')
          ]
+

From ac8cbdd1689d07be533226155bf340c1697510cf Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Fri, 25 Nov 2011 20:25:12 +0530
Subject: [PATCH 09/21] ...

---
 src/calibre/manual/gui.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/manual/gui.rst b/src/calibre/manual/gui.rst
index 464e11889c..34dad57d93 100755
--- a/src/calibre/manual/gui.rst
+++ b/src/calibre/manual/gui.rst
@@ -351,7 +351,7 @@ You can build advanced search queries easily using the :guilabel:`Advanced Searc
 clicking the button |sbi|.
 
 Available fields for searching are: ``tag, title, author, publisher, series, series_index, rating, cover,
-comments, format, identifiers, date, pubdate, search, size`` and custom columns. If a device is plugged in, the ``ondevice`` field becomes available. To find the search name (actually called the `lookup name`) for a custom column, hover your mouse over the column header in the library view.
+comments, format, identifiers, date, pubdate, search, size`` and custom columns. If a device is plugged in, the ``ondevice`` field becomes available, when searching the calibre library view. To find the search name (actually called the `lookup name`) for a custom column, hover your mouse over the column header in the library view.
 
 The syntax for searching for dates is::
 

From d134be5b1b89c87fafa01c5fa3a45202b9732848 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 26 Nov 2011 07:58:02 +0530
Subject: [PATCH 10/21] ...

---
 recipes/independent.recipe | 45 +++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/recipes/independent.recipe b/recipes/independent.recipe
index 7403163e6a..ebe0a30fd2 100644
--- a/recipes/independent.recipe
+++ b/recipes/independent.recipe
@@ -39,7 +39,9 @@ class TheIndependentNew(BasicNewsRecipe):
     encoding                = 'utf-8'
     remove_tags             =[
                                dict(attrs={'id' : ['RelatedArtTag','renderBiography']}),
-                               dict(attrs={'class' : ['autoplay','openBiogPopup']})
+                               dict(attrs={'class' : ['autoplay','openBiogPopup']}),
+                               dict(name='img',attrs={'alt' : ['Get Adobe Flash player']}),
+                               dict(attrs={'style' : re.compile('.*')}),
                              ]
 
     keep_only_tags          =[dict(attrs={'id':'main'})]
@@ -113,6 +115,7 @@ class TheIndependentNew(BasicNewsRecipe):
                     return None
 
         items_to_extract = []
+        slideshow_elements = []
 
         for item in soup.findAll(attrs={'class' : re.compile("widget.*")}):
             remove = True
@@ -131,6 +134,7 @@ class TheIndependentNew(BasicNewsRecipe):
             if (pattern.search(item['class'])) is not None:
                 if self._FETCH_IMAGES:
                     remove = False
+                    slideshow_elements.append(item)
                 else:
                     remove = True
 
@@ -148,28 +152,29 @@ class TheIndependentNew(BasicNewsRecipe):
         items_to_extract = []
 
         if self._FETCH_IMAGES:
-            for item in soup.findAll('a',attrs={'href' : re.compile('.*')}):
-                if item.img is not None:
-                    #use full size image
-                    img = item.findNext('img')
+            for element in slideshow_elements:
+                for item in element.findAll('a',attrs={'href' : re.compile('.*')}):
+                    if item.img is not None:
+                        #use full size image
+                        img = item.findNext('img')
 
-                    img['src'] = item['href']
+                        img['src'] = item['href']
 
-                    #insert caption if available
-                    if img['title'] is not None and (len(img['title']) > 1):
-                        tag = Tag(soup,'h3')
-                        text = NavigableString(img['title'])
-                        tag.insert(0,text)
+                        #insert caption if available
+                        if img.get('title') and (len(img['title']) > 1):
+                            tag = Tag(soup,'h3')
+                            text = NavigableString(img['title'])
+                            tag.insert(0,text)
 
-                        #picture before text
-                        img.extract()
-                        item.insert(0,img)
-                        item.insert(1,tag)
+                            #picture before text
+                            img.extract()
+                            item.insert(0,img)
+                            item.insert(1,tag)
 
-                    # remove link
-                    item.name = "div"
-                    item["class"]='image'
-                    del item["href"]
+                        # remove link
+                        item.name = "div"
+                        item["class"]='image'
+                        del item["href"]
 
 
         #remove empty subtitles
@@ -283,7 +288,7 @@ class TheIndependentNew(BasicNewsRecipe):
         items_to_extract = []
         for item in soup.findAll('div', attrs={'class' : 'image'}):
             img = item.findNext('img')
-            if img is not None and img['src'] is not None:
+            if img and img.get('src'):
                 # broken images still point to remote url
                 pattern = re.compile('http://www.independent.co.uk.*')
                 if pattern.match(img["src"]) is not None:

From 187ca2a518959a96627a55b0331cd77fc4274570 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 26 Nov 2011 10:28:54 +0530
Subject: [PATCH 11/21] Fix #896412 (Improvement for #886763 (or a little bug
 in its implementation))

---
 src/calibre/ebooks/metadata/__init__.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py
index d9399e9a3e..8dc14d52df 100644
--- a/src/calibre/ebooks/metadata/__init__.py
+++ b/src/calibre/ebooks/metadata/__init__.py
@@ -109,12 +109,16 @@ def get_title_sort_pat(lang=None):
     q = canonicalize_lang(q) if q else q
     data = tweaks['per_language_title_sort_articles']
     ans = data.get(q, None)
-    if ans is None:
-        ans = data['eng']
-    ans = frozenset(ans + data['eng'])
+    try:
+        ans = frozenset(ans) if ans else frozenset(data['eng'])
+    except:
+        ans = frozenset((r'A\s+', r'The\s+', r'An\s+'))
     ans = '|'.join(ans)
     ans = '^(%s)'%ans
-    ans = re.compile(ans, re.IGNORECASE)
+    try:
+        ans = re.compile(ans, re.IGNORECASE)
+    except:
+        ans = re.compile(r'^(A|The|An)\s+', re.IGNORECASE)
     _title_pats[lang] = ans
     return ans
 

From 209344e319a96d5cec13ab8b724cb437152abc1b Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 26 Nov 2011 10:46:07 +0530
Subject: [PATCH 12/21] ...

---
 resources/default_tweaks.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 8c82f1d4e6..ecd3888bf7 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -242,6 +242,8 @@ per_language_title_sort_articles = {
         # Greek
         'ell'  : (r'O\s+', r'I\s+', r'To\s+', r'Ta\s+', r'Tus\s+', r'Tis\s+',
                   r"'Enas\s+", r"'Mia\s+", r"'Ena\s+", r"'Enan\s+", ),
+        # Hungarian
+        'hun'  : (r'A\s+', 'Az\s+', 'Egy\s+',),
 }
 default_language_for_title_sort = None
 title_sort_articles=r'^(A|The|An)\s+'

From 9c7775e43d9043ec2b307f5231238502cffd222d Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 26 Nov 2011 12:17:28 +0530
Subject: [PATCH 13/21] Fix #896508 (unused variable warnings)

---
 src/calibre/gui2/pictureflow/pictureflow.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/gui2/pictureflow/pictureflow.cpp b/src/calibre/gui2/pictureflow/pictureflow.cpp
index b82747841c..4e9d8a402d 100644
--- a/src/calibre/gui2/pictureflow/pictureflow.cpp
+++ b/src/calibre/gui2/pictureflow/pictureflow.cpp
@@ -1409,8 +1409,8 @@ void PictureFlow::dataChanged() { d->dataChanged(); }
 void PictureFlow::emitcurrentChanged(int index) { emit currentChanged(index); }
 
 int FlowImages::count() { return 0; }
-QImage FlowImages::image(int index) { index=0; return QImage(); }
-QString FlowImages::caption(int index) {index=0; return QString(); }
-QString FlowImages::subtitle(int index) {index=0; return QString(); }
+QImage FlowImages::image(int index) { Q_UNUSED(index); return QImage(); }
+QString FlowImages::caption(int index) { Q_UNUSED(index); return QString(); }
+QString FlowImages::subtitle(int index) { Q_UNUSED(index); return QString(); }
 
 // }}}

From ee53cb611db7562c539df662dccc156002397093 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 26 Nov 2011 17:59:23 +0530
Subject: [PATCH 14/21] ...

---
 src/calibre/library/cli.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py
index 43d4e8276b..86f8a070f3 100644
--- a/src/calibre/library/cli.py
+++ b/src/calibre/library/cli.py
@@ -342,7 +342,8 @@ def remove_option_parser():
 
 Remove the books identified by ids from the database. ids should be a comma separated \
 list of id numbers (you can get id numbers by using the list command). For example, \
-23,34,57-85
+23,34,57-85 (when specifying a range, the last number in the range is not
+included).
 '''))
 
 def command_remove(args, dbpath):

From 98de6945b397e4f18b8a73b7e8b2215b32b2df0d Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 26 Nov 2011 18:38:49 +0530
Subject: [PATCH 15/21] =?UTF-8?q?When=20searching=20for=20author=20names?=
 =?UTF-8?q?=20with=20accented=20characters,=20allow=20the=20non=20accented?=
 =?UTF-8?q?=20version=20to=20match.=20For=20example,=20searching=20for=20N?=
 =?UTF-8?q?ino=20will=20now=20match=20Ni=C3=B1o.=20Fixes=20#879729=20([Enh?=
 =?UTF-8?q?ancement]=20Searching=20for=20accented=20and=20non-standard=20l?=
 =?UTF-8?q?etters)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/calibre/library/caches.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 81db8830b7..15d9123b81 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -15,7 +15,7 @@ from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import parse_date, now, UNDEFINED_DATE, clean_date_for_sort
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
-from calibre.utils.localization import canonicalize_lang, lang_map
+from calibre.utils.localization import canonicalize_lang, lang_map, get_udc
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre import prints
@@ -217,6 +217,7 @@ class ResultCache(SearchQueryParser): # {{{
         self.FIELD_MAP = FIELD_MAP
         self.db_prefs = db_prefs
         self.composites = {}
+        self.udc = get_udc()
         for key in field_metadata:
             if field_metadata[key]['datatype'] == 'composite':
                 self.composites[field_metadata[key]['rec_index']] = key
@@ -261,6 +262,15 @@ class ResultCache(SearchQueryParser): # {{{
 
     # Search functions {{{
 
+    def ascii_name(self, name):
+        try:
+            ans = self.udc.decode(name)
+            if ans == name:
+                ans = False
+        except:
+            ans = False
+        return ans
+
     def universal_set(self):
         return set([i[0] for i in self._data if i is not None])
 
@@ -734,6 +744,8 @@ class ResultCache(SearchQueryParser): # {{{
                 else:
                     q = query
 
+                au_loc = self.FIELD_MAP['authors']
+
                 for id_ in candidates:
                     item = self._data[id_]
                     if item is None: continue
@@ -776,6 +788,9 @@ class ResultCache(SearchQueryParser): # {{{
                     if loc not in exclude_fields: # time for text matching
                         if is_multiple_cols[loc] is not None:
                             vals = [v.strip() for v in item[loc].split(is_multiple_cols[loc])]
+                            if loc == au_loc:
+                                vals += filter(None, map(self.ascii_name,
+                                    vals))
                         else:
                             vals = [item[loc]] ### make into list to make _match happy
                         if _match(q, vals, matchkind):

From 4fcbaa8919921d426bc4ad9f02eafb1a35835b6c Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 26 Nov 2011 20:13:13 +0530
Subject: [PATCH 16/21] Fix #896585 (extra articles for dutch language)

---
 resources/default_tweaks.py      |  4 +++-
 src/calibre/gui2/duplicates.py   | 11 +++++++++++
 src/calibre/library/database2.py | 17 +++++++++++++++--
 3 files changed, 29 insertions(+), 3 deletions(-)
 create mode 100644 src/calibre/gui2/duplicates.py

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index ecd3888bf7..224038b0f9 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -232,7 +232,9 @@ per_language_title_sort_articles = {
         'deu'  : (r'Der\s+', r'Die\s+', r'Das\s+', r'Den\s+', r'Ein\s+',
                   r'Eine\s+', r'Einen\s+', ),
         # Dutch
-        'nld'  : (r'De\s+', r'Het\s+', r'Een\s+', ),
+        'nld'  : (r'De\s+', r'Het\s+', r'Een\s+', r"'n\s+", r"'s\s+", r'Ene\s+',
+                  r'Ener\s+', r'Enes\s+', r'Den\s+', r'Der\s+', r'Des\s+',
+                  r"'t\s+"),
         # Swedish
         'swe'  : (r'En\s+', r'Ett\s+', r'Det\s+', r'Den\s+', r'De\s+', ),
         # Turkish
diff --git a/src/calibre/gui2/duplicates.py b/src/calibre/gui2/duplicates.py
new file mode 100644
index 0000000000..cc6da1e995
--- /dev/null
+++ b/src/calibre/gui2/duplicates.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index a917aa09b2..3782149512 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -13,7 +13,7 @@ import threading, random
 from itertools import repeat
 from math import ceil
 
-from calibre import prints
+from calibre import prints, force_unicode
 from calibre.ebooks.metadata import (title_sort, author_to_author_sort,
         string_to_authors, authors_to_string, get_title_sort_pat)
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
@@ -33,7 +33,7 @@ from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
 from calibre.utils.config import prefs, tweaks, from_json, to_json
-from calibre.utils.icu import sort_key, strcmp
+from calibre.utils.icu import sort_key, strcmp, lower
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
 from calibre.utils.magick.draw import save_cover_data_to
@@ -1003,6 +1003,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             return bool(self.conn.get('SELECT id FROM books where title=?', (title,), all=False))
         return False
 
+    def books_with_same_title(self, mi, all_matches=True):
+        title = mi.title
+        ans = set()
+        if title:
+            title = lower(force_unicode(title))
+            for book_id in self.all_ids():
+                x = self.title(book_id, index_is_id=True)
+                if lower(x) == title:
+                    ans.add(book_id)
+                    if not all_matches:
+                        break
+        return ans
+
     def find_identical_books(self, mi):
         fuzzy_title_patterns = [(re.compile(pat, re.IGNORECASE) if
             isinstance(pat, basestring) else pat, repl) for pat, repl in

From 4e59919287d7ce9949aef69d4c2983810d7e5042 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= <tomek3d@gmail.com>
Date: Sat, 26 Nov 2011 21:39:53 +0100
Subject: [PATCH 17/21] many fixes for ebookpoint plugin

---
 src/calibre/gui2/store/stores/ebookpoint_plugin.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/calibre/gui2/store/stores/ebookpoint_plugin.py b/src/calibre/gui2/store/stores/ebookpoint_plugin.py
index 98f0c9685f..19b2e0a428 100644
--- a/src/calibre/gui2/store/stores/ebookpoint_plugin.py
+++ b/src/calibre/gui2/store/stores/ebookpoint_plugin.py
@@ -42,9 +42,8 @@ class EbookpointStore(BasicStoreConfig, StorePlugin):
             d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
-    def search(self, query, max_results=10, timeout=60):
-        url = 'http://ebookpoint.pl/search.scgi?szukaj=' + urllib.quote(query) + '&serwisyall=0&x=0&y=0'
-        ebook_string = 'eBook.'
+    def search(self, query, max_results=25, timeout=60):
+        url = 'http://ebookpoint.pl/search.scgi?szukaj=' + urllib.quote_plus(query.decode('utf-8').encode('iso-8859-2')) + '&serwisyall=0&x=0&y=0'
 
         br = browser()
 
@@ -60,19 +59,19 @@ class EbookpointStore(BasicStoreConfig, StorePlugin):
                     continue
 
                 cover_url = ''.join(data.xpath('.//a[@class="cover"]/img/@src'))
-                title = ''.join(data.xpath('.//h3/a/text()'))
-                title = re.sub(ebook_string, '', title)
+                title = ''.join(data.xpath('.//h3/a/@title'))
+                title = re.sub('eBook.', '', title)
                 author = ''.join(data.xpath('.//p[@class="author"]/text()'))
                 price = ''.join(data.xpath('.//p[@class="price"]/ins/text()'))
 
                 with closing(br.open(id.strip(), timeout=timeout)) as nf:
                     idata = html.fromstring(nf.read())
-                    formats = ', '.join(idata.xpath('//div[@class="col-left"]/h2[contains(., "' + ebook_string + '")]/@class'))
+                    formats = ', '.join(idata.xpath('//dd[@class="radio-line"]/label/text()'))
 
                 counter -= 1
 
                 s = SearchResult()
-                s.cover_url = 'http://ebookpoint.pl' + cover_url
+                s.cover_url = 'http://ebookpoint.pl' + re.sub('72x9', '65x8',cover_url)
                 s.title = title.strip()
                 s.author = author.strip()
                 s.price = re.sub(r'\.',',',price)

From b95c532ba3f5eb8b645d9839cccf84d4de856cea Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 27 Nov 2011 07:42:57 +0530
Subject: [PATCH 18/21] ...

---
 resources/default_tweaks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 224038b0f9..c71e94a70e 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -219,7 +219,7 @@ per_language_title_sort_articles = {
                   r'Una\s+', r'Unos\s+', r'Unas\s+'),
         # French
         'fra'  : (r'Le\s+', r'La\s+', r"L'", r'Les\s+', r'Un\s+', r'Une\s+',
-                  r'Des\s+'),
+                  r'Des\s+', r'De\s+', r'La\s+', r'De\s+La\s+', r"D'"),
         # Italian
         'ita'  : (r'Lo\s+', r'Il\s+', r"L'", r'La\s+', r'Gli\s+', r'I\s+',
                   r'Le\s+', ),
@@ -230,7 +230,7 @@ per_language_title_sort_articles = {
         'ron'  : (r'Un\s+', r'O\s+', r'Nişte\s+', ),
         # German
         'deu'  : (r'Der\s+', r'Die\s+', r'Das\s+', r'Den\s+', r'Ein\s+',
-                  r'Eine\s+', r'Einen\s+', ),
+                  r'Eine\s+', r'Einen\s+', r'Dem\s+', ),
         # Dutch
         'nld'  : (r'De\s+', r'Het\s+', r'Een\s+', r"'n\s+", r"'s\s+", r'Ene\s+',
                   r'Ener\s+', r'Enes\s+', r'Den\s+', r'Der\s+', r'Des\s+',

From 8e7a83fc9088f90bd6b4e5ccff66de4a9f6a6e9d Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 27 Nov 2011 07:47:27 +0530
Subject: [PATCH 19/21] Cosmopolitan UK by Dave Asbury

---
 recipes/cosmopolitan_uk.recipe | 51 ++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 recipes/cosmopolitan_uk.recipe

diff --git a/recipes/cosmopolitan_uk.recipe b/recipes/cosmopolitan_uk.recipe
new file mode 100644
index 0000000000..21317063ab
--- /dev/null
+++ b/recipes/cosmopolitan_uk.recipe
@@ -0,0 +1,51 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+#from calibre import __appname__
+from calibre.utils.magick import Image
+class AdvancedUserRecipe1306097511(BasicNewsRecipe):
+    title          = u'Cosmopolitan UK'
+    description = 'Fashion, beauty and Gossip for women from COSMOPOLITAN -UK'
+
+    __author__ = 'Dave Asbury'
+    # greyscale code by Starson
+    cover_url = 'http://www.cosmopolitan.magazine.co.uk/files/4613/2085/8988/Cosmo_Cover3.jpg'
+    no_stylesheets = True
+    oldest_article = 7
+    max_articles_per_feed = 20
+    remove_empty_feeds = True
+    remove_javascript     = True
+
+    preprocess_regexps = [
+    (re.compile(r'<!-- Begin tmpl module_competition_offer -->.*?<!-- End tmpl module_competition_offer-->', re.IGNORECASE | re.DOTALL), lambda match: '')]
+    language = 'en_GB'
+
+
+    masthead_url        = 'http://www.cosmopolitan.co.uk/cm/cosmopolitanuk/site_images/header/cosmouk_logo_home.gif'
+
+
+    keep_only_tags = [
+                              dict(attrs={'class' : ['dateAuthor', 'publishDate']}),
+                              dict(name='div',attrs ={'id' : ['main_content']})
+                              ]
+    remove_tags    = [
+                              dict(name='div',attrs={'class' : ['blogInfo','viral_toolbar','comment_number','prevEntry nav']}),
+                              dict(name='div',attrs={'class' : 'blog_module_about_the_authors'}),
+                              dict(attrs={'id': ['breadcrumbs','comment','related_links_list','right_rail','content_sec_fb_more','content_sec_mostpopularstories','content-sec_fb_frame_viewfb_bot']}),
+                              dict(attrs={'class' : ['read_liked_that_header','fb_back_next_area']})
+                            ]
+
+    feeds          = [
+        (u'Love & Sex', u'http://www.cosmopolitan.co.uk/love-sex/rss/'), (u'Men', u'http://cosmopolitan.co.uk/men/rss/'), (u'Fashion', u'http://cosmopolitan.co.uk/fashion/rss/'), (u'Hair & Beauty', u'http://cosmopolitan.co.uk/beauty-hair/rss/'), (u'LifeStyle', u'http://cosmopolitan.co.uk/lifestyle/rss/'), (u'Cosmo On Campus', u'http://cosmopolitan.co.uk/campus/rss/'), (u'Celebrity Gossip', u'http://cosmopolitan.co.uk/celebrity-gossip/rss/')]
+
+    def postprocess_html(self, soup, first):
+        #process all the images
+        for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')):
+            iurl = tag['src']
+            img = Image()
+            img.open(iurl)
+            if img < 0:
+                raise RuntimeError('Out of memory')
+            img.type = "GrayscaleType"
+            img.save(iurl)
+        return soup
+

From f1fca7cbb984ce61ac68b983b8226e11bf6b9517 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 27 Nov 2011 07:59:12 +0530
Subject: [PATCH 20/21] ...

---
 resources/default_tweaks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index c71e94a70e..a359f73be1 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -219,7 +219,7 @@ per_language_title_sort_articles = {
                   r'Una\s+', r'Unos\s+', r'Unas\s+'),
         # French
         'fra'  : (r'Le\s+', r'La\s+', r"L'", r'Les\s+', r'Un\s+', r'Une\s+',
-                  r'Des\s+', r'De\s+', r'La\s+', r'De\s+La\s+', r"D'"),
+                  r'Des\s+', r'De\s+La\s+', r'De\s+', r"D'"),
         # Italian
         'ita'  : (r'Lo\s+', r'Il\s+', r"L'", r'La\s+', r'Gli\s+', r'I\s+',
                   r'Le\s+', ),

From f55e5efa2a82a8f54f83775c732f6337130bb23e Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 27 Nov 2011 10:08:07 +0530
Subject: [PATCH 21/21] Fix #896620 (Not possible to include language in CSV
 catalogue)

---
 src/calibre/customize/__init__.py | 2 +-
 src/calibre/library/catalog.py    | 6 ++++--
 src/calibre/library/database2.py  | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py
index d087eb5351..e4f8bf99bf 100644
--- a/src/calibre/customize/__init__.py
+++ b/src/calibre/customize/__init__.py
@@ -449,7 +449,7 @@ class CatalogPlugin(Plugin): # {{{
                           ['author_sort','authors','comments','cover','formats',
                            'id','isbn','ondevice','pubdate','publisher','rating',
                            'series_index','series','size','tags','timestamp',
-                           'title_sort','title','uuid'])
+                           'title_sort','title','uuid','languages'])
         all_custom_fields = set(db.custom_field_keys())
         all_fields = all_std_fields.union(all_custom_fields)
 
diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py
index 53432af7b7..a9c4b1a309 100644
--- a/src/calibre/library/catalog.py
+++ b/src/calibre/library/catalog.py
@@ -29,7 +29,8 @@ from calibre.utils.zipfile import ZipFile
 
 FIELDS = ['all', 'title', 'title_sort', 'author_sort', 'authors', 'comments',
           'cover', 'formats','id', 'isbn', 'ondevice', 'pubdate', 'publisher',
-          'rating', 'series_index', 'series', 'size', 'tags', 'timestamp', 'uuid']
+          'rating', 'series_index', 'series', 'size', 'tags', 'timestamp',
+          'uuid', 'languages']
 
 #Allowed fields for template
 TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate', 'title_sort',
@@ -601,7 +602,7 @@ class BIBTEX(CatalogPlugin): # {{{
                     bibtexc, db, citation_bibtex, addfiles_bibtex))
 # }}}
 
-class EPUB_MOBI(CatalogPlugin):
+class EPUB_MOBI(CatalogPlugin): # {{{
     'ePub catalog generator'
 
     Option = namedtuple('Option', 'option, default, dest, action, help')
@@ -5177,3 +5178,4 @@ Author '{0}':
 
         # returns to gui2.actions.catalog:catalog_generated()
         return catalog.error
+# }}}
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 3782149512..f99830ca5a 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -3378,7 +3378,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             prefix = self.library_path
         FIELDS = set(['title', 'sort', 'authors', 'author_sort', 'publisher', 'rating',
             'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
-            'uuid', 'pubdate', 'last_modified', 'identifiers'])
+            'uuid', 'pubdate', 'last_modified', 'identifiers', 'languages'])
         for x in self.custom_column_num_map:
             FIELDS.add(x)
         data = []