Sync to trunk.

.bzrignore

@@ -14,6 +14,7 @@ resources/scripts.pickle
 resources/ebook-convert-complete.pickle
 resources/builtin_recipes.xml
 resources/builtin_recipes.zip
+resources/template-functions.json
 setup/installer/windows/calibre/build.log
 src/calibre/translations/.errors
 src/cssutils/.svn/

Changelog.yaml

@@ -19,6 +19,135 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.5
+  date: 2011-06-10
+
+  new features:
+    - title: "A new 'portable' calibre build, useful if you like to carry around calibre and its library on a USB key"
+      type: major
+      description: "For details, see: http://calibre-ebook.com/download_portable"
+
+    - title: "E-book viewer: Remember the last used font size multiplier."
+      tickets: [774343]
+
+    - title: "Preliminary support for the Kobo Touch. Drivers for the ZTE v9 tablet, Samsung S2, Notion Ink Adam and PocketBook 360+"
+
+    - title: "When downloading metadata merge rather than replace tags"
+
+    - title: "Edit metadata dialog: When pasting in an ISBN, if not valid ISBN if present on the clipboard popup a box for the user to enter the ISBN"
+
+    - title: "Windows build: Add code to load .pyd python extensions from a zip file. This allows many more files in the calibre installation to be zipped up, speeding up the installer."
+    - title: "Add an action to remove all formats from the selected books to the remove books button"
+
+  
+  bug fixes:
+    - title: "Various minor bug fixes to the column coloring code"
+
+    - title: "Fix the not() template function"
+
+    - title: "Nook Color/TSR: When sending books to the storage card place them in the My Files/Books subdirectory. Also do not upload cover thumbnails as users report that the NC/TSR don't use them."
+      tickets: [792842]
+
+    - title: "Get Books: Update plugins for Amazon and B&N stores to handle website changes. Enable some stores by default on first run. Add Zixo store"
+      tickets: [792762]
+
+    - title: "Comic Input: Replace the # character in filenames as it can cause problem with conversion/vieweing."
+      tickets: [792723]
+
+    - title: "When writing files to zipfile, reset timestamp if it doesn't fit in 1980's vintage storage structures"
+
+    - title: "Amazon metadata plugin: Fix parsing of published date from amazon.de when it has februar in it"
+
+  improved recipes:
+    - Ambito
+    - GoComics
+    - Le Monde Diplomatique
+    - Max Planck
+    - express.de
+
+  new recipes:
+    - title: Ambito Financiero
+      author: Darko Miletic
+
+    - title: Stiin Tas Technica
+      author: Silviu Cotoara
+
+    - title: "Metro News NL"
+      author: DrMerry
+
+    - title: "Brigitte.de, Polizeipresse DE and Heise Online"
+      author: schuster
+
+
+
+- version: 0.8.4
+  date: 2011-06-03
+
+  new features:
+    - title: "New and much simpler interface for specifying column coloring via Preferences->Look & Feel->Column Coloring"
+
+    - title: "Driver for Trekstor eBook Player 5M, Samsung Galaxy SII I9100, Motorola Defy and miBuk GAMMA 6.2"
+      tickets: [792091, 791216]
+
+    - title: "Get Books: Add EpubBud, WH Smits and E-book Shoppe stores"
+
+    - title: "When deleting 'all formats except ...', do not delete if it leaves a book with no formats"
+
+    - title: "Change default toolbar to make it a little more new user friendly. The icons have been re-arranged and now the text is always visiblke by default. You can change that in Preferences->Look & Feeel and Preferences->Toolbar"
+
+    - title: "Windows installer: Remember and use previous settings for installing desktop icons, adding to path, etc. This makes the installer a little slower, complaints should go to Microsoft."
+
+    - title: "Template language: Add str_in_list and on_device formatter functions. Make debugging templates a little easier"
+
+    - title: "Allow the user to specify formatting for number type custom columns"
+  
+  bug fixes:
+    - title: "Fix typo in NOOK TSR driver that prevented it from working on windows"
+
+    - title: "Fix quotes in identifiers causing Tag Browser to be blank."
+      tickets: [791044]
+
+    - title: "Speedup auto complete when there are lots of items (>2500) the downside being that non ASCII characters are not sorted correctly. The threshold can be controlled via Preferences->Tweaks"
+      tickets: [792191]
+
+    - title: "RTF Output: Fix handling of curly brackets"
+      tickets: [791805]
+
+    - title: "Fix searching in Get Books not working with non ASCII characters"
+      tickets: [791788]
+
+    - title: "Fix excessive memory consumption when moving very large files during a metadata change"
+      tickets: [791806]
+
+    - title: "Fix series index being overwritten even when series is turned off in bulk metadata download"
+      tickets: [789990]
+
+    - title: "Fix regression in templates where id and other non standard fields no longer worked."
+
+    - title: "EPUB Output: Fix crash caused by ids with non-ascii characters in them"
+
+    - title: "Try to preserve the timestamps of files in a ZIP container"
+
+    - title: "After adding books always select the most recently added book."
+      tickets: [789343]
+
+  improved recipes:
+    - bild.de
+    - CNN
+    - BBC News (fast)
+    - Dilema Veche
+
+  new recipes:
+    - title: Metro UK
+      author: Dave Asbury
+
+    - title: Alt om Herning and Version2.dk
+      author: Rasmus Lauritsen
+
+    - title: Observatorul cultural
+      author: song2
+
+
 - version: 0.8.3
   date: 2011-05-27
 

recipes/ambito.recipe

@@ -1,7 +1,5 @@
-#!/usr/bin/env  python
-
 __license__   = 'GPL v3'
-__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 ambito.com
 '''
@@ -11,51 +9,56 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class Ambito(BasicNewsRecipe):
     title                 = 'Ambito.com'
     __author__            = 'Darko Miletic'
-    description           = 'Informacion Libre las 24 horas'
+    description           = 'Ambito.com con noticias del Diario Ambito Financiero de Buenos Aires'
-    publisher             = 'Ambito.com'
+    publisher             = 'Editorial Nefir S.A.'
-    category              = 'news, politics, Argentina'
+    category              = 'news, politics, economy, finances, Argentina'
     oldest_article        = 2
-    max_articles_per_feed = 100
     no_stylesheets        = True
-    encoding              = 'iso-8859-1'
+    encoding              = 'cp1252'
-    cover_url             = 'http://www.ambito.com/img/logo_.jpg'
+    masthead_url          = 'http://www.ambito.com/img/logo_.jpg'
-    remove_javascript     = True
     use_embedded_content  = False
+    language              = 'es_AR'
+    publication_type      = 'newsportal'
+    extra_css             = """
+                                body{font-family: "Trebuchet MS",Verdana,sans-serif}
+                                .volanta{font-size: small}
+                                .t2_portada{font-size: xx-large; font-family: Georgia,serif; color: #026698}
+                            """
 
-    html2lrf_options = [
-                          '--comment', description
-                        , '--category', category
-                        , '--publisher', publisher
-                        ]
 
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
 
     keep_only_tags    = [dict(name='div', attrs={'align':'justify'})]
-
+    remove_tags       = [dict(name=['object','link','embed','iframe','meta','link','table','img'])]
-    remove_tags       = [dict(name=['object','link'])]
+    remove_attributes = ['align']
 
     feeds = [
               (u'Principales Noticias', u'http://www.ambito.com/rss/noticiasp.asp'                         )
              ,(u'Economia'            , u'http://www.ambito.com/rss/noticias.asp?S=Econom%EDa'             )
              ,(u'Politica'            , u'http://www.ambito.com/rss/noticias.asp?S=Pol%EDtica'             )
              ,(u'Informacion General' , u'http://www.ambito.com/rss/noticias.asp?S=Informaci%F3n%20General')
-             ,(u'Agro'                , u'http://www.ambito.com/rss/noticias.asp?S=Agro'                   )
+             ,(u'Campo'               , u'http://www.ambito.com/rss/noticias.asp?S=Agro'                   )
              ,(u'Internacionales'     , u'http://www.ambito.com/rss/noticias.asp?S=Internacionales'        )
              ,(u'Deportes'            , u'http://www.ambito.com/rss/noticias.asp?S=Deportes'               )
              ,(u'Espectaculos'        , u'http://www.ambito.com/rss/noticias.asp?S=Espect%E1culos'         )
-             ,(u'Tecnologia'          , u'http://www.ambito.com/rss/noticias.asp?S=Tecnologia'             )
+             ,(u'Tecnologia'          , u'http://www.ambito.com/rss/noticias.asp?S=Tecnolog%EDa'           )
-             ,(u'Salud'               , u'http://www.ambito.com/rss/noticias.asp?S=Salud'                  )
              ,(u'Ambito Nacional'     , u'http://www.ambito.com/rss/noticias.asp?S=Ambito%20Nacional'      )
             ]
 
     def print_version(self, url):
-        return url.replace('http://www.ambito.com/noticia.asp?','http://www.ambito.com/noticias/imprimir.asp?')
+        return url.replace('/noticia.asp?','/noticias/imprimir.asp?')
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Language" content="es-AR"/>'
-        soup.head.insert(0,mtag)
         for item in soup.findAll(style=True):
             del item['style']
+        for item in soup.findAll('a'):
+            str = item.string
+            if str is None:
+               str = self.tag_to_string(item)
+            item.replaceWith(str)
         return soup
-
-    language = 'es_AR'

recipes/ambito_financiero.recipe

@@ -0,0 +1,87 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+ambito.com/diario
+'''
+
+import time
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Ambito_Financiero(BasicNewsRecipe):
+    title                 = 'Ambito Financiero'
+    __author__            = 'Darko Miletic'
+    description           = 'Informacion Libre las 24 horas'
+    publisher             = 'Editorial Nefir S.A.'
+    category              = 'news, politics, economy, Argentina'
+    no_stylesheets        = True
+    encoding              = 'cp1252'
+    masthead_url          = 'http://www.ambito.com/diario/img/logo_af.gif'
+    publication_type      = 'newspaper'
+    needs_subscription    = 'optional'
+    use_embedded_content  = False
+    language              = 'es_AR'
+    PREFIX                = 'http://www.ambito.com'
+    INDEX                 = PREFIX + '/diario/index.asp'
+    LOGIN                 = PREFIX + '/diario/login/entrada.asp'
+    extra_css             = """
+                                body{font-family: "Trebuchet MS",Verdana,sans-serif}
+                                .volanta{font-size: small}
+                                .t2_portada{font-size: xx-large; font-family: Georgia,serif; color: #026698}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    keep_only_tags    = [dict(name='div', attrs={'align':'justify'})]
+    remove_tags       = [dict(name=['object','link','embed','iframe','meta','link','table','img'])]
+    remove_attributes = ['align']
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        br.open(self.INDEX)
+        if self.username is not None and self.password is not None:
+            br.open(self.LOGIN)
+            br.select_form(name='frmlogin')
+            br['USER_NAME'] = self.username
+            br['USER_PASS'] = self.password
+            br.submit()
+        return br
+
+    def print_version(self, url):
+        return url.replace('/diario/noticia.asp?','/noticias/imprimir.asp?')
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            str = item.string
+            if str is None:
+               str = self.tag_to_string(item)
+            item.replaceWith(str)
+        return soup
+
+    def parse_index(self):
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('img',attrs={'class':'fotodespliegue'})
+        if cover_item:
+           self.cover_url = self.PREFIX + cover_item['src']
+        articles = []
+        checker  = []
+        for feed_link in soup.findAll('a', attrs={'class':['t0_portada','t2_portada','bajada']}):
+            url   = self.PREFIX + feed_link['href']
+            title = self.tag_to_string(feed_link)
+            date  = strftime("%a, %d %b %Y %H:%M:%S +0000",time.gmtime())
+            if url not in checker:
+                checker.append(url)
+                articles.append({
+                                  'title'      :title
+                                 ,'date'       :date
+                                 ,'url'        :url
+                                 ,'description':u''
+                                })
+        return [(self.title, articles)]

recipes/bild_de.recipe

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 from calibre.web.feeds.recipes import BasicNewsRecipe
 class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
     title          = u'Bild.de'
     __author__  = 'schuster'
     oldest_article = 1
-    max_articles_per_feed = 50
+    max_articles_per_feed = 100
     no_stylesheets         = True
     use_embedded_content   = False
     language               = 'de'
@@ -12,11 +13,25 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
 # get cover from myspace
     cover_url = 'http://a3.l3-images.myspacecdn.com/images02/56/0232f842170b4d349779f8379c27e073/l.jpg'
+    masthead_url = 'http://a3.l3-images.myspacecdn.com/images02/56/0232f842170b4d349779f8379c27e073/l.jpg'
 
 # set what to fetch on the site
     remove_tags_before =  dict(name = 'h2', attrs={'id':'cover'})
     remove_tags_after = dict(name ='div', attrs={'class':'back'})
 
+
+# remove things on the site that we don't want
+    remove_tags = [dict(name='div', attrs={'class':'credit'}),
+        dict(name='div', attrs={'class':'index'}),
+        dict(name='div', attrs={'id':'zstart31'}),
+        dict(name='div', attrs={'class':'hentry'}),
+        dict(name='div', attrs={'class':'back'}),
+        dict(name='div', attrs={'class':'pagination'}),
+        dict(name='div', attrs={'class':'header'}),
+        dict(name='div', attrs={'class':'element floatL'}),
+        dict(name='div', attrs={'class':'stWrap'})
+]
+
 #  thanx to kiklop74 for code (see sticky thread -> Recipes - Re-usable code)
 # this one removes a lot of direct-link's
     def preprocess_html(self, soup):
@@ -42,5 +57,18 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
                           (u'Unterhaltung', u'http://rss.bild.de/bild-unterhaltung.xml'),
                           (u'Sport', u'http://rss.bild.de/bild-sport.xml'),
                           (u'Lifestyle', u'http://rss.bild.de/bild-lifestyle.xml'),
-                          (u'Ratgeber', u'http://rss.bild.de/bild-ratgeber.xml')
+                          (u'Ratgeber', u'http://rss.bild.de/bild-ratgeber.xml'),
+                          (u'Reg. - Berlin', u'http://rss.bild.de/bild-berlin.xml'),
+                          (u'Reg. - Bremen', u'http://rss.bild.de/bild-bremen.xml'),
+                          (u'Reg. - Dresden', u'http://rss.bild.de/bild-dresden.xml'),
+                          (u'Reg. - Düsseldorf', u'http://rss.bild.de/bild-duesseldorf.xml'),
+                          (u'Reg. - Frankfurt-Main', u'http://rss.bild.de/bild-frankfurt-main.xml'),
+                          (u'Reg. - Hamburg', u'http://rss.bild.de/bild-hamburg.xml'),
+                          (u'Reg. - Hannover', u'http://rss.bild.de/bild-hannover.xml'),
+                          (u'Reg. - Köln', u'http://rss.bild.de/bild-koeln.xml'),
+                          (u'Reg. - Leipzig', u'http://rss.bild.de/bild-leipzig.xml'),
+                          (u'Reg. - München', u'http://rss.bild.de/bild-muenchen.xml'),
+                          (u'Reg. - Ruhrgebiet', u'http://rss.bild.de/bild-ruhrgebiet.xml'),
+                          (u'Reg. - Stuttgart', u'http://rss.bild.de/bild-stuttgart.xml')
 ]
+

recipes/brigitte_de.recipe

@@ -0,0 +1,36 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+    title          = u'Brigitte.de'
+    __author__  = 'schuster'
+    oldest_article = 14
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    remove_empty_feeds = True
+    timeout = 10
+    cover_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg'
+    masthead_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg'
+
+
+    remove_tags = [dict(attrs={'class':['linklist', 'head', 'indent right relatedContent', 'artikel-meta segment', 'segment', 'comment commentFormWrapper segment borderBG', 'segment borderBG comments', 'segment borderBG box', 'center', 'segment nextPageLink', 'inCar']}),
+                dict(id=['header', 'artTools', 'context', 'interact', 'footer-navigation', 'bwNet', 'copy', 'keyboardNavigationHint']),
+                dict(name=['hjtrs', 'kud'])]
+
+    feeds          = [(u'Mode', u'http://www.brigitte.de/mode/feed.rss'),
+                          (u'Beauty', u'http://www.brigitte.de/beauty/feed.rss'),
+                          (u'Luxus', u'http://www.brigitte.de/luxus/feed.rss'),
+                          (u'Figur', u'http://www.brigitte.de/figur/feed.rss'),
+                          (u'Gesundheit', u'http://www.brigitte.de/gesundheit/feed.rss'),
+                          (u'Liebe&Sex', u'http://www.brigitte.de/liebe-sex/feed.rss'),
+                          (u'Gesellschaft', u'http://www.brigitte.de/gesellschaft/feed.rss'),
+                          (u'Kultur', u'http://www.brigitte.de/kultur/feed.rss'),
+                          (u'Reise', u'http://www.brigitte.de/reise/feed.rss'),
+                          (u'Kochen', u'http://www.brigitte.de/kochen/feed.rss'),
+                          (u'Wohnen', u'http://www.brigitte.de/wohnen/feed.rss'),
+                          (u'Job', u'http://www.brigitte.de/job/feed.rss'),
+                          (u'Erfahrungen', u'http://www.brigitte.de/erfahrungen/feed.rss'),
+]

recipes/daily_mirror.recipe

@@ -0,0 +1,52 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1306061239(BasicNewsRecipe):
+    title          = u'The Daily Mirror'
+    description = 'News as provide by The Daily Mirror -UK'
+
+    __author__ = 'Dave Asbury'
+    language = 'en_GB'
+
+    cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg'
+
+    masthead_url = 'http://www.nmauk.co.uk/nma/images/daily_mirror.gif'
+
+
+    oldest_article = 1
+    max_articles_per_feed = 100
+    remove_empty_feeds = True
+    remove_javascript     = True
+    no_stylesheets = True
+
+    keep_only_tags = [
+    dict(name='h1'),
+    dict(attrs={'class':['article-attr']}),
+    dict(name='div', attrs={'class' : [ 'article-body', 'crosshead']})
+
+
+    ]
+
+    remove_tags = [
+           dict(name='div', attrs={'class' : ['caption', 'article-resize']}),
+           dict( attrs={'class':'append-html'})
+           ]
+
+
+
+
+    feeds          = [
+
+        (u'News', u'http://www.mirror.co.uk/news/rss.xml')
+        ,(u'Tech News', u'http://www.mirror.co.uk/news/technology/rss.xml')
+        ,(u'Weird World','http://www.mirror.co.uk/news/weird-world/rss.xml')
+        ,(u'Film Gossip','http://www.mirror.co.uk/celebs/film/rss.xml')
+        ,(u'Music News','http://www.mirror.co.uk/celebs/music/rss.xml')
+        ,(u'Celebs and Tv Gossip','http://www.mirror.co.uk/celebs/tv/rss.xml')
+        ,(u'Sport','http://www.mirror.co.uk/sport/rss.xml')
+         ,(u'Life Style','http://www.mirror.co.uk/life-style/rss.xml')
+         ,(u'Advice','http://www.mirror.co.uk/advice/rss.xml')
+             ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml')
+
+           # example of commented out feed not needed ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml')
+  ]
+

recipes/express_de.recipe

@@ -1,5 +1,4 @@
 from calibre.web.feeds.news import BasicNewsRecipe
-
 class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
     title          = u'Express.de'
@@ -12,7 +11,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
     extra_css = '''
                     h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small;}
                     h1{ font-family:Arial,Helvetica,sans-serif;  font-size:x-large; font-weight:bold;}
-
                 '''
     remove_javascript      = True
     remove_tags_befor = [dict(name='div', attrs={'class':'Datum'})]
@@ -25,6 +23,7 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
            dict(id='Logo'),
            dict(id='MainLinkSpacer'),
            dict(id='MainLinks'),
+           dict(id='ContainerPfad'), #neu
            dict(title='Diese Seite Bookmarken'),
 
                                            dict(name='span'),
@@ -44,7 +43,8 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
            dict(name='div', attrs={'class':'HeaderSearch'}),
            dict(name='div', attrs={'class':'sbutton'}),
            dict(name='div', attrs={'class':'active'}),
-
+           dict(name='div', attrs={'class':'MoreNews'}), #neu
+           dict(name='div', attrs={'class':'ContentBoxSubline'}) #neu
 ]
 
 
@@ -68,7 +68,5 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
                           (u'Fortuna D~Dorf', u'http://www.express.de/sport/fussball/fortuna/-/3292/3292/-/view/asFeed/-/index.xml'),
                           (u'Basketball News', u'http://www.express.de/sport/basketball/-/3190/3190/-/view/asFeed/-/index.xml'),
                           (u'Big Brother', u'http://www.express.de/news/promi-show/big-brother/-/2402/2402/-/view/asFeed/-/index.xml'),
-
-
-
                     ]
+

recipes/go_comics.recipe

@@ -11,8 +11,8 @@ import mechanize, re
 class GoComics(BasicNewsRecipe):
     title               = 'GoComics'
     __author__          = 'Starson17'
-    __version__         = '1.05'
+    __version__         = '1.06'
-    __date__            = '19 may 2011'
+    __date__            = '07 June 2011'
     description         = u'200+ Comics - Customize for more days/comics: Defaults to 7 days, 25 comics - 20 general, 5 editorial.'
     category            = 'news, comics'
     language            = 'en'
@@ -56,16 +56,16 @@ class GoComics(BasicNewsRecipe):
     def parse_index(self):
         feeds = []
         for title, url in [
-                            ######## COMICS - GENERAL ########
                        (u"2 Cows and a Chicken", u"http://www.gocomics.com/2cowsandachicken"),
-                            # (u"9 to 5", u"http://www.gocomics.com/9to5"),
+                       #(u"9 Chickweed Lane", u"http://www.gocomics.com/9chickweedlane"),
-                            # (u"The Academia Waltz", u"http://www.gocomics.com/academiawaltz"),
+                       (u"9 to 5", u"http://www.gocomics.com/9to5"),
-                            # (u"Adam@Home", u"http://www.gocomics.com/adamathome"),
+                       #(u"Adam At Home", u"http://www.gocomics.com/adamathome"),
-                            # (u"Agnes", u"http://www.gocomics.com/agnes"),
+                       (u"Agnes", u"http://www.gocomics.com/agnes"),
+                       #(u"Alley Oop", u"http://www.gocomics.com/alleyoop"),
                        #(u"Andy Capp", u"http://www.gocomics.com/andycapp"),
                        #(u"Animal Crackers", u"http://www.gocomics.com/animalcrackers"),
                        #(u"Annie", u"http://www.gocomics.com/annie"),
-                            (u"The Argyle Sweater", u"http://www.gocomics.com/theargylesweater"),
+                       #(u"Arlo & Janis", u"http://www.gocomics.com/arloandjanis"),
                        #(u"Ask Shagg", u"http://www.gocomics.com/askshagg"),
                        (u"B.C.", u"http://www.gocomics.com/bc"),
                        #(u"Back in the Day", u"http://www.gocomics.com/backintheday"),
@@ -73,73 +73,82 @@ class GoComics(BasicNewsRecipe):
                        #(u"Baldo", u"http://www.gocomics.com/baldo"),
                        #(u"Ballard Street", u"http://www.gocomics.com/ballardstreet"),
                        #(u"Barkeater Lake", u"http://www.gocomics.com/barkeaterlake"),
-                            # (u"The Barn", u"http://www.gocomics.com/thebarn"),
                        #(u"Basic Instructions", u"http://www.gocomics.com/basicinstructions"),
+                       #(u"Ben", u"http://www.gocomics.com/ben"),
+                       #(u"Betty", u"http://www.gocomics.com/betty"),
                        #(u"Bewley", u"http://www.gocomics.com/bewley"),
+                       #(u"Big Nate", u"http://www.gocomics.com/bignate"),
                        #(u"Big Top", u"http://www.gocomics.com/bigtop"),
                        #(u"Biographic", u"http://www.gocomics.com/biographic"),
-                            (u"Birdbrains", u"http://www.gocomics.com/birdbrains"),
+                       #(u"Birdbrains", u"http://www.gocomics.com/birdbrains"),
                        #(u"Bleeker: The Rechargeable Dog", u"http://www.gocomics.com/bleeker"),
                        #(u"Bliss", u"http://www.gocomics.com/bliss"),
                        (u"Bloom County", u"http://www.gocomics.com/bloomcounty"),
                        #(u"Bo Nanas", u"http://www.gocomics.com/bonanas"),
                        #(u"Bob the Squirrel", u"http://www.gocomics.com/bobthesquirrel"),
-                            # (u"The Boiling Point", u"http://www.gocomics.com/theboilingpoint"),
                        #(u"Boomerangs", u"http://www.gocomics.com/boomerangs"),
-                            # (u"The Boondocks", u"http://www.gocomics.com/boondocks"),
                        #(u"Bottomliners", u"http://www.gocomics.com/bottomliners"),
                        #(u"Bound and Gagged", u"http://www.gocomics.com/boundandgagged"),
                        #(u"Brainwaves", u"http://www.gocomics.com/brainwaves"),
                        #(u"Brenda Starr", u"http://www.gocomics.com/brendastarr"),
+                       #(u"Brevity", u"http://www.gocomics.com/brevity"),
                        #(u"Brewster Rockit", u"http://www.gocomics.com/brewsterrockit"),
                        #(u"Broom Hilda", u"http://www.gocomics.com/broomhilda"),
                        (u"Calvin and Hobbes", u"http://www.gocomics.com/calvinandhobbes"),
                        #(u"Candorville", u"http://www.gocomics.com/candorville"),
                        #(u"Cathy", u"http://www.gocomics.com/cathy"),
                        #(u"C'est la Vie", u"http://www.gocomics.com/cestlavie"),
+                       #(u"Cheap Thrills", u"http://www.gocomics.com/cheapthrills"),
                        #(u"Chuckle Bros", u"http://www.gocomics.com/chucklebros"),
                        #(u"Citizen Dog", u"http://www.gocomics.com/citizendog"),
-                            # (u"The City", u"http://www.gocomics.com/thecity"),
                        #(u"Cleats", u"http://www.gocomics.com/cleats"),
                        #(u"Close to Home", u"http://www.gocomics.com/closetohome"),
+                       #(u"Committed", u"http://www.gocomics.com/committed"),
                        #(u"Compu-toon", u"http://www.gocomics.com/compu-toon"),
                        #(u"Cornered", u"http://www.gocomics.com/cornered"),
-                            (u"Cul de Sac", u"http://www.gocomics.com/culdesac"),
+                       #(u"Cow & Boy", u"http://www.gocomics.com/cow&boy"),
+                       #(u"Cul de Sac", u"http://www.gocomics.com/culdesac"),
                        #(u"Daddy's Home", u"http://www.gocomics.com/daddyshome"),
                        #(u"Deep Cover", u"http://www.gocomics.com/deepcover"),
                        #(u"Dick Tracy", u"http://www.gocomics.com/dicktracy"),
-                            # (u"The Dinette Set", u"http://www.gocomics.com/dinetteset"),
+                       (u"Dog Eat Doug", u"http://www.gocomics.com/dogeatdoug"),
-                            # (u"Dog Eat Doug", u"http://www.gocomics.com/dogeatdoug"),
                        #(u"Domestic Abuse", u"http://www.gocomics.com/domesticabuse"),
-                            # (u"Doodles", u"http://www.gocomics.com/doodles"),
+                       (u"Doodles", u"http://www.gocomics.com/doodles"),
                        (u"Doonesbury", u"http://www.gocomics.com/doonesbury"),
-                            # (u"The Doozies", u"http://www.gocomics.com/thedoozies"),
+                       #(u"Drabble", u"http://www.gocomics.com/drabble"),
-                            # (u"The Duplex", u"http://www.gocomics.com/duplex"),
                        #(u"Eek!", u"http://www.gocomics.com/eek"),
-                            # (u"The Elderberries", u"http://www.gocomics.com/theelderberries"),
+                       #(u"F Minus", u"http://www.gocomics.com/fminus"),
+                       #(u"Family Tree", u"http://www.gocomics.com/familytree"),
+                       #(u"Farcus", u"http://www.gocomics.com/farcus"),
+                       (u"Fat Cats Classics", u"http://www.gocomics.com/fatcatsclassics"),
+                       #(u"Ferd'nand", u"http://www.gocomics.com/ferdnand"),
                        #(u"Flight Deck", u"http://www.gocomics.com/flightdeck"),
-                            # (u"Flo and Friends", u"http://www.gocomics.com/floandfriends"),
+                       (u"Flo and Friends", u"http://www.gocomics.com/floandfriends"),
-                            # (u"The Flying McCoys", u"http://www.gocomics.com/theflyingmccoys"),
+                       #(u"For Better or For Worse", u"http://www.gocomics.com/forbetterorforworse"),
-                            (u"For Better or For Worse", u"http://www.gocomics.com/forbetterorforworse"),
                        #(u"For Heaven's Sake", u"http://www.gocomics.com/forheavenssake"),
                        #(u"Fort Knox", u"http://www.gocomics.com/fortknox"),
-                            # (u"FoxTrot", u"http://www.gocomics.com/foxtrot"),
+                       #(u"FoxTrot Classics", u"http://www.gocomics.com/foxtrotclassics"),
-                            (u"FoxTrot Classics", u"http://www.gocomics.com/foxtrotclassics"),
+                       (u"FoxTrot", u"http://www.gocomics.com/foxtrot"),
                        #(u"Frank & Ernest", u"http://www.gocomics.com/frankandernest"),
+                       #(u"Frazz", u"http://www.gocomics.com/frazz"),
                        #(u"Fred Basset", u"http://www.gocomics.com/fredbasset"),
                        #(u"Free Range", u"http://www.gocomics.com/freerange"),
                        #(u"Frog Applause", u"http://www.gocomics.com/frogapplause"),
-                            # (u"The Fusco Brothers", u"http://www.gocomics.com/thefuscobrothers"),
-                            (u"Garfield", u"http://www.gocomics.com/garfield"),
                        #(u"Garfield Minus Garfield", u"http://www.gocomics.com/garfieldminusgarfield"),
+                       (u"Garfield", u"http://www.gocomics.com/garfield"),
                        #(u"Gasoline Alley", u"http://www.gocomics.com/gasolinealley"),
+                       #(u"Geech Classics", u"http://www.gocomics.com/geechclassics"),
+                       #(u"Get Fuzzy", u"http://www.gocomics.com/getfuzzy"),
                        #(u"Gil Thorp", u"http://www.gocomics.com/gilthorp"),
                        #(u"Ginger Meggs", u"http://www.gocomics.com/gingermeggs"),
                        #(u"Girls & Sports", u"http://www.gocomics.com/girlsandsports"),
+                       #(u"Graffiti", u"http://www.gocomics.com/graffiti"),
+                       #(u"Grand Avenue", u"http://www.gocomics.com/grandavenue"),
                        #(u"Haiku Ewe", u"http://www.gocomics.com/haikuewe"),
                        #(u"Heart of the City", u"http://www.gocomics.com/heartofthecity"),
-                            # (u"Heathcliff", u"http://www.gocomics.com/heathcliff"),
+                       (u"Heathcliff", u"http://www.gocomics.com/heathcliff"),
                        #(u"Herb and Jamaal", u"http://www.gocomics.com/herbandjamaal"),
+                       #(u"Herman", u"http://www.gocomics.com/herman"),
                        #(u"Home and Away", u"http://www.gocomics.com/homeandaway"),
                        #(u"Housebroken", u"http://www.gocomics.com/housebroken"),
                        #(u"Hubert and Abby", u"http://www.gocomics.com/hubertandabby"),
@@ -148,133 +157,217 @@ class GoComics(BasicNewsRecipe):
                        #(u"In the Sticks", u"http://www.gocomics.com/inthesticks"),
                        #(u"Ink Pen", u"http://www.gocomics.com/inkpen"),
                        #(u"It's All About You", u"http://www.gocomics.com/itsallaboutyou"),
+                       #(u"Jane's World", u"http://www.gocomics.com/janesworld"),
                        #(u"Joe Vanilla", u"http://www.gocomics.com/joevanilla"),
+                       #(u"Jump Start", u"http://www.gocomics.com/jumpstart"),
+                       #(u"Kit 'N' Carlyle", u"http://www.gocomics.com/kitandcarlyle"),
                        #(u"La Cucaracha", u"http://www.gocomics.com/lacucaracha"),
                        #(u"Last Kiss", u"http://www.gocomics.com/lastkiss"),
                        #(u"Legend of Bill", u"http://www.gocomics.com/legendofbill"),
                        #(u"Liberty Meadows", u"http://www.gocomics.com/libertymeadows"),
-                            (u"Lio", u"http://www.gocomics.com/lio"),
+                       #(u"Li'l Abner Classics", u"http://www.gocomics.com/lilabnerclassics"),
+                       #(u"Lio", u"http://www.gocomics.com/lio"),
                        #(u"Little Dog Lost", u"http://www.gocomics.com/littledoglost"),
                        #(u"Little Otto", u"http://www.gocomics.com/littleotto"),
+                       #(u"Lola", u"http://www.gocomics.com/lola"),
                        #(u"Loose Parts", u"http://www.gocomics.com/looseparts"),
                        #(u"Love Is...", u"http://www.gocomics.com/loveis"),
+                       #(u"Luann", u"http://www.gocomics.com/luann"),
                        #(u"Maintaining", u"http://www.gocomics.com/maintaining"),
-                            # (u"The Meaning of Lila", u"http://www.gocomics.com/meaningoflila"),
+                       (u"Marmaduke", u"http://www.gocomics.com/marmaduke"),
+                       #(u"Meg! Classics", u"http://www.gocomics.com/megclassics"),
                        #(u"Middle-Aged White Guy", u"http://www.gocomics.com/middleagedwhiteguy"),
-                            # (u"The Middletons", u"http://www.gocomics.com/themiddletons"),
+                       #(u"Minimum Security", u"http://www.gocomics.com/minimumsecurity"),
-                            # (u"Momma", u"http://www.gocomics.com/momma"),
+                       #(u"Moderately Confused", u"http://www.gocomics.com/moderatelyconfused"),
-                            # (u"Mutt & Jeff", u"http://www.gocomics.com/muttandjeff"),
+                       (u"Momma", u"http://www.gocomics.com/momma"),
+                       #(u"Monty", u"http://www.gocomics.com/monty"),
+                       #(u"Motley Classics", u"http://www.gocomics.com/motleyclassics"),
+                       (u"Mutt & Jeff", u"http://www.gocomics.com/muttandjeff"),
                        #(u"Mythtickle", u"http://www.gocomics.com/mythtickle"),
+                       #(u"Nancy", u"http://www.gocomics.com/nancy"),
+                       #(u"Natural Selection", u"http://www.gocomics.com/naturalselection"),
                        #(u"Nest Heads", u"http://www.gocomics.com/nestheads"),
                        #(u"NEUROTICA", u"http://www.gocomics.com/neurotica"),
-                            (u"New Adventures of Queen Victoria", u"http://www.gocomics.com/thenewadventuresofqueenvictoria"),
+                       #(u"New Adventures of Queen Victoria", u"http://www.gocomics.com/thenewadventuresofqueenvictoria"),
-                            (u"Non Sequitur", u"http://www.gocomics.com/nonsequitur"),
+                       #(u"Non Sequitur", u"http://www.gocomics.com/nonsequitur"),
-                            # (u"The Norm", u"http://www.gocomics.com/thenorm"),
+                       #(u"Off The Mark", u"http://www.gocomics.com/offthemark"),
                        #(u"On A Claire Day", u"http://www.gocomics.com/onaclaireday"),
+                       #(u"One Big Happy Classics", u"http://www.gocomics.com/onebighappyclassics"),
                        #(u"One Big Happy", u"http://www.gocomics.com/onebighappy"),
-                            # (u"The Other Coast", u"http://www.gocomics.com/theothercoast"),
                        #(u"Out of the Gene Pool Re-Runs", u"http://www.gocomics.com/outofthegenepool"),
+                       #(u"Over the Hedge", u"http://www.gocomics.com/overthehedge"),
                        #(u"Overboard", u"http://www.gocomics.com/overboard"),
-                            # (u"Pibgorn", u"http://www.gocomics.com/pibgorn"),
+                       #(u"PC and Pixel", u"http://www.gocomics.com/pcandpixel"),
+                       (u"Peanuts", u"http://www.gocomics.com/peanuts"),
+                       #(u"Pearls Before Swine", u"http://www.gocomics.com/pearlsbeforeswine"),
                        #(u"Pibgorn Sketches", u"http://www.gocomics.com/pibgornsketches"),
+                       #(u"Pibgorn", u"http://www.gocomics.com/pibgorn"),
                        (u"Pickles", u"http://www.gocomics.com/pickles"),
                        #(u"Pinkerton", u"http://www.gocomics.com/pinkerton"),
                        #(u"Pluggers", u"http://www.gocomics.com/pluggers"),
-                            (u"Pooch Cafe", u"http://www.gocomics.com/poochcafe"),
+                       #(u"Pooch Cafe", u"http://www.gocomics.com/poochcafe"),
                        #(u"PreTeena", u"http://www.gocomics.com/preteena"),
-                            # (u"The Quigmans", u"http://www.gocomics.com/thequigmans"),
+                       #(u"Prickly City", u"http://www.gocomics.com/pricklycity"),
                        #(u"Rabbits Against Magic", u"http://www.gocomics.com/rabbitsagainstmagic"),
-                            (u"Real Life Adventures", u"http://www.gocomics.com/reallifeadventures"),
+                       #(u"Raising Duncan Classics", u"http://www.gocomics.com/raisingduncanclassics"),
+                       #(u"Real Life Adventures", u"http://www.gocomics.com/reallifeadventures"),
+                       #(u"Reality Check", u"http://www.gocomics.com/realitycheck"),
                        #(u"Red and Rover", u"http://www.gocomics.com/redandrover"),
                        #(u"Red Meat", u"http://www.gocomics.com/redmeat"),
                        #(u"Reynolds Unwrapped", u"http://www.gocomics.com/reynoldsunwrapped"),
+                       #(u"Rip Haywire", u"http://www.gocomics.com/riphaywire"),
+                       #(u"Ripley's Believe It or Not!", u"http://www.gocomics.com/ripleysbelieveitornot"),
                        #(u"Ronaldinho Gaucho", u"http://www.gocomics.com/ronaldinhogaucho"),
+                       #(u"Rose Is Rose", u"http://www.gocomics.com/roseisrose"),
                        #(u"Rubes", u"http://www.gocomics.com/rubes"),
+                       #(u"Rudy Park", u"http://www.gocomics.com/rudypark"),
                        #(u"Scary Gary", u"http://www.gocomics.com/scarygary"),
-                            (u"Shoe", u"http://www.gocomics.com/shoe"),
+                       #(u"Shirley and Son Classics", u"http://www.gocomics.com/shirleyandsonclassics"),
+                       #(u"Shoe", u"http://www.gocomics.com/shoe"),
                        #(u"Shoecabbage", u"http://www.gocomics.com/shoecabbage"),
                        #(u"Skin Horse", u"http://www.gocomics.com/skinhorse"),
                        #(u"Slowpoke", u"http://www.gocomics.com/slowpoke"),
+                       #(u"Soup To Nutz", u"http://www.gocomics.com/souptonutz"),
                        #(u"Speed Bump", u"http://www.gocomics.com/speedbump"),
+                       #(u"Spot The Frog", u"http://www.gocomics.com/spotthefrog"),
                        #(u"State of the Union", u"http://www.gocomics.com/stateoftheunion"),
-                            (u"Stone Soup", u"http://www.gocomics.com/stonesoup"),
+                       #(u"Stone Soup", u"http://www.gocomics.com/stonesoup"),
                        #(u"Strange Brew", u"http://www.gocomics.com/strangebrew"),
                        #(u"Sylvia", u"http://www.gocomics.com/sylvia"),
                        #(u"Tank McNamara", u"http://www.gocomics.com/tankmcnamara"),
-                            # (u"Tiny Sepuku", u"http://www.gocomics.com/tinysepuku"),
+                       #(u"Tarzan Classics", u"http://www.gocomics.com/tarzanclassics"),
+                       #(u"That's Life", u"http://www.gocomics.com/thatslife"),
+                       #(u"The Academia Waltz", u"http://www.gocomics.com/academiawaltz"),
+                       #(u"The Argyle Sweater", u"http://www.gocomics.com/theargylesweater"),
+                       #(u"The Barn", u"http://www.gocomics.com/thebarn"),
+                       #(u"The Boiling Point", u"http://www.gocomics.com/theboilingpoint"),
+                       #(u"The Boondocks", u"http://www.gocomics.com/boondocks"),
+                       #(u"The Born Loser", u"http://www.gocomics.com/thebornloser"),
+                       #(u"The Buckets", u"http://www.gocomics.com/thebuckets"),
+                       #(u"The City", u"http://www.gocomics.com/thecity"),
+                       #(u"The Dinette Set", u"http://www.gocomics.com/dinetteset"),
+                       #(u"The Doozies", u"http://www.gocomics.com/thedoozies"),
+                       #(u"The Duplex", u"http://www.gocomics.com/duplex"),
+                       #(u"The Elderberries", u"http://www.gocomics.com/theelderberries"),
+                       #(u"The Flying McCoys", u"http://www.gocomics.com/theflyingmccoys"),
+                       #(u"The Fusco Brothers", u"http://www.gocomics.com/thefuscobrothers"),
+                       #(u"The Grizzwells", u"http://www.gocomics.com/thegrizzwells"),
+                       #(u"The Humble Stumble", u"http://www.gocomics.com/thehumblestumble"),
+                       #(u"The Knight Life", u"http://www.gocomics.com/theknightlife"),
+                       #(u"The Meaning of Lila", u"http://www.gocomics.com/meaningoflila"),
+                       #(u"The Middletons", u"http://www.gocomics.com/themiddletons"),
+                       #(u"The Norm", u"http://www.gocomics.com/thenorm"),
+                       #(u"The Other Coast", u"http://www.gocomics.com/theothercoast"),
+                       #(u"The Quigmans", u"http://www.gocomics.com/thequigmans"),
+                       #(u"The Sunshine Club", u"http://www.gocomics.com/thesunshineclub"),
+                       #(u"Tiny Sepuk", u"http://www.gocomics.com/tinysepuk"),
                        #(u"TOBY", u"http://www.gocomics.com/toby"),
                        #(u"Tom the Dancing Bug", u"http://www.gocomics.com/tomthedancingbug"),
                        #(u"Too Much Coffee Man", u"http://www.gocomics.com/toomuchcoffeeman"),
+                       #(u"Unstrange Phenomena", u"http://www.gocomics.com/unstrangephenomena"),
                        #(u"W.T. Duck", u"http://www.gocomics.com/wtduck"),
                        #(u"Watch Your Head", u"http://www.gocomics.com/watchyourhead"),
                        #(u"Wee Pals", u"http://www.gocomics.com/weepals"),
                        #(u"Winnie the Pooh", u"http://www.gocomics.com/winniethepooh"),
-                            (u"Wizard of Id", u"http://www.gocomics.com/wizardofid"),
+                       #(u"Wizard of Id", u"http://www.gocomics.com/wizardofid"),
+                       #(u"Working Daze", u"http://www.gocomics.com/workingdaze"),
                        #(u"Working It Out", u"http://www.gocomics.com/workingitout"),
                        #(u"Yenny", u"http://www.gocomics.com/yenny"),
                        #(u"Zack Hill", u"http://www.gocomics.com/zackhill"),
                        (u"Ziggy", u"http://www.gocomics.com/ziggy"),
-                            ######## COMICS - EDITORIAL ########
+                       #
-                            ("Lalo Alcaraz","http://www.gocomics.com/laloalcaraz"),
+                       ######## EDITORIAL CARTOONS #####################
-                            ("Nick Anderson","http://www.gocomics.com/nickanderson"),
+                       (u"Adam Zyglis", u"http://www.gocomics.com/adamzyglis"),
-                            ("Chuck Asay","http://www.gocomics.com/chuckasay"),
+                       #(u"Andy Singer", u"http://www.gocomics.com/andysinger"),
-                            ("Tony Auth","http://www.gocomics.com/tonyauth"),
+                       #(u"Ben Sargent",u"http://www.gocomics.com/bensargent"),
-                            ("Donna Barstow","http://www.gocomics.com/donnabarstow"),
+                       #(u"Bill Day", u"http://www.gocomics.com/billday"),
-                            # ("Bruce Beattie","http://www.gocomics.com/brucebeattie"),
+                       #(u"Bill Schorr", u"http://www.gocomics.com/billschorr"),
-                            # ("Clay Bennett","http://www.gocomics.com/claybennett"),
+                       #(u"Bob Englehart", u"http://www.gocomics.com/bobenglehart"),
-                            # ("Lisa Benson","http://www.gocomics.com/lisabenson"),
+                       (u"Bob Gorrell",u"http://www.gocomics.com/bobgorrell"),
-                            # ("Steve Benson","http://www.gocomics.com/stevebenson"),
+                       #(u"Brian Fairrington", u"http://www.gocomics.com/brianfairrington"),
-                            # ("Chip Bok","http://www.gocomics.com/chipbok"),
+                       #(u"Bruce Beattie", u"http://www.gocomics.com/brucebeattie"),
-                            # ("Steve Breen","http://www.gocomics.com/stevebreen"),
+                       #(u"Cam Cardow", u"http://www.gocomics.com/camcardow"),
-                            # ("Chris Britt","http://www.gocomics.com/chrisbritt"),
+                       #(u"Chan Lowe",u"http://www.gocomics.com/chanlowe"),
-                            # ("Stuart Carlson","http://www.gocomics.com/stuartcarlson"),
+                       #(u"Chip Bok",u"http://www.gocomics.com/chipbok"),
-                            # ("Ken Catalino","http://www.gocomics.com/kencatalino"),
+                       #(u"Chris Britt",u"http://www.gocomics.com/chrisbritt"),
-                            # ("Paul Conrad","http://www.gocomics.com/paulconrad"),
+                       #(u"Chuck Asay",u"http://www.gocomics.com/chuckasay"),
-                            # ("Jeff Danziger","http://www.gocomics.com/jeffdanziger"),
+                       #(u"Clay Bennett",u"http://www.gocomics.com/claybennett"),
-                            # ("Matt Davies","http://www.gocomics.com/mattdavies"),
+                       #(u"Clay Jones",u"http://www.gocomics.com/clayjones"),
-                            # ("John Deering","http://www.gocomics.com/johndeering"),
+                       #(u"Dan Wasserman",u"http://www.gocomics.com/danwasserman"),
-                            # ("Bob Gorrell","http://www.gocomics.com/bobgorrell"),
+                       #(u"Dana Summers",u"http://www.gocomics.com/danasummers"),
-                            # ("Walt Handelsman","http://www.gocomics.com/walthandelsman"),
+                       #(u"Daryl Cagle", u"http://www.gocomics.com/darylcagle"),
-                            # ("Clay Jones","http://www.gocomics.com/clayjones"),
+                       #(u"David Fitzsimmons", u"http://www.gocomics.com/davidfitzsimmons"),
-                            # ("Kevin Kallaugher","http://www.gocomics.com/kevinkallaugher"),
+                       (u"Dick Locher",u"http://www.gocomics.com/dicklocher"),
-                            # ("Steve Kelley","http://www.gocomics.com/stevekelley"),
+                       #(u"Don Wright",u"http://www.gocomics.com/donwright"),
-                            # ("Dick Locher","http://www.gocomics.com/dicklocher"),
+                       #(u"Donna Barstow",u"http://www.gocomics.com/donnabarstow"),
-                            # ("Chan Lowe","http://www.gocomics.com/chanlowe"),
+                       #(u"Drew Litton", u"http://www.gocomics.com/drewlitton"),
-                            # ("Mike Luckovich","http://www.gocomics.com/mikeluckovich"),
+                       #(u"Drew Sheneman",u"http://www.gocomics.com/drewsheneman"),
-                            # ("Gary Markstein","http://www.gocomics.com/garymarkstein"),
+                       #(u"Ed Stein", u"http://www.gocomics.com/edstein"),
-                            # ("Glenn McCoy","http://www.gocomics.com/glennmccoy"),
+                       #(u"Eric Allie", u"http://www.gocomics.com/ericallie"),
-                            # ("Jim Morin","http://www.gocomics.com/jimmorin"),
+                       #(u"Gary Markstein", u"http://www.gocomics.com/garymarkstein"),
-                            # ("Jack Ohman","http://www.gocomics.com/jackohman"),
+                       #(u"Gary McCoy", u"http://www.gocomics.com/garymccoy"),
-                            # ("Pat Oliphant","http://www.gocomics.com/patoliphant"),
+                       #(u"Gary Varvel", u"http://www.gocomics.com/garyvarvel"),
-                            # ("Joel Pett","http://www.gocomics.com/joelpett"),
+                       #(u"Glenn McCoy",u"http://www.gocomics.com/glennmccoy"),
-                            # ("Ted Rall","http://www.gocomics.com/tedrall"),
+                       #(u"Henry Payne", u"http://www.gocomics.com/henrypayne"),
-                            # ("Michael Ramirez","http://www.gocomics.com/michaelramirez"),
+                       #(u"Jack Ohman",u"http://www.gocomics.com/jackohman"),
-                            # ("Marshall Ramsey","http://www.gocomics.com/marshallramsey"),
+                       #(u"JD Crowe", u"http://www.gocomics.com/jdcrowe"),
-                            # ("Steve Sack","http://www.gocomics.com/stevesack"),
+                       #(u"Jeff Danziger",u"http://www.gocomics.com/jeffdanziger"),
-                            # ("Ben Sargent","http://www.gocomics.com/bensargent"),
+                       #(u"Jeff Parker", u"http://www.gocomics.com/jeffparker"),
-                            # ("Drew Sheneman","http://www.gocomics.com/drewsheneman"),
+                       #(u"Jeff Stahler", u"http://www.gocomics.com/jeffstahler"),
-                            # ("John Sherffius","http://www.gocomics.com/johnsherffius"),
+                       #(u"Jerry Holbert", u"http://www.gocomics.com/jerryholbert"),
-                            # ("Small World","http://www.gocomics.com/smallworld"),
+                       #(u"Jim Morin",u"http://www.gocomics.com/jimmorin"),
-                            # ("Scott Stantis","http://www.gocomics.com/scottstantis"),
+                       #(u"Joel Pett",u"http://www.gocomics.com/joelpett"),
-                            # ("Wayne Stayskal","http://www.gocomics.com/waynestayskal"),
+                       #(u"John Cole", u"http://www.gocomics.com/johncole"),
-                            # ("Dana Summers","http://www.gocomics.com/danasummers"),
+                       #(u"John Darkow", u"http://www.gocomics.com/johndarkow"),
-                            # ("Paul Szep","http://www.gocomics.com/paulszep"),
+                       #(u"John Deering",u"http://www.gocomics.com/johndeering"),
-                            # ("Mike Thompson","http://www.gocomics.com/mikethompson"),
+                       #(u"John Sherffius", u"http://www.gocomics.com/johnsherffius"),
-                            # ("Tom Toles","http://www.gocomics.com/tomtoles"),
+                       #(u"Ken Catalino",u"http://www.gocomics.com/kencatalino"),
-                            # ("Gary Varvel","http://www.gocomics.com/garyvarvel"),
+                       #(u"Kerry Waghorn",u"http://www.gocomics.com/facesinthenews"),
-                            # ("ViewsAfrica","http://www.gocomics.com/viewsafrica"),
+                       #(u"Kevin Kallaugher",u"http://www.gocomics.com/kevinkallaugher"),
-                            # ("ViewsAmerica","http://www.gocomics.com/viewsamerica"),
+                       #(u"Lalo Alcaraz",u"http://www.gocomics.com/laloalcaraz"),
-                            # ("ViewsAsia","http://www.gocomics.com/viewsasia"),
+                       #(u"Larry Wright", u"http://www.gocomics.com/larrywright"),
-                            # ("ViewsBusiness","http://www.gocomics.com/viewsbusiness"),
+                       #(u"Lisa Benson", u"http://www.gocomics.com/lisabenson"),
-                            # ("ViewsEurope","http://www.gocomics.com/viewseurope"),
+                       #(u"Marshall Ramsey", u"http://www.gocomics.com/marshallramsey"),
-                            # ("ViewsLatinAmerica","http://www.gocomics.com/viewslatinamerica"),
+                       #(u"Matt Bors", u"http://www.gocomics.com/mattbors"),
-                            # ("ViewsMidEast","http://www.gocomics.com/viewsmideast"),
+                       #(u"Matt Davies",u"http://www.gocomics.com/mattdavies"),
-                            # ("Views of the World","http://www.gocomics.com/viewsoftheworld"),
+                       #(u"Michael Ramirez", u"http://www.gocomics.com/michaelramirez"),
-                            # ("Kerry Waghorn","http://www.gocomics.com/facesinthenews"),
+                       #(u"Mike Keefe", u"http://www.gocomics.com/mikekeefe"),
-                            # ("Dan Wasserman","http://www.gocomics.com/danwasserman"),
+                       #(u"Mike Luckovich", u"http://www.gocomics.com/mikeluckovich"),
-                            # ("Signe Wilkinson","http://www.gocomics.com/signewilkinson"),
+                       #(u"MIke Thompson", u"http://www.gocomics.com/mikethompson"),
-                            # ("Wit of the World","http://www.gocomics.com/witoftheworld"),
+                       #(u"Monte Wolverton", u"http://www.gocomics.com/montewolverton"),
-                            # ("Don Wright","http://www.gocomics.com/donwright"),
+                       #(u"Mr. Fish", u"http://www.gocomics.com/mrfish"),
+                       #(u"Nate Beeler", u"http://www.gocomics.com/natebeeler"),
+                       #(u"Nick Anderson", u"http://www.gocomics.com/nickanderson"),
+                       #(u"Pat Bagley", u"http://www.gocomics.com/patbagley"),
+                       #(u"Pat Oliphant",u"http://www.gocomics.com/patoliphant"),
+                       #(u"Paul Conrad",u"http://www.gocomics.com/paulconrad"),
+                       #(u"Paul Szep", u"http://www.gocomics.com/paulszep"),
+                       #(u"RJ Matson", u"http://www.gocomics.com/rjmatson"),
+                       #(u"Rob Rogers", u"http://www.gocomics.com/robrogers"),
+                       #(u"Robert Ariail", u"http://www.gocomics.com/robertariail"),
+                       #(u"Scott Stantis", u"http://www.gocomics.com/scottstantis"),
+                       #(u"Signe Wilkinson", u"http://www.gocomics.com/signewilkinson"),
+                       #(u"Small World",u"http://www.gocomics.com/smallworld"),
+                       #(u"Steve Benson", u"http://www.gocomics.com/stevebenson"),
+                       #(u"Steve Breen", u"http://www.gocomics.com/stevebreen"),
+                       #(u"Steve Kelley", u"http://www.gocomics.com/stevekelley"),
+                       #(u"Steve Sack", u"http://www.gocomics.com/stevesack"),
+                       #(u"Stuart Carlson",u"http://www.gocomics.com/stuartcarlson"),
+                       #(u"Ted Rall",u"http://www.gocomics.com/tedrall"),
+                       #(u"(Th)ink", u"http://www.gocomics.com/think"),
+                       #(u"Tom Toles",u"http://www.gocomics.com/tomtoles"),
+                       (u"Tony Auth",u"http://www.gocomics.com/tonyauth"),
+                       #(u"Views of the World",u"http://www.gocomics.com/viewsoftheworld"),
+                       #(u"ViewsAfrica",u"http://www.gocomics.com/viewsafrica"),
+                       #(u"ViewsAmerica",u"http://www.gocomics.com/viewsamerica"),
+                       #(u"ViewsAsia",u"http://www.gocomics.com/viewsasia"),
+                       #(u"ViewsBusiness",u"http://www.gocomics.com/viewsbusiness"),
+                       #(u"ViewsEurope",u"http://www.gocomics.com/viewseurope"),
+                       #(u"ViewsLatinAmerica",u"http://www.gocomics.com/viewslatinamerica"),
+                       #(u"ViewsMidEast",u"http://www.gocomics.com/viewsmideast"),
+                       (u"Walt Handelsman",u"http://www.gocomics.com/walthandelsman"),
+                       #(u"Wayne Stayskal",u"http://www.gocomics.com/waynestayskal"),
+                       #(u"Wit of the World",u"http://www.gocomics.com/witoftheworld"),
                              ]:
             print 'Working on: ', title
             articles = self.make_links(url)
@@ -352,3 +445,4 @@ class GoComics(BasicNewsRecipe):
                     p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
                     body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
 		'''
+

recipes/guardian.recipe

@@ -87,7 +87,13 @@ class Guardian(BasicNewsRecipe):
         idx = soup.find('div', id='book-index')
         for s in idx.findAll('strong', attrs={'class':'book'}):
             a = s.find('a', href=True)
-            yield (self.tag_to_string(a), a['href'])
+            section_title = self.tag_to_string(a)
+            if not section_title in self.ignore_sections:
+                prefix = ''
+                if section_title != 'Main section':
+                    prefix = section_title + ': '
+                for subsection in s.parent.findAll('a', attrs={'class':'book-section'}):
+                    yield (prefix + self.tag_to_string(subsection), subsection['href'])
     
     def find_articles(self, url):
         soup = self.index_to_soup(url)
@@ -114,10 +120,7 @@ class Guardian(BasicNewsRecipe):
         try:
             feeds = []
             for title, href in self.find_sections():
-                if not title in self.ignore_sections:
                 feeds.append((title, list(self.find_articles(href))))
             return feeds
         except:
             raise NotImplementedError
-
-

recipes/hbr_blogs.recipe

@@ -6,7 +6,7 @@ class HBR(BasicNewsRecipe):
     title = 'Harvard Business Review Blogs'
     description = 'To subscribe go to http://hbr.harvardbusiness.org'
     needs_subscription = True
-    __author__ = 'Kovid Goyal and Sujata Raman, enhanced by BrianG'
+    __author__ = 'Kovid Goyal, enhanced by BrianG'
     language = 'en'
     no_stylesheets = True
 

recipes/heise_online.recipe

@@ -0,0 +1,52 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+    title = 'Heise-online'
+    description = 'News vom Heise-Verlag'
+    __author__ = 'schuster'
+    use_embedded_content   = False
+    language = 'de'
+    oldest_article = 2
+    max_articles_per_feed = 35
+    rescale_images = True
+    remove_empty_feeds = True
+    timeout = 5
+    no_stylesheets = True
+
+
+    remove_tags_after = dict(name ='p', attrs={'class':'editor'})
+    remove_tags = [dict(id='navi_top_container'),
+                            dict(id='navi_bottom'),
+                            dict(id='mitte_rechts'),
+                            dict(id='navigation'),
+                            dict(id='subnavi'),
+                            dict(id='social_bookmarks'),
+                            dict(id='permalink'),
+                            dict(id='content_foren'),
+                            dict(id='seiten_navi'),
+                            dict(id='adbottom'),
+                            dict(id='sitemap')]
+
+    feeds =  [
+                   ('Newsticker', 'http://www.heise.de/newsticker/heise.rdf'),
+                   ('Auto', 'http://www.heise.de/autos/rss/news.rdf'),
+                   ('Foto ', 'http://www.heise.de/foto/rss/news-atom.xml'),
+                   ('Mac&i', 'http://www.heise.de/mac-and-i/news.rdf'),
+                   ('Mobile ', 'http://www.heise.de/mobil/newsticker/heise-atom.xml'),
+                   ('Netz ', 'http://www.heise.de/netze/rss/netze-atom.xml'),
+                   ('Open ', 'http://www.heise.de/open/news/news-atom.xml'),
+                   ('Resale ', 'http://www.heise.de/resale/rss/resale.rdf'),
+                   ('Security ', 'http://www.heise.de/security/news/news-atom.xml'),
+                   ('C`t', 'http://www.heise.de/ct/rss/artikel-atom.xml'),
+                   ('iX', 'http://www.heise.de/ix/news/news.rdf'),
+                   ('Mach-flott', 'http://www.heise.de/mach-flott/rss/mach-flott-atom.xml'),
+                   ('Blog: Babel-Bulletin', 'http://www.heise.de/developer/rss/babel-bulletin/blog.rdf'),
+                   ('Blog: Der Dotnet-Doktor', 'http://www.heise.de/developer/rss/dotnet-doktor/blog.rdf'),
+                   ('Blog: Bernds Management-Welt', 'http://www.heise.de/developer/rss/bernds-management-welt/blog.rdf'),
+                   ('Blog: IT conversation', 'http://www.heise.de/developer/rss/world-of-it/blog.rdf'),
+                   ('Blog: Kais bewegtes Web', 'http://www.heise.de/developer/rss/kais-bewegtes-web/blog.rdf')
+]
+
+    def print_version(self, url):
+        return url + '?view=print'
+

recipes/lemonde_dip.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 mondediplo.com
 '''
@@ -11,7 +11,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class LeMondeDiplomatiqueEn(BasicNewsRecipe):
     title                  = 'Le Monde diplomatique - English edition'
     __author__             = 'Darko Miletic'
-    description            = 'Real journalism making sense of the world around us'
+    description            = "Le Monde diplomatique is the place you go when you want to know what's really happening. This is a major international paper that is truly independent, that sees the world in fresh ways, that focuses on places no other publications reach. We offer a clear, considered view of the conflicting interests and complexities of a modern global world. LMD in English is a concise version of the Paris-based parent edition, publishing all the major stories each month, expertly translated, and with some London-based commissions too. We offer a taster of LMD quality on our website where a selection of articles are available each month."
     publisher              = 'Le Monde diplomatique'
     category               = 'news, politics, world'
     no_stylesheets         = True
@@ -26,7 +26,13 @@ class LeMondeDiplomatiqueEn(BasicNewsRecipe):
     INDEX                  = PREFIX + strftime('%Y/%m/')
     use_embedded_content   = False
     language               = 'en'
-    extra_css              = ' body{font-family: "Luxi sans","Lucida sans","Lucida Grande",Lucida,"Lucida Sans Unicode",sans-serif} .surtitre{font-size: 1.2em; font-variant: small-caps; margin-bottom: 0.5em} .chapo{font-size: 1.2em; font-weight: bold; margin: 1em 0 0.5em} .texte{font-family: Georgia,"Times New Roman",serif} h1{color: #990000} .notes{border-top: 1px solid #CCCCCC; font-size: 0.9em; line-height: 1.4em} '
+    extra_css              = """
+                                body{font-family: "Luxi sans","Lucida sans","Lucida Grande",Lucida,"Lucida Sans Unicode",sans-serif}
+                                .surtitre{font-size: 1.2em; font-variant: small-caps; margin-bottom: 0.5em}
+                                .chapo{font-size: 1.2em; font-weight: bold; margin: 1em 0 0.5em}
+                                .texte{font-family: Georgia,"Times New Roman",serif} h1{color: #990000}
+                                .notes{border-top: 1px solid #CCCCCC; font-size: 0.9em; line-height: 1.4em}
+                            """
 
     conversion_options = {
                           'comment'   : description
@@ -51,7 +57,7 @@ class LeMondeDiplomatiqueEn(BasicNewsRecipe):
                         , dict(name='div',attrs={'class':'notes surlignable'})
                         ]
     remove_tags = [dict(name=['object','link','script','iframe','base'])]
-    remove_attributes = ['height','width']
+    remove_attributes = ['height','width','name','lang']
 
     def parse_index(self):
         articles = []
@@ -75,3 +81,24 @@ class LeMondeDiplomatiqueEn(BasicNewsRecipe):
                                 })
         return [(self.title, articles)]
 
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('div',attrs={'class':'current'})
+        if cover_item:
+           ap = cover_item.find('img',attrs={'class':'spip_logos'})
+           if ap:
+              cover_url = self.INDEX + ap['src']
+        return cover_url
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               str = self.tag_to_string(item)
+               item.replaceWith(str)
+        return soup

recipes/max_planck.recipe

@@ -3,9 +3,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
     title          = u'Max-Planck-Inst.'
     __author__  = 'schuster'
-    remove_tags = [dict(attrs={'class':['clearfix', 'lens', 'col2_box_list', 'col2_box_teaser group_ext no_print', 'dotted_line', 'col2_box_teaser', 'box_image small', 'bold', 'col2_box_teaser no_print', 'print_kontakt']}),
-                dict(id=['ie_clearing', 'col2', 'col2_content']),
-                dict(name=['script', 'noscript', 'style'])]
     oldest_article = 30
     max_articles_per_feed = 100
     no_stylesheets         = True
@@ -13,6 +10,11 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
     language               = 'de'
     remove_javascript      = True
 
+    remove_tags = [dict(attrs={'class':['box_url', 'print_kontakt']}),
+                dict(id=['skiplinks'])]
+
+
+
     def print_version(self, url):
         split_url = url.split("/")
         print_url = 'http://www.mpg.de/print/' +  split_url[3]

recipes/metro_news_nl.recipe

@@ -0,0 +1,45 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1306097511(BasicNewsRecipe):
+    title = u'Metro Nieuws NL'
+    oldest_article = 2
+    max_articles_per_feed = 100
+    __author__     = u'DrMerry'
+    description    = u'Metro Nederland'
+    language       = u'nl'
+    simultaneous_downloads = 5
+    delay          = 1
+#    timefmt        = ' [%A, %d %B, %Y]'
+    timefmt        = ''
+    no_stylesheets = True
+    remove_javascript = True
+    remove_empty_feeds = True
+    cover_url      = 'http://www.readmetro.com/img/en/metroholland/last/1/small.jpg'
+    remove_empty_feeds = True
+    publication_type = 'newspaper'
+    remove_tags_before = dict(name='div', attrs={'id':'date'})
+    remove_tags_after = dict(name='div', attrs={'id':'column-1-3'})
+    encoding              = 'utf-8'
+    extra_css = '#date {font-size: 10px} .article-image-caption {font-size: 8px}'
+
+    remove_tags = [dict(name='div', attrs={'class':[ 'metroCommentFormWrap',
+        'commentForm', 'metroCommentInnerWrap', 'article-slideshow-counter-container', 'article-slideshow-control', 'ad', 'header-links',
+        'art-rgt','pluck-app pluck-comm', 'share-and-byline', 'article-tools-below-title', 'col-179 ', 'related-links', 'clear padding-top-15', 'share-tools', 'article-page-auto-pushes', 'footer-edit']}),
+        dict(name='div', attrs={'id':['article-2', 'article-4', 'article-1', 'navigation', 'footer', 'header', 'comments', 'sidebar']}),
+        dict(name='iframe')]
+
+    feeds = [
+        (u'Binnenland', u'http://www.metronieuws.nl/rss.xml?c=1277377288-3'),
+        (u'Economie', u'http://www.metronieuws.nl/rss.xml?c=1278070988-0'),
+        (u'Den Haag', u'http://www.metronieuws.nl/rss.xml?c=1289013337-3'),
+        (u'Rotterdam', u'http://www.metronieuws.nl/rss.xml?c=1289013337-2'),
+        (u'Amsterdam', u'http://www.metronieuws.nl/rss.xml?c=1289013337-1'),
+        (u'Columns', u'http://www.metronieuws.nl/rss.xml?c=1277377288-17'),
+        (u'Entertainment', u'http://www.metronieuws.nl/rss.xml?c=1277377288-2'),
+        (u'Dot', u'http://www.metronieuws.nl/rss.xml?c=1283166782-12'),
+        (u'Familie', u'http://www.metronieuws.nl/rss.xml?c=1283166782-9'),
+        (u'Blogs', u'http://www.metronieuws.nl/rss.xml?c=1295586825-6'),
+        (u'Reizen', u'http://www.metronieuws.nl/rss.xml?c=1277377288-13'),
+        (u'Carrière', u'http://www.metronieuws.nl/rss.xml?c=1278070988-1'),
+        (u'Sport', u'http://www.metronieuws.nl/rss.xml?c=1277377288-12')
+        ]

recipes/newsweek.recipe

@@ -69,7 +69,11 @@ class Newsweek(BasicNewsRecipe):
         for section, shref in self.newsweek_sections():
             self.log('Processing section', section, shref)
             articles = []
+            try:
                 soups = [self.index_to_soup(shref)]
+            except:
+                self.log.warn('Section %s not found, skipping'%section)
+                continue
             na = soups[0].find('a', rel='next')
             if na:
                 soups.append(self.index_to_soup(self.BASE_URL+na['href']))

recipes/nme.recipe

@@ -0,0 +1,42 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1306061239(BasicNewsRecipe):
+    title          = u'New Musical Express Magazine'
+    __author__ = "scissors"
+    language = 'en'
+    remove_empty_feeds = True
+    remove_javascript     = True
+    no_stylesheets = True
+    oldest_article = 7
+    max_articles_per_feed = 100
+    cover_url = 'http://tawanda3000.files.wordpress.com/2011/02/nme-logo.jpg'
+
+    remove_tags = [
+    dict( attrs={'class':'clear_icons'}),
+        dict( attrs={'class':'share_links'}),
+                        dict( attrs={'id':'right_panel'}),
+        dict( attrs={'class':'today box'})
+
+]
+
+    keep_only_tags = [
+
+    dict(name='h1'),
+    #dict(name='h3'),
+    dict(attrs={'class' :  'BText'}),
+    dict(attrs={'class' :  'Bmore'}),
+    dict(attrs={'class' : 'bPosts'}),
+    dict(attrs={'class' :  'text'}),
+    dict(attrs={'id' :  'article_gallery'}),
+    dict(attrs={'class' :  'article_text'})
+]
+
+
+
+
+    feeds          = [
+    (u'NME News', u'http://feeds2.feedburner.com/nmecom/rss/newsxml'),
+    (u'Reviews', u'http://feeds2.feedburner.com/nme/SdML'),
+    (u'Blogs', u'http://www.nme.com/blog/index.php?blog=140&tempskin=_rss2'),
+
+    ]

recipes/polizeipress_de.recipe

@@ -0,0 +1,35 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+    title          = u'Polizeipresse - Deutschland'
+    __author__  = 'schuster'
+    description            =    'Tagesaktuelle "Polizeiberichte" aus ganz Deutschland (bis auf Ortsebene).' 'Um deinen Ort/Stadt/Kreis usw. einzubinden, gehe auf "http://www.presseportal.de/polizeipresse/" und suche im oberen "Suchfeld" nach dem Namen.'   'Oberhalb der Suchergebnisse (Folgen:) auf den üblichen link zu den RSS-Feeds klicken und den RSS-link im Rezept unter "feeds" eintragen wie üblich.'   'Die Auswahl von Orten kann vereinfacht werden wenn man den Suchbegriff wie folgt eingibt:' '"Stadt-Ort".'
+    oldest_article = 21
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    masthead_url = 'http://www.alt-heliservice.de/images/34_BPOL_Logo_4C_g_schutzbereich.jpg'
+    cover_url = 'http://berlinstadtservice.de/buerger/Bundespolizei-Logo.png'
+
+    remove_tags = [
+        dict(name='div', attrs={'id':'logo'}),
+        dict(name='div', attrs={'id':'origin'}),
+        dict(name='pre', attrs={'class':'xml_contact'})]
+
+    def print_version(self,url):
+        segments = url.split('/')
+        printURL = 'http://www.presseportal.de/print.htx?nr=' + '/'.join(segments[5:6]) + '&type=polizei'
+        return printURL
+
+    feeds          = [(u'Frimmerdorf', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-frimmersdorf&w=public_service'),
+          (u'Neurath', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-neurath&w=public_service'),
+          (u'Gustorf', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-gustorf&w=public_service'),
+          (u'Neuenhausen', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-neuenhausen&w=public_service'),
+          (u'Wevelinghoven', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-Wevelinghoven&w=public_service'),
+          (u'Grevenbroich ges.', u'http://www.presseportal.de/rss/rss2_vts.htx?q=grevenbroich&w=public_service'),
+          (u'Kreis Neuss ges.', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Rhein-Kreis+Neuss&w=public_service'),
+          ]
+

recipes/stiintasitehnica.recipe

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+stiintasitehnica.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Stiintasitehnica(BasicNewsRecipe):
+    title                 = u'\u0218tiin\u021b\u0103 \u015fi Tehnic\u0103'
+    __author__            = u'Silviu Cotoar\u0103'
+    description           = u'\u0218tiin\u021b\u0103 \u015fi Tehnic\u0103'
+    publisher             = u'\u0218tiin\u021b\u0103 \u015fi Tehnic\u0103'
+    oldest_article        = 50
+    language              = 'ro'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    category              = u'Ziare,Reviste,Stiinta,Tehnica'
+    encoding              = 'utf-8'
+    cover_url             = 'http://www.stiintasitehnica.com/images/logo.jpg'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+                            ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+                        dict(name='div', attrs={'id':'mainColumn2'})
+                     ]
+
+    remove_tags = [
+                          dict(name='span', attrs={'class':['redEar']})
+                        , dict(name='table', attrs={'class':['connect_widget_interactive_area']})
+                        , dict(name='div', attrs={'class':['panel-overlay']})
+                        , dict(name='div', attrs={'id':['pointer']})
+                        , dict(name='img', attrs={'class':['nav-next', 'nav-prev']})
+                        , dict(name='table', attrs={'class':['connect_widget_interactive_area']})
+                        , dict(name='hr', attrs={'class':['dotted']})
+                  ]
+
+    remove_tags_after = [
+                            dict(name='hr', attrs={'class':['dotted']})
+                        ]
+
+    feeds          = [
+                        (u'Feeds', u'http://www.stiintasitehnica.com/rss/stiri.xml')
+                     ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/telegraph_uk.recipe

@@ -49,6 +49,7 @@ class TelegraphUK(BasicNewsRecipe):
                          (u'UK News'        , u'http://www.telegraph.co.uk/news/uknews/rss'                                      )
                         ,(u'World News'     , u'http://www.telegraph.co.uk/news/worldnews/rss'                                   )
                         ,(u'Politics'       , u'http://www.telegraph.co.uk/news/newstopics/politics/rss'                         )
+                        ,(u'Finance'        , u'http://www.telegraph.co.uk/finance/rss'                                          )
                         ,(u'Technology News', u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologynews/rss'   )
                         ,(u'UK News'        , u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologyreviews/rss')
                         ,(u'Science News'   , u'http://www.telegraph.co.uk/scienceandtechnology/science/sciencenews/rss'         )

recipes/time_magazine.recipe

@@ -10,8 +10,8 @@ import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Time(BasicNewsRecipe):
-    recipe_disabled = ('This recipe has been disabled as TIME no longer'
+    #recipe_disabled = ('This recipe has been disabled as TIME no longer'
-            ' publish complete articles on the web.')
+    #        ' publish complete articles on the web.')
     title                 = u'Time'
     __author__            = 'Kovid Goyal and Sujata Raman'
     description           = 'Weekly magazine'

recipes/zaobao.recipe

@@ -82,7 +82,7 @@ class ZAOBAO(BasicNewsRecipe):
         return soup
 
     def parse_feeds(self):
-        self.log_debug(_('ZAOBAO overrided parse_feeds()'))
+        self.log(_('ZAOBAO overrided parse_feeds()'))
         parsed_feeds = BasicNewsRecipe.parse_feeds(self)
 
         for id, obj in enumerate(self.INDEXES):
@@ -99,7 +99,7 @@ class ZAOBAO(BasicNewsRecipe):
                     a_title = self.tag_to_string(a)
                     date = ''
                     description = ''
-                    self.log_debug(_('adding %s at %s')%(a_title,a_url))
+                    self.log(_('adding %s at %s')%(a_title,a_url))
                     articles.append({
                                     'title':a_title,
                                     'date':date,
@@ -110,23 +110,23 @@ class ZAOBAO(BasicNewsRecipe):
             pfeeds = feeds_from_index([(title, articles)], oldest_article=self.oldest_article,
                                      max_articles_per_feed=self.max_articles_per_feed)
 
-            self.log_debug(_('adding %s to feed')%(title))
+            self.log(_('adding %s to feed')%(title))
             for feed in pfeeds:
-                self.log_debug(_('adding feed: %s')%(feed.title))
+                self.log(_('adding feed: %s')%(feed.title))
                 feed.description = self.DESC_SENSE
                 parsed_feeds.append(feed)
                 for a, article in enumerate(feed):
-                    self.log_debug(_('added article %s from %s')%(article.title, article.url))
+                    self.log(_('added article %s from %s')%(article.title, article.url))
-                self.log_debug(_('added feed %s')%(feed.title))
+                self.log(_('added feed %s')%(feed.title))
 
         for i, feed in enumerate(parsed_feeds):
             # workaorund a strange problem: Somethimes the xml encoding is not apllied correctly by parse()
             weired_encoding_detected = False
             if not isinstance(feed.description, unicode) and self.encoding and feed.description:
-                self.log_debug(_('Feed %s is not encoded correctly, manually replace it')%(feed.title))
+                self.log(_('Feed %s is not encoded correctly, manually replace it')%(feed.title))
                 feed.description = feed.description.decode(self.encoding, 'replace')
             elif feed.description.find(self.DESC_SENSE) == -1 and self.encoding and feed.description:
-                self.log_debug(_('Feed %s is weired encoded, manually redo all')%(feed.title))
+                self.log(_('Feed %s is weired encoded, manually redo all')%(feed.title))
                 feed.description = feed.description.encode('cp1252', 'replace').decode(self.encoding, 'replace')
                 weired_encoding_detected = True
 
@@ -148,7 +148,7 @@ class ZAOBAO(BasicNewsRecipe):
                         article.text_summary = article.text_summary.encode('cp1252', 'replace').decode(self.encoding, 'replace')
 
                 if article.title == "Untitled article":
-                    self.log_debug(_('Removing empty article %s from %s')%(article.title, article.url))
+                    self.log(_('Removing empty article %s from %s')%(article.title, article.url))
                     # remove the article
                     feed.articles[a:a+1] = []
         return parsed_feeds

resources/content_server/browse/browse.html

@@ -20,7 +20,7 @@
     <script type="text/javascript"
         src="{prefix}/static/jquery.multiselect.min.js"></script>
 
-
+    <script type="text/javascript" src="{prefix}/static/stacktrace.js"></script>
     <script type="text/javascript" src="{prefix}/static/browse/browse.js"></script>
 
     <script type="text/javascript">

resources/content_server/browse/browse.js

@@ -129,7 +129,13 @@ function toplevel() {
 // }}}
 
 function render_error(msg) {
-    return '<div class="ui-widget"><div class="ui-state-error ui-corner-all" style="padding: 0pt 0.7em"><p><span class="ui-icon ui-icon-alert" style="float: left; margin-right: 0.3em">&nbsp;</span><strong>Error: </strong>'+msg+"</p></div></div>"
+    var st = "";
+    try {
+        var st = printStackTrace();
+        st = st.join('\n\n');
+    } catch(e) {
+    }
+    return '<div class="ui-widget"><div class="ui-state-error ui-corner-all" style="padding: 0pt 0.7em"><p><span class="ui-icon ui-icon-alert" style="float: left; margin-right: 0.3em">&nbsp;</span><strong>Error: </strong>'+msg+"<pre>"+st+"</pre></p></div></div>"
 }
 
 // Category feed {{{

resources/content_server/stacktrace.js

@@ -0,0 +1,371 @@
+// Domain Public by Eric Wendelin http://eriwen.com/ (2008)
+//                  Luke Smith http://lucassmith.name/ (2008)
+//                  Loic Dachary <loic@dachary.org> (2008)
+//                  Johan Euphrosine <proppy@aminche.com> (2008)
+//                  Oyvind Sean Kinsey http://kinsey.no/blog (2010)
+//                  Victor Homyakov <victor-homyakov@users.sourceforge.net> (2010)
+//
+// Information and discussions
+// http://jspoker.pokersource.info/skin/test-printstacktrace.html
+// http://eriwen.com/javascript/js-stack-trace/
+// http://eriwen.com/javascript/stacktrace-update/
+// http://pastie.org/253058
+//
+// guessFunctionNameFromLines comes from firebug
+//
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2007, Parakey Inc.
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above
+//   copyright notice, this list of conditions and the
+//   following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above
+//   copyright notice, this list of conditions and the
+//   following disclaimer in the documentation and/or other
+//   materials provided with the distribution.
+//
+// * Neither the name of Parakey Inc. nor the names of its
+//   contributors may be used to endorse or promote products
+//   derived from this software without specific prior
+//   written permission of Parakey Inc.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+// FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/**
+ * Main function giving a function stack trace with a forced or passed in Error
+ *
+ * @cfg {Error} e The error to create a stacktrace from (optional)
+ * @cfg {Boolean} guess If we should try to resolve the names of anonymous functions
+ * @return {Array} of Strings with functions, lines, files, and arguments where possible
+ */
+function printStackTrace(options) {
+    options = options || {guess: true};
+    var ex = options.e || null, guess = !!options.guess;
+    var p = new printStackTrace.implementation(), result = p.run(ex);
+    return (guess) ? p.guessAnonymousFunctions(result) : result;
+}
+
+printStackTrace.implementation = function() {
+};
+
+printStackTrace.implementation.prototype = {
+    run: function(ex) {
+        ex = ex || this.createException();
+        // Do not use the stored mode: different exceptions in Chrome
+        // may or may not have arguments or stack
+        var mode = this.mode(ex);
+        // Use either the stored mode, or resolve it
+        //var mode = this._mode || this.mode(ex);
+        if (mode === 'other') {
+            return this.other(arguments.callee);
+        } else {
+            return this[mode](ex);
+        }
+    },
+
+    createException: function() {
+        try {
+            this.undef();
+            return null;
+        } catch (e) {
+            return e;
+        }
+    },
+
+    /**
+     * @return {String} mode of operation for the environment in question.
+     */
+    mode: function(e) {
+        if (e['arguments'] && e.stack) {
+            return (this._mode = 'chrome');
+        } else if (e.message && typeof window !== 'undefined' && window.opera) {
+            return (this._mode = e.stacktrace ? 'opera10' : 'opera');
+        } else if (e.stack) {
+            return (this._mode = 'firefox');
+        }
+        return (this._mode = 'other');
+    },
+
+    /**
+     * Given a context, function name, and callback function, overwrite it so that it calls
+     * printStackTrace() first with a callback and then runs the rest of the body.
+     *
+     * @param {Object} context of execution (e.g. window)
+     * @param {String} functionName to instrument
+     * @param {Function} function to call with a stack trace on invocation
+     */
+    instrumentFunction: function(context, functionName, callback) {
+        context = context || window;
+        var original = context[functionName];
+        context[functionName] = function instrumented() {
+            callback.call(this, printStackTrace().slice(4));
+            return context[functionName]._instrumented.apply(this, arguments);
+        };
+        context[functionName]._instrumented = original;
+    },
+
+    /**
+     * Given a context and function name of a function that has been
+     * instrumented, revert the function to it's original (non-instrumented)
+     * state.
+     *
+     * @param {Object} context of execution (e.g. window)
+     * @param {String} functionName to de-instrument
+     */
+    deinstrumentFunction: function(context, functionName) {
+        if (context[functionName].constructor === Function &&
+                context[functionName]._instrumented &&
+                context[functionName]._instrumented.constructor === Function) {
+            context[functionName] = context[functionName]._instrumented;
+        }
+    },
+
+    /**
+     * Given an Error object, return a formatted Array based on Chrome's stack string.
+     *
+     * @param e - Error object to inspect
+     * @return Array<String> of function calls, files and line numbers
+     */
+    chrome: function(e) {
+        //return e.stack.replace(/^[^\(]+?[\n$]/gm, '').replace(/^\s+at\s+/gm, '').replace(/^Object.<anonymous>\s*\(/gm, '{anonymous}()@').split('\n');
+        return e.stack.replace(/^\S[^\(]+?[\n$]/gm, '').
+          replace(/^\s+at\s+/gm, '').
+          replace(/^([^\(]+?)([\n$])/gm, '{anonymous}()@$1$2').
+          replace(/^Object.<anonymous>\s*\(([^\)]+)\)/gm, '{anonymous}()@$1').split('\n');
+    },
+
+    /**
+     * Given an Error object, return a formatted Array based on Firefox's stack string.
+     *
+     * @param e - Error object to inspect
+     * @return Array<String> of function calls, files and line numbers
+     */
+    firefox: function(e) {
+        return e.stack.replace(/(?:\n@:0)?\s+$/m, '').replace(/^\(/gm, '{anonymous}(').split('\n');
+    },
+
+    /**
+     * Given an Error object, return a formatted Array based on Opera 10's stacktrace string.
+     *
+     * @param e - Error object to inspect
+     * @return Array<String> of function calls, files and line numbers
+     */
+    opera10: function(e) {
+        var stack = e.stacktrace;
+        var lines = stack.split('\n'), ANON = '{anonymous}', lineRE = /.*line (\d+), column (\d+) in ((<anonymous function\:?\s*(\S+))|([^\(]+)\([^\)]*\))(?: in )?(.*)\s*$/i, i, j, len;
+        for (i = 2, j = 0, len = lines.length; i < len - 2; i++) {
+            if (lineRE.test(lines[i])) {
+                var location = RegExp.$6 + ':' + RegExp.$1 + ':' + RegExp.$2;
+                var fnName = RegExp.$3;
+                fnName = fnName.replace(/<anonymous function\:?\s?(\S+)?>/g, ANON);
+                lines[j++] = fnName + '@' + location;
+            }
+        }
+
+        lines.splice(j, lines.length - j);
+        return lines;
+    },
+
+    // Opera 7.x-9.x only!
+    opera: function(e) {
+        var lines = e.message.split('\n'), ANON = '{anonymous}', lineRE = /Line\s+(\d+).*script\s+(http\S+)(?:.*in\s+function\s+(\S+))?/i, i, j, len;
+
+        for (i = 4, j = 0, len = lines.length; i < len; i += 2) {
+            //TODO: RegExp.exec() would probably be cleaner here
+            if (lineRE.test(lines[i])) {
+                lines[j++] = (RegExp.$3 ? RegExp.$3 + '()@' + RegExp.$2 + RegExp.$1 : ANON + '()@' + RegExp.$2 + ':' + RegExp.$1) + ' -- ' + lines[i + 1].replace(/^\s+/, '');
+            }
+        }
+
+        lines.splice(j, lines.length - j);
+        return lines;
+    },
+
+    // Safari, IE, and others
+    other: function(curr) {
+        var ANON = '{anonymous}', fnRE = /function\s*([\w\-$]+)?\s*\(/i, stack = [], fn, args, maxStackSize = 10;
+        while (curr && stack.length < maxStackSize) {
+            fn = fnRE.test(curr.toString()) ? RegExp.$1 || ANON : ANON;
+            args = Array.prototype.slice.call(curr['arguments'] || []);
+            stack[stack.length] = fn + '(' + this.stringifyArguments(args) + ')';
+            curr = curr.caller;
+        }
+        return stack;
+    },
+
+    /**
+     * Given arguments array as a String, subsituting type names for non-string types.
+     *
+     * @param {Arguments} object
+     * @return {Array} of Strings with stringified arguments
+     */
+    stringifyArguments: function(args) {
+        var slice = Array.prototype.slice;
+        for (var i = 0; i < args.length; ++i) {
+            var arg = args[i];
+            if (arg === undefined) {
+                args[i] = 'undefined';
+            } else if (arg === null) {
+                args[i] = 'null';
+            } else if (arg.constructor) {
+                if (arg.constructor === Array) {
+                    if (arg.length < 3) {
+                        args[i] = '[' + this.stringifyArguments(arg) + ']';
+                    } else {
+                        args[i] = '[' + this.stringifyArguments(slice.call(arg, 0, 1)) + '...' + this.stringifyArguments(slice.call(arg, -1)) + ']';
+                    }
+                } else if (arg.constructor === Object) {
+                    args[i] = '#object';
+                } else if (arg.constructor === Function) {
+                    args[i] = '#function';
+                } else if (arg.constructor === String) {
+                    args[i] = '"' + arg + '"';
+                }
+            }
+        }
+        return args.join(',');
+    },
+
+    sourceCache: {},
+
+    /**
+     * @return the text from a given URL.
+     */
+    ajax: function(url) {
+        var req = this.createXMLHTTPObject();
+        if (!req) {
+            return;
+        }
+        req.open('GET', url, false);
+        req.setRequestHeader('User-Agent', 'XMLHTTP/1.0');
+        req.send('');
+        return req.responseText;
+    },
+
+    /**
+     * Try XHR methods in order and store XHR factory.
+     *
+     * @return <Function> XHR function or equivalent
+     */
+    createXMLHTTPObject: function() {
+        var xmlhttp, XMLHttpFactories = [
+            function() {
+                return new XMLHttpRequest();
+            }, function() {
+                return new ActiveXObject('Msxml2.XMLHTTP');
+            }, function() {
+                return new ActiveXObject('Msxml3.XMLHTTP');
+            }, function() {
+                return new ActiveXObject('Microsoft.XMLHTTP');
+            }
+        ];
+        for (var i = 0; i < XMLHttpFactories.length; i++) {
+            try {
+                xmlhttp = XMLHttpFactories[i]();
+                // Use memoization to cache the factory
+                this.createXMLHTTPObject = XMLHttpFactories[i];
+                return xmlhttp;
+            } catch (e) {
+            }
+        }
+    },
+
+    /**
+     * Given a URL, check if it is in the same domain (so we can get the source
+     * via Ajax).
+     *
+     * @param url <String> source url
+     * @return False if we need a cross-domain request
+     */
+    isSameDomain: function(url) {
+        return url.indexOf(location.hostname) !== -1;
+    },
+
+    /**
+     * Get source code from given URL if in the same domain.
+     *
+     * @param url <String> JS source URL
+     * @return <Array> Array of source code lines
+     */
+    getSource: function(url) {
+        if (!(url in this.sourceCache)) {
+            this.sourceCache[url] = this.ajax(url).split('\n');
+        }
+        return this.sourceCache[url];
+    },
+
+    guessAnonymousFunctions: function(stack) {
+        for (var i = 0; i < stack.length; ++i) {
+            var reStack = /\{anonymous\}\(.*\)@(\w+:\/\/([\-\w\.]+)+(:\d+)?[^:]+):(\d+):?(\d+)?/;
+            var frame = stack[i], m = reStack.exec(frame);
+            if (m) {
+                var file = m[1], lineno = m[4], charno = m[7] || 0; //m[7] is character position in Chrome
+                if (file && this.isSameDomain(file) && lineno) {
+                    var functionName = this.guessAnonymousFunction(file, lineno, charno);
+                    stack[i] = frame.replace('{anonymous}', functionName);
+                }
+            }
+        }
+        return stack;
+    },
+
+    guessAnonymousFunction: function(url, lineNo, charNo) {
+        var ret;
+        try {
+            ret = this.findFunctionName(this.getSource(url), lineNo);
+        } catch (e) {
+            ret = 'getSource failed with url: ' + url + ', exception: ' + e.toString();
+        }
+        return ret;
+    },
+
+    findFunctionName: function(source, lineNo) {
+        // FIXME findFunctionName fails for compressed source
+        // (more than one function on the same line)
+        // TODO use captured args
+        // function {name}({args}) m[1]=name m[2]=args
+        var reFunctionDeclaration = /function\s+([^(]*?)\s*\(([^)]*)\)/;
+        // {name} = function ({args}) TODO args capture
+        // /['"]?([0-9A-Za-z_]+)['"]?\s*[:=]\s*function(?:[^(]*)/
+        var reFunctionExpression = /['"]?([0-9A-Za-z_]+)['"]?\s*[:=]\s*function\b/;
+        // {name} = eval()
+        var reFunctionEvaluation = /['"]?([0-9A-Za-z_]+)['"]?\s*[:=]\s*(?:eval|new Function)\b/;
+        // Walk backwards in the source lines until we find
+        // the line which matches one of the patterns above
+        var code = "", line, maxLines = 10, m;
+        for (var i = 0; i < maxLines; ++i) {
+            // FIXME lineNo is 1-based, source[] is 0-based
+            line = source[lineNo - i];
+            if (line) {
+                code = line + code;
+                m = reFunctionExpression.exec(code);
+                if (m && m[1]) {
+                    return m[1];
+                }
+                m = reFunctionDeclaration.exec(code);
+                if (m && m[1]) {
+                    //return m[1] + "(" + (m[2] || "") + ")";
+                    return m[1];
+                }
+                m = reFunctionEvaluation.exec(code);
+                if (m && m[1]) {
+                    return m[1];
+                }
+            }
+        }
+        return '(?)';
+    }
+};

resources/default_tweaks.py

@@ -37,7 +37,6 @@ series_index_auto_increment = 'next'
 # Can be either True or False
 authors_completer_append_separator = False
 
-
 #: Author sort name algorithm
 # The algorithm used to copy author to author_sort
 # Possible values are:
@@ -71,6 +70,15 @@ author_name_suffixes = ('Jr', 'Sr', 'Inc', 'Ph.D', 'Phd',
 #   categories_use_field_for_author_name = 'author_sort'
 categories_use_field_for_author_name = 'author'
 
+#: Completion sort order: choose when to change from lexicographic to ASCII-like
+# Calibre normally uses locale-dependent lexicographic ordering when showing
+# completion values. This means that the sort order is correct for the user's
+# language. However, this can be slow. Performance is improved by switching to
+# ascii ordering. This tweak controls when that switch happens. Set it to zero
+# to always use ascii ordering. Set it to something larger than zero to switch
+# to ascii ordering for performance reasons.
+completion_change_to_ascii_sorting = 2500
+
 #: Control partitioning of Tag Browser
 # When partitioning the tags browser, the format of the subcategory label is
 # controlled by a template: categories_collapsed_name_template if sorting by
@@ -93,7 +101,6 @@ categories_collapsed_name_template = r'{first.sort:shorten(4,,0)} - {last.sort:s
 categories_collapsed_rating_template = r'{first.avg_rating:4.2f:ifempty(0)} - {last.avg_rating:4.2f:ifempty(0)}'
 categories_collapsed_popularity_template = r'{first.count:d} - {last.count:d}'
 
-
 #: Specify columns to sort the booklist by on startup
 # Provide a set of columns to be sorted on when calibre starts
 #  The argument is None if saved sort history is to be used
@@ -244,17 +251,14 @@ sony_collection_name_template='{value}{category:| (|)}'
 # Default: empty (no rules), so no collection attributes are named.
 sony_collection_sorting_rules = []
 
-
 #: Control how tags are applied when copying books to another library
 # Set this to True to ensure that tags in 'Tags to add when adding
 # a book' are added when copying books to another library
 add_new_book_tags_when_importing_books = False
 
-
 #: Set the maximum number of tags to show per book in the content server
 max_content_server_tags_shown=5
 
-
 #: Set custom metadata fields that the content server will or will not display.
 # content_server_will_display is a list of custom fields to be displayed.
 # content_server_wont_display is a list of custom fields not to be displayed.
@@ -296,7 +300,6 @@ generate_cover_foot_font = None
 # Example: doubleclick_on_library_view = 'do_nothing'
 doubleclick_on_library_view = 'open_viewer'
 
-
 #: Language to use when sorting.
 # Setting this tweak will force sorting to use the
 # collating order for the specified language. This might be useful if you run

resources/template-functions.json

@@ -1,42 +0,0 @@
-{
-    "and": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if not args[i]:\n            return ''\n        i += 1\n    return '1'\n", 
-    "contains": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, test, value_if_present, value_if_not):\n    if re.search(test, val, flags=re.I):\n        return value_if_present\n    else:\n        return value_if_not\n", 
-    "divide": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x / y)\n", 
-    "uppercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.upper()\n", 
-    "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    res = ''\n    for i in range(0, len(args)):\n        res += args[i]\n    return res\n", 
-    "in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, sep, pat, fv, nfv):\n    l = [v.strip() for v in val.split(sep) if v.strip()]\n    if l:\n        for v in l:\n            if re.search(pat, v, flags=re.I):\n                return fv\n    return nfv\n", 
-    "multiply": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x * y)\n", 
-    "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n    if val:\n        return val\n    else:\n        return value_if_empty\n", 
-    "booksize": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.book_size is not None:\n        try:\n            return str(mi.book_size)\n        except:\n            pass\n    return ''\n", 
-    "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n    if not val:\n        return ''\n    vals = [v.strip() for v in val.split(',')]\n    for v in vals:\n        if v.startswith(key+':'):\n            return v[len(key)+1:]\n    return ''\n", 
-    "strcmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    v = strcmp(x, y)\n    if v < 0:\n        return lt\n    if v == 0:\n        return eq\n    return gt\n", 
-    "first_non_empty": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return args[i]\n        i += 1\n    return ''\n", 
-    "re": "def evaluate(self, formatter, kwargs, mi, locals, val, pattern, replacement):\n    return re.sub(pattern, replacement, val, flags=re.I)\n", 
-    "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
-    "list_item": "def evaluate(self, formatter, kwargs, mi, locals, val, index, sep):\n    if not val:\n        return ''\n    index = int(index)\n    val = val.split(sep)\n    try:\n        return val[index]\n    except:\n        return ''\n", 
-    "shorten": "def evaluate(self, formatter, kwargs, mi, locals,\n             val, leading, center_string, trailing):\n    l = max(0, int(leading))\n    t = max(0, int(trailing))\n    if len(val) > l + len(center_string) + t:\n        return val[0:l] + center_string + ('' if t == 0 else val[-t:])\n    else:\n        return val\n", 
-    "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return formatter.get_value(name, [], kwargs)\n", 
-    "add": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x + y)\n", 
-    "lookup": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if len(args) == 2: # here for backwards compatibility\n        if val:\n            return formatter.vformat('{'+args[0].strip()+'}', [], kwargs)\n        else:\n            return formatter.vformat('{'+args[1].strip()+'}', [], kwargs)\n    if (len(args) % 2) != 1:\n        raise ValueError(_('lookup requires either 2 or an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return formatter.vformat('{' + args[i].strip() + '}', [], kwargs)\n        if re.search(args[i], val, flags=re.I):\n            return formatter.vformat('{'+args[i+1].strip() + '}', [], kwargs)\n        i += 2\n", 
-    "template": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    template = template.replace('[[', '{').replace(']]', '}')\n    return formatter.__class__().safe_format(template, kwargs, 'TEMPLATE', mi)\n", 
-    "print": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    print args\n    return None\n", 
-    "merge_lists": "def evaluate(self, formatter, kwargs, mi, locals, list1, list2, separator):\n    l1 = [l.strip() for l in list1.split(separator) if l.strip()]\n    l2 = [l.strip() for l in list2.split(separator) if l.strip()]\n    lcl1 = set([icu_lower(l) for l in l1])\n    res = []\n    for i in l1:\n        res.append(i)\n    for i in l2:\n        if icu_lower(i) not in lcl1:\n            res.append(i)\n    return ', '.join(sorted(res, key=sort_key))\n", 
-    "str_in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, sep, str, fv, nfv):\n    l = [v.strip() for v in val.split(sep) if v.strip()]\n    c = [v.strip() for v in str.split(sep) if v.strip()]\n    if l:\n        for v in l:\n            for t in c:\n                if strcmp(t, v) == 0:\n                    return fv\n    return nfv\n", 
-    "titlecase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return titlecase(val)\n", 
-    "subitems": "def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index):\n    if not val:\n        return ''\n    si = int(start_index)\n    ei = int(end_index)\n    items = [v.strip() for v in val.split(',')]\n    rv = set()\n    for item in items:\n        component = item.split('.')\n        try:\n            if ei == 0:\n                rv.add('.'.join(component[si:]))\n            else:\n                rv.add('.'.join(component[si:ei]))\n        except:\n            pass\n    return ', '.join(sorted(rv, key=sort_key))\n", 
-    "sublist": "def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index, sep):\n    if not val:\n        return ''\n    si = int(start_index)\n    ei = int(end_index)\n    val = val.split(sep)\n    try:\n        if ei == 0:\n            return sep.join(val[si:])\n        else:\n            return sep.join(val[si:ei])\n    except:\n        return ''\n", 
-    "test": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):\n    if val:\n        return value_if_set\n    else:\n        return value_not_set\n", 
-    "eval": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    from formatter import eval_formatter\n    template = template.replace('[[', '{').replace(']]', '}')\n    return eval_formatter.safe_format(template, locals, 'EVAL', None)\n", 
-    "not": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return '1'\n        i += 1\n    return ''\n", 
-    "format_date": "def evaluate(self, formatter, kwargs, mi, locals, val, format_string):\n    if not val:\n        return ''\n    try:\n        dt = parse_date(val)\n        s = format_date(dt, format_string)\n    except:\n        s = 'BAD DATE'\n    return s\n", 
-    "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
-    "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n    return unicode(len(val.split(sep)))\n", 
-    "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.lower()\n", 
-    "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n    return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", 
-    "or": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return '1'\n        i += 1\n    return ''\n", 
-    "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if (len(args) % 2) != 1:\n        raise ValueError(_('switch requires an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return args[i]\n        if re.search(args[i], val, flags=re.I):\n            return args[i+1]\n        i += 2\n", 
-    "ondevice": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.ondevice_col:\n        return _('Yes')\n    return ''\n", 
-    "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
-    "raw_field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return unicode(getattr(mi, name, None))\n", 
-    "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    if x < y:\n        return lt\n    if x == y:\n        return eq\n    return gt\n"
-}

resources/templates/fb2.xsl

@@ -32,16 +32,11 @@
                     <xsl:value-of select="fb:description/fb:title-info/fb:book-title"/>
                 </title>
                 <style type="text/css">
-                    a { color : #0002CC }
+                    body { text-align : justify }
-
-                    a:hover { color : #BF0000 }
-                    
-                    body { background-color : #FEFEFE; color : #000000; font-family : Verdana, Geneva, Arial, Helvetica, sans-serif; text-align : justify }
                     
                     h1{ font-size : 160%; font-style : normal; font-weight : bold; text-align : left; border : 1px solid Black;  background-color : #E7E7E7; margin-left : 0px;  page-break-before : always; }
                     
                     h2{ font-size : 130%; font-style : normal; font-weight : bold; text-align : left; background-color : #EEEEEE;  border : 1px solid Gray;  page-break-before : always; }
-                    
                     h3{ font-size : 110%; font-style : normal; font-weight : bold; text-align : left;  background-color : #F1F1F1;  border : 1px solid Silver;}
                     
                     h4{ font-size : 100%; font-style : normal; font-weight : bold; text-align : left; border : 1px solid Gray;  background-color : #F4F4F4;}
@@ -56,13 +51,11 @@
                     
                     hr { color : Black }
                     
-                    div {font-family : "Times New Roman", Times, serif; text-align : justify}
-                    
                     ul {margin-left: 0}
                     
                     .epigraph{width:50%; margin-left : 35%;}
                     
-                    div.paragraph { text-align: justify; text-indent: 2em; }
+                    div.paragraph { text-indent: 2em; }
                 </style>
                 <link rel="stylesheet" type="text/css" href="inline-styles.css" />
             </head>

setup/install.py

@@ -8,8 +8,8 @@ __docformat__ = 'restructuredtext en'
 
 import sys, os, textwrap, subprocess, shutil, tempfile, atexit, stat, shlex
 
-from setup import Command, islinux, isfreebsd, isbsd, basenames, modules, functions, \
+from setup import (Command, islinux, isbsd, basenames, modules, functions,
-        __appname__, __version__
+        __appname__, __version__)
 
 HEADER = '''\
 #!/usr/bin/env python2

setup/installer/windows/MemoryModule.c

@@ -0,0 +1,689 @@
+/*
+ * Memory DLL loading code
+ * Version 0.0.2 with additions from Thomas Heller
+ *
+ * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
+ * http://www.joachim-bauch.de
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is MemoryModule.c
+ *
+ * The Initial Developer of the Original Code is Joachim Bauch.
+ *
+ * Portions created by Joachim Bauch are Copyright (C) 2004-2005
+ * Joachim Bauch. All Rights Reserved.
+ *
+ * Portions Copyright (C) 2005 Thomas Heller.
+ *
+ */
+
+// disable warnings about pointer <-> DWORD conversions
+#pragma warning( disable : 4311 4312 )
+
+#include <Windows.h>
+#include <winnt.h>
+#if DEBUG_OUTPUT
+#include <stdio.h>
+#endif
+
+#ifndef IMAGE_SIZEOF_BASE_RELOCATION
+// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!?
+# define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION))
+#endif
+#include "MemoryModule.h"
+
+/*
+  XXX We need to protect at least walking the 'loaded' linked list with a lock!
+*/
+
+/******************************************************************/
+FINDPROC findproc;
+void *findproc_data = NULL;
+
+struct NAME_TABLE {
+	char *name;
+	DWORD ordinal;
+};
+
+typedef struct tagMEMORYMODULE {
+	PIMAGE_NT_HEADERS headers;
+	unsigned char *codeBase;
+	HMODULE *modules;
+	int numModules;
+	int initialized;
+
+	struct NAME_TABLE *name_table;
+
+	char *name;
+	int refcount;
+	struct tagMEMORYMODULE *next, *prev;
+} MEMORYMODULE, *PMEMORYMODULE;
+
+typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved);
+
+#define GET_HEADER_DICTIONARY(module, idx)	&(module)->headers->OptionalHeader.DataDirectory[idx]
+
+MEMORYMODULE *loaded; /* linked list of loaded memory modules */
+
+/* private - insert a loaded library in a linked list */
+static void _Register(char *name, MEMORYMODULE *module)
+{
+	module->next = loaded;
+	if (loaded)
+		loaded->prev = module;
+	module->prev = NULL;
+	loaded = module;
+}
+
+/* private - remove a loaded library from a linked list */
+static void _Unregister(MEMORYMODULE *module)
+{
+	free(module->name);
+	if (module->prev)
+		module->prev->next = module->next;
+	if (module->next)
+		module->next->prev = module->prev;
+	if (module == loaded)
+		loaded = module->next;
+}
+
+/* public - replacement for GetModuleHandle() */
+HMODULE MyGetModuleHandle(LPCTSTR lpModuleName)
+{
+	MEMORYMODULE *p = loaded;
+	while (p) {
+		// If already loaded, only increment the reference count
+		if (0 == stricmp(lpModuleName, p->name)) {
+			return (HMODULE)p;
+		}
+		p = p->next;
+	}
+	return GetModuleHandle(lpModuleName);
+}
+
+/* public - replacement for LoadLibrary, but searches FIRST for memory
+   libraries, then for normal libraries.  So, it will load libraries AS memory
+   module if they are found by findproc().
+*/
+HMODULE MyLoadLibrary(char *lpFileName)
+{
+	MEMORYMODULE *p = loaded;
+	HMODULE hMod;
+
+	while (p) {
+		// If already loaded, only increment the reference count
+		if (0 == stricmp(lpFileName, p->name)) {
+			p->refcount++;
+			return (HMODULE)p;
+		}
+		p = p->next;
+	}
+	if (findproc && findproc_data) {
+		void *pdata = findproc(lpFileName, findproc_data);
+		if (pdata) {
+			hMod = MemoryLoadLibrary(lpFileName, pdata);
+			free(p);
+			return hMod;
+		}
+	}
+	hMod = LoadLibrary(lpFileName);
+	return hMod;
+}
+
+/* public - replacement for GetProcAddress() */
+FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName)
+{
+	MEMORYMODULE *p = loaded;
+	while (p) {
+		if ((HMODULE)p == hModule)
+			return MemoryGetProcAddress(p, lpProcName);
+		p = p->next;
+	}
+	return GetProcAddress(hModule, lpProcName);
+}
+
+/* public - replacement for FreeLibrary() */
+BOOL MyFreeLibrary(HMODULE hModule)
+{
+	MEMORYMODULE *p = loaded;
+	while (p) {
+		if ((HMODULE)p == hModule) {
+			if (--p->refcount == 0) {
+				_Unregister(p);
+				MemoryFreeLibrary(p);
+			}
+			return TRUE;
+		}
+		p = p->next;
+	}
+	return FreeLibrary(hModule);
+}
+
+#if DEBUG_OUTPUT
+static void
+OutputLastError(const char *msg)
+{
+	LPVOID tmp;
+	char *tmpmsg;
+	FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+		NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL);
+	tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3);
+	sprintf(tmpmsg, "%s: %s", msg, tmp);
+	OutputDebugString(tmpmsg);
+	LocalFree(tmpmsg);
+	LocalFree(tmp);
+}
+#endif
+
+/*
+static int dprintf(char *fmt, ...)
+{
+	char Buffer[4096];
+	va_list marker;
+	int result;
+	
+	va_start(marker, fmt);
+	result = vsprintf(Buffer, fmt, marker);
+	OutputDebugString(Buffer);
+	return result;
+}
+*/
+
+static void
+CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module)
+{
+	int i, size;
+	unsigned char *codeBase = module->codeBase;
+	unsigned char *dest;
+	PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
+	for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++)
+	{
+		if (section->SizeOfRawData == 0)
+		{
+			// section doesn't contain data in the dll itself, but may define
+			// uninitialized data
+			size = old_headers->OptionalHeader.SectionAlignment;
+			if (size > 0)
+			{
+				dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
+					size,
+					MEM_COMMIT,
+					PAGE_READWRITE);
+
+				section->Misc.PhysicalAddress = (DWORD)dest;
+				memset(dest, 0, size);
+			}
+
+			// section is empty
+			continue;
+		}
+
+		// commit memory block and copy data from dll
+		dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
+							section->SizeOfRawData,
+							MEM_COMMIT,
+							PAGE_READWRITE);
+		memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData);
+		section->Misc.PhysicalAddress = (DWORD)dest;
+	}
+}
+
+// Protection flags for memory pages (Executable, Readable, Writeable)
+static int ProtectionFlags[2][2][2] = {
+	{
+		// not executable
+		{PAGE_NOACCESS, PAGE_WRITECOPY},
+		{PAGE_READONLY, PAGE_READWRITE},
+	}, {
+		// executable
+		{PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY},
+		{PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE},
+	},
+};
+
+static void
+FinalizeSections(PMEMORYMODULE module)
+{
+	int i;
+	PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
+	
+	// loop through all sections and change access flags
+	for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++)
+	{
+		DWORD protect, oldProtect, size;
+		int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0;
+		int readable =   (section->Characteristics & IMAGE_SCN_MEM_READ) != 0;
+		int writeable =  (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0;
+
+		if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE)
+		{
+			// section is not needed any more and can safely be freed
+			VirtualFree((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, MEM_DECOMMIT);
+			continue;
+		}
+
+		// determine protection flags based on characteristics
+		protect = ProtectionFlags[executable][readable][writeable];
+		if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED)
+			protect |= PAGE_NOCACHE;
+
+		// determine size of region
+		size = section->SizeOfRawData;
+		if (size == 0)
+		{
+			if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA)
+				size = module->headers->OptionalHeader.SizeOfInitializedData;
+			else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)
+				size = module->headers->OptionalHeader.SizeOfUninitializedData;
+		}
+
+		if (size > 0)
+		{
+			// change memory access flags
+			if (VirtualProtect((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, protect, &oldProtect) == 0)
+#if DEBUG_OUTPUT
+				OutputLastError("Error protecting memory page")
+#endif
+			;
+		}
+	}
+}
+
+static void
+PerformBaseRelocation(PMEMORYMODULE module, DWORD delta)
+{
+	DWORD i;
+	unsigned char *codeBase = module->codeBase;
+
+	PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC);
+	if (directory->Size > 0)
+	{
+		PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION)(codeBase + directory->VirtualAddress);
+		for (; relocation->VirtualAddress > 0; )
+		{
+			unsigned char *dest = (unsigned char *)(codeBase + relocation->VirtualAddress);
+			unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION);
+			for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++)
+			{
+				DWORD *patchAddrHL;
+				int type, offset;
+
+				// the upper 4 bits define the type of relocation
+				type = *relInfo >> 12;
+				// the lower 12 bits define the offset
+				offset = *relInfo & 0xfff;
+				
+				switch (type)
+				{
+				case IMAGE_REL_BASED_ABSOLUTE:
+					// skip relocation
+					break;
+
+				case IMAGE_REL_BASED_HIGHLOW:
+					// change complete 32 bit address
+					patchAddrHL = (DWORD *)(dest + offset);
+					*patchAddrHL += delta;
+					break;
+
+				default:
+					//printf("Unknown relocation: %d\n", type);
+					break;
+				}
+			}
+
+			// advance to next relocation block
+			relocation = (PIMAGE_BASE_RELOCATION)(((DWORD)relocation) + relocation->SizeOfBlock);
+		}
+	}
+}
+
+static int
+BuildImportTable(PMEMORYMODULE module)
+{
+	int result=1;
+	unsigned char *codeBase = module->codeBase;
+
+	PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT);
+	if (directory->Size > 0)
+	{
+		PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR)(codeBase + directory->VirtualAddress);
+		for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++)
+		{
+			DWORD *thunkRef, *funcRef;
+			HMODULE handle;
+
+			handle = MyLoadLibrary(codeBase + importDesc->Name);
+			if (handle == INVALID_HANDLE_VALUE)
+			{
+				//LastError should already be set
+#if DEBUG_OUTPUT
+				OutputLastError("Can't load library");
+#endif
+				result = 0;
+				break;
+			}
+
+			module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE)));
+			if (module->modules == NULL)
+			{
+				SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+				result = 0;
+				break;
+			}
+
+			module->modules[module->numModules++] = handle;
+			if (importDesc->OriginalFirstThunk)
+			{
+				thunkRef = (DWORD *)(codeBase + importDesc->OriginalFirstThunk);
+				funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+			} else {
+				// no hint table
+				thunkRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+				funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+			}
+			for (; *thunkRef; thunkRef++, funcRef++)
+			{
+				if IMAGE_SNAP_BY_ORDINAL(*thunkRef) {
+					*funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef));
+				} else {
+					PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME)(codeBase + *thunkRef);
+					*funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)&thunkData->Name);
+				}
+				if (*funcRef == 0)
+				{
+					SetLastError(ERROR_PROC_NOT_FOUND);
+					result = 0;
+					break;
+				}
+			}
+
+			if (!result)
+				break;
+		}
+	}
+
+	return result;
+}
+
+/*
+  MemoryLoadLibrary - load a library AS MEMORY MODULE, or return
+  existing MEMORY MODULE with increased refcount.
+
+  This allows to load a library AGAIN as memory module which is
+  already loaded as HMODULE!
+
+*/
+HMEMORYMODULE MemoryLoadLibrary(char *name, const void *data)
+{
+	PMEMORYMODULE result;
+	PIMAGE_DOS_HEADER dos_header;
+	PIMAGE_NT_HEADERS old_header;
+	unsigned char *code, *headers;
+	DWORD locationDelta;
+	DllEntryProc DllEntry;
+	BOOL successfull;
+	MEMORYMODULE *p = loaded;
+
+	while (p) {
+		// If already loaded, only increment the reference count
+		if (0 == stricmp(name, p->name)) {
+			p->refcount++;
+			return (HMODULE)p;
+		}
+		p = p->next;
+	}
+
+	/* Do NOT check for GetModuleHandle here! */
+
+	dos_header = (PIMAGE_DOS_HEADER)data;
+	if (dos_header->e_magic != IMAGE_DOS_SIGNATURE)
+	{
+		SetLastError(ERROR_BAD_FORMAT);
+#if DEBUG_OUTPUT
+		OutputDebugString("Not a valid executable file.\n");
+#endif
+		return NULL;
+	}
+
+	old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew];
+	if (old_header->Signature != IMAGE_NT_SIGNATURE)
+	{
+		SetLastError(ERROR_BAD_FORMAT);
+#if DEBUG_OUTPUT
+		OutputDebugString("No PE header found.\n");
+#endif
+		return NULL;
+	}
+
+	// reserve memory for image of library
+	code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase),
+		old_header->OptionalHeader.SizeOfImage,
+		MEM_RESERVE,
+		PAGE_READWRITE);
+
+    if (code == NULL)
+        // try to allocate memory at arbitrary position
+        code = (unsigned char *)VirtualAlloc(NULL,
+            old_header->OptionalHeader.SizeOfImage,
+            MEM_RESERVE,
+            PAGE_READWRITE);
+    
+	if (code == NULL)
+	{
+		SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+#if DEBUG_OUTPUT
+		OutputLastError("Can't reserve memory");
+#endif
+		return NULL;
+	}
+
+	result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE));
+	result->codeBase = code;
+	result->numModules = 0;
+	result->modules = NULL;
+	result->initialized = 0;
+	result->next = result->prev = NULL;
+	result->refcount = 1;
+	result->name = strdup(name);
+	result->name_table = NULL;
+
+	// XXX: is it correct to commit the complete memory region at once?
+    //      calling DllEntry raises an exception if we don't...
+	VirtualAlloc(code,
+		old_header->OptionalHeader.SizeOfImage,
+		MEM_COMMIT,
+		PAGE_READWRITE);
+
+	// commit memory for headers
+	headers = (unsigned char *)VirtualAlloc(code,
+		old_header->OptionalHeader.SizeOfHeaders,
+		MEM_COMMIT,
+		PAGE_READWRITE);
+	
+	// copy PE header to code
+	memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders);
+	result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew];
+
+	// update position
+	result->headers->OptionalHeader.ImageBase = (DWORD)code;
+
+	// copy sections from DLL file block to new memory location
+	CopySections(data, old_header, result);
+
+	// adjust base address of imported data
+	locationDelta = (DWORD)(code - old_header->OptionalHeader.ImageBase);
+	if (locationDelta != 0)
+		PerformBaseRelocation(result, locationDelta);
+
+	// load required dlls and adjust function table of imports
+	if (!BuildImportTable(result))
+		goto error;
+
+	// mark memory pages depending on section headers and release
+	// sections that are marked as "discardable"
+	FinalizeSections(result);
+
+	// get entry point of loaded library
+	if (result->headers->OptionalHeader.AddressOfEntryPoint != 0)
+	{
+		DllEntry = (DllEntryProc)(code + result->headers->OptionalHeader.AddressOfEntryPoint);
+		if (DllEntry == 0)
+		{
+			SetLastError(ERROR_BAD_FORMAT); /* XXX ? */
+#if DEBUG_OUTPUT
+			OutputDebugString("Library has no entry point.\n");
+#endif
+			goto error;
+		}
+
+		// notify library about attaching to process
+		successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0);
+		if (!successfull)
+		{
+#if DEBUG_OUTPUT
+			OutputDebugString("Can't attach library.\n");
+#endif
+			goto error;
+		}
+		result->initialized = 1;
+	}
+
+	_Register(name, result);
+
+	return (HMEMORYMODULE)result;
+
+error:
+	// cleanup
+	free(result->name);
+	MemoryFreeLibrary(result);
+	return NULL;
+}
+
+int _compare(const struct NAME_TABLE *p1, const struct NAME_TABLE *p2)
+{
+	return stricmp(p1->name, p2->name);
+}
+
+int _find(const char **name, const struct NAME_TABLE *p)
+{
+	return stricmp(*name, p->name);
+}
+
+struct NAME_TABLE *GetNameTable(PMEMORYMODULE module)
+{
+	unsigned char *codeBase;
+	PIMAGE_EXPORT_DIRECTORY exports;
+	PIMAGE_DATA_DIRECTORY directory;
+	DWORD i, *nameRef;
+	WORD *ordinal;
+	struct NAME_TABLE *p, *ptab;
+
+	if (module->name_table)
+		return module->name_table;
+
+	codeBase = module->codeBase;
+	directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXPORT);
+	exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
+
+	nameRef = (DWORD *)(codeBase + exports->AddressOfNames);
+	ordinal = (WORD *)(codeBase + exports->AddressOfNameOrdinals);
+
+	p = ((PMEMORYMODULE)module)->name_table = (struct NAME_TABLE *)malloc(sizeof(struct NAME_TABLE)
+									      * exports->NumberOfNames);
+	if (p == NULL)
+		return NULL;
+	ptab = p;
+	for (i=0; i<exports->NumberOfNames; ++i) {
+		p->name = (char *)(codeBase + *nameRef++);
+		p->ordinal = *ordinal++;
+		++p;
+	}
+	qsort(ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _compare);
+	return ptab;
+}
+
+FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name)
+{
+	unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase;
+	int idx=-1;
+	PIMAGE_EXPORT_DIRECTORY exports;
+	PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT);
+
+	if (directory->Size == 0)
+		// no export table found
+		return NULL;
+
+	exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
+	if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0)
+		// DLL doesn't export anything
+		return NULL;
+
+	if (HIWORD(name)) {
+		struct NAME_TABLE *ptab;
+		struct NAME_TABLE *found;
+		ptab = GetNameTable((PMEMORYMODULE)module);
+		if (ptab == NULL)
+			// some failure
+			return NULL;
+		found = bsearch(&name, ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _find);
+		if (found == NULL)
+			// exported symbol not found
+			return NULL;
+	
+		idx = found->ordinal;
+	}
+	else
+		idx = LOWORD(name) - exports->Base;
+
+	if ((DWORD)idx > exports->NumberOfFunctions)
+		// name <-> ordinal number don't match
+		return NULL;
+	
+	// AddressOfFunctions contains the RVAs to the "real" functions
+	return (FARPROC)(codeBase + *(DWORD *)(codeBase + exports->AddressOfFunctions + (idx*4)));
+}
+
+void MemoryFreeLibrary(HMEMORYMODULE mod)
+{
+	int i;
+	PMEMORYMODULE module = (PMEMORYMODULE)mod;
+
+	if (module != NULL)
+	{
+		if (module->initialized != 0)
+		{
+			// notify library about detaching from process
+			DllEntryProc DllEntry = (DllEntryProc)(module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint);
+			(*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0);
+			module->initialized = 0;
+		}
+
+		if (module->modules != NULL)
+		{
+			// free previously opened libraries
+			for (i=0; i<module->numModules; i++)
+				if (module->modules[i] != INVALID_HANDLE_VALUE)
+					MyFreeLibrary(module->modules[i]);
+
+			free(module->modules);
+		}
+
+		if (module->codeBase != NULL)
+			// release memory of library
+			VirtualFree(module->codeBase, 0, MEM_RELEASE);
+
+		if (module->name_table != NULL)
+			free(module->name_table);
+
+		HeapFree(GetProcessHeap(), 0, module);
+	}
+}

setup/installer/windows/MemoryModule.h

@@ -0,0 +1,58 @@
+/*
+ * Memory DLL loading code
+ * Version 0.0.2
+ *
+ * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
+ * http://www.joachim-bauch.de
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is MemoryModule.h
+ *
+ * The Initial Developer of the Original Code is Joachim Bauch.
+ *
+ * Portions created by Joachim Bauch are Copyright (C) 2004-2005
+ * Joachim Bauch. All Rights Reserved.
+ *
+ */
+
+#ifndef __MEMORY_MODULE_HEADER
+#define __MEMORY_MODULE_HEADER
+
+#include <Windows.h>
+
+typedef void *HMEMORYMODULE;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void *(*FINDPROC)();
+
+extern FINDPROC findproc;
+extern void *findproc_data;
+
+HMEMORYMODULE MemoryLoadLibrary(char *, const void *);
+
+FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *);
+
+void MemoryFreeLibrary(HMEMORYMODULE);
+
+BOOL MyFreeLibrary(HMODULE hModule);
+HMODULE MyLoadLibrary(char *lpFileName);
+FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName);
+HMODULE MyGetModuleHandle(LPCTSTR lpModuleName);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // __MEMORY_MODULE_HEADER

setup/installer/windows/__init__.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import os, shutil, subprocess
 
-from setup import Command, __appname__
+from setup import Command, __appname__, __version__
 from setup.installer import VMInstaller
 
 class Win(Command):
@@ -43,4 +43,11 @@ class Win32(VMInstaller):
             self.warn('Failed to freeze')
             raise SystemExit(1)
 
+        installer = 'dist/%s-portable-%s.zip'%(__appname__, __version__)
+        subprocess.check_call(('scp',
+            'xp_build:build/%s/%s'%(__appname__, installer), 'dist'))
+        if not os.path.exists(installer):
+            self.warn('Failed to get portable installer')
+            raise SystemExit(1)
+
 

setup/installer/windows/en-us.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <WixLocalization Culture="en-us" xmlns="http://schemas.microsoft.com/wix/2006/localization">
-    <String Id="AdvancedWelcomeEulaDlgDescriptionPerUser">If you are upgrading from a {app} version older than 0.6.17, please uninstall {app} first. Click Advanced to change installation settings.</String>
+    <String Id="AdvancedWelcomeEulaDlgDescriptionPerUser">Click Advanced to change installation settings.</String>
     <String Id="ProgressTextFileCost">Computing space requirements, this may take upto five minutes...</String>
     <String Id="ProgressTextCostInitialize">Computing space requirements, this may take upto five minutes...</String>
     <String Id="ProgressTextCostFinalize">Computing space requirements, this may take upto five minutes...</String>

setup/installer/windows/freeze.py

@@ -8,19 +8,19 @@ __docformat__ = 'restructuredtext en'
 
 import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time
 
-from setup import Command, modules, functions, basenames, __version__, \
+from setup import (Command, modules, functions, basenames, __version__,
-    __appname__
+    __appname__)
 from setup.build_environment import msvc, MT, RC
 from setup.installer.windows.wix import WixMixIn
 
 OPENSSL_DIR = r'Q:\openssl'
 QT_DIR = 'Q:\\Qt\\4.7.3'
 QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
-LIBUSB_DIR       = 'C:\\libusb'
 LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
 SW               = r'C:\cygwin\home\kovid\sw'
 IMAGEMAGICK      = os.path.join(SW, 'build', 'ImageMagick-6.6.6',
         'VisualMagick', 'bin')
+CRT = r'C:\Microsoft.VC90.CRT'
 
 VERSION = re.sub('[a-z]\d+', '', __version__)
 WINVER = VERSION+'.0'
@@ -50,7 +50,7 @@ def walk(dir):
 
 class Win32Freeze(Command, WixMixIn):
 
-    description = 'Free windows calibre installation'
+    description = 'Freeze windows calibre installation'
 
     def add_options(self, parser):
         parser.add_option('--no-ice', default=False, action='store_true',
@@ -71,33 +71,69 @@ class Win32Freeze(Command, WixMixIn):
         self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc')
         self.py_ver = ''.join(map(str, sys.version_info[:2]))
         self.lib_dir = self.j(self.base, 'Lib')
-        self.pydlib = self.j(self.base, 'pydlib')
         self.pylib = self.j(self.base, 'pylib.zip')
+        self.dll_dir = self.j(self.base, 'DLLs')
+        self.plugins_dir = os.path.join(self.base, 'plugins2')
+        self.portable_base = self.j(self.d(self.base), 'Calibre Portable')
+        self.obj_dir = self.j(self.src_root, 'build', 'launcher')
 
         self.initbase()
         self.build_launchers()
+        self.add_plugins()
         self.freeze()
         self.embed_manifests()
         self.install_site_py()
         self.archive_lib_dir()
+        self.remove_CRT_from_manifests()
         self.create_installer()
+        self.build_portable()
+
+    def remove_CRT_from_manifests(self):
+        '''
+        The dependency on the CRT is removed from the manifests of all DLLs.
+        This allows the CRT loaded by the .exe files to be used instead.
+        '''
+        search_pat = re.compile(r'(?is)<dependency>.*Microsoft\.VC\d+\.CRT')
+        repl_pat = re.compile(
+            r'(?is)<dependency>.*?Microsoft\.VC\d+\.CRT.*?</dependency>')
+
+        for dll in glob.glob(self.j(self.dll_dir, '*.dll')):
+            bn = self.b(dll)
+            with open(dll, 'rb') as f:
+                raw = f.read()
+            match = search_pat.search(raw)
+            if match is None:
+                continue
+            self.info('Removing CRT dependency from manifest of: %s'%bn)
+            # Blank out the bytes corresponding to the dependency specification
+            nraw = repl_pat.sub(lambda m: b' '*len(m.group()), raw)
+            if len(nraw) != len(raw):
+                raise Exception('Something went wrong with %s'%bn)
+            with open(dll, 'wb') as f:
+                f.write(nraw)
 
     def initbase(self):
         if self.e(self.base):
             shutil.rmtree(self.base)
         os.makedirs(self.base)
 
+    def add_plugins(self):
+        self.info('Adding plugins...')
+        tgt = self.plugins_dir
+        if os.path.exists(tgt):
+            shutil.rmtree(tgt)
+        os.mkdir(tgt)
+        base = self.j(self.SRC, 'calibre', 'plugins')
+        for f in glob.glob(self.j(base, '*.pyd')):
+            # We dont want the manifests as the manifest in the exe will be
+            # used instead
+            shutil.copy2(f, tgt)
+
     def freeze(self):
         shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
 
-        self.info('Adding plugins...')
+        self.info('Adding CRT')
-        tgt = os.path.join(self.base, 'plugins')
+        shutil.copytree(CRT, self.j(self.base, os.path.basename(CRT)))
-        if not os.path.exists(tgt):
-            os.mkdir(tgt)
-        base = self.j(self.SRC, 'calibre', 'plugins')
-        for pat in ('*.pyd', '*.manifest'):
-            for f in glob.glob(self.j(base, pat)):
-                shutil.copy2(f, tgt)
 
         self.info('Adding resources...')
         tgt = self.j(self.base, 'resources')
@@ -106,7 +142,6 @@ class Win32Freeze(Command, WixMixIn):
         shutil.copytree(self.j(self.src_root, 'resources'), tgt)
 
         self.info('Adding Qt and python...')
-        self.dll_dir = self.j(self.base, 'DLLs')
         shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir,
                 ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*'))
         for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')):
@@ -144,6 +179,24 @@ class Win32Freeze(Command, WixMixIn):
         shutil.copytree(self.j(comext, 'shell'), self.j(sp_dir, 'win32com', 'shell'))
         shutil.rmtree(comext)
 
+        # Fix PyCrypto, removing the bootstrap .py modules that load the .pyd
+        # modules, since they do not work when in a zip file
+        for crypto_dir in glob.glob(self.j(sp_dir, 'pycrypto-*', 'Crypto')):
+            for dirpath, dirnames, filenames in os.walk(crypto_dir):
+                for f in filenames:
+                    name, ext = os.path.splitext(f)
+                    if ext == '.pyd':
+                        with open(self.j(dirpath, name+'.py')) as f:
+                            raw = f.read().strip()
+                        if (not raw.startswith('def __bootstrap__') or not
+                                raw.endswith('__bootstrap__()')):
+                            raise Exception('The PyCrypto file %r has non'
+                                    ' bootstrap code'%self.j(dirpath, f))
+                        for ext in ('.py', '.pyc', '.pyo'):
+                            x = self.j(dirpath, name+ext)
+                            if os.path.exists(x):
+                                os.remove(x)
+
         for pat in (r'PyQt4\uic\port_v3', ):
             x = glob.glob(self.j(self.lib_dir, 'site-packages', pat))[0]
             shutil.rmtree(x)
@@ -194,14 +247,13 @@ class Win32Freeze(Command, WixMixIn):
             if os.path.exists(tg):
                 shutil.rmtree(tg)
             shutil.copytree(imfd, tg)
+        for dirpath, dirnames, filenames in os.walk(tdir):
+            for x in filenames:
+                if not x.endswith('.dll'):
+                    os.remove(self.j(dirpath, x))
 
         print
         print 'Adding third party dependencies'
-        tdir = os.path.join(self.base, 'driver')
-        os.makedirs(tdir)
-        for pat in ('*.dll', '*.sys', '*.cat', '*.inf'):
-            for f in glob.glob(os.path.join(LIBUSB_DIR, pat)):
-                shutil.copyfile(f, os.path.join(tdir, os.path.basename(f)))
         print '\tAdding unrar'
         shutil.copyfile(LIBUNRAR,
                 os.path.join(self.dll_dir, os.path.basename(LIBUNRAR)))
@@ -256,7 +308,7 @@ class Win32Freeze(Command, WixMixIn):
     def embed_resources(self, module, desc=None):
         icon_base = self.j(self.src_root, 'icons')
         icon_map = {'calibre':'library', 'ebook-viewer':'viewer',
-                'lrfviewer':'viewer'}
+                'lrfviewer':'viewer', 'calibre-portable':'library'}
         file_type = 'DLL' if module.endswith('.dll') else 'APP'
         template = open(self.rc_template, 'rb').read()
         bname = self.b(module)
@@ -313,13 +365,67 @@ class Win32Freeze(Command, WixMixIn):
             self.info(p.stderr.read())
             sys.exit(1)
 
+    def build_portable(self):
+        base  = self.portable_base
+        if os.path.exists(base):
+            shutil.rmtree(base)
+        os.makedirs(base)
+        src = self.j(self.src_root, 'setup', 'installer', 'windows',
+                'portable.c')
+        obj = self.j(self.obj_dir, self.b(src)+'.obj')
+        cflags  = '/c /EHsc /MT /W3 /Ox /nologo /D_UNICODE'.split()
+
+        if self.newer(obj, [src]):
+            self.info('Compiling', obj)
+            cmd = [msvc.cc] + cflags + ['/Fo'+obj, '/Tc'+src]
+            self.run_builder(cmd)
+
+        exe = self.j(base, 'calibre-portable.exe')
+        if self.newer(exe, [obj]):
+            self.info('Linking', exe)
+            cmd = [msvc.linker] + ['/INCREMENTAL:NO', '/MACHINE:X86',
+                    '/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:WINDOWS',
+                    '/RELEASE',
+                    '/OUT:'+exe, self.embed_resources(exe),
+                    obj, 'User32.lib']
+            self.run_builder(cmd)
+
+        self.info('Creating portable installer')
+        shutil.copytree(self.base, self.j(base, 'Calibre'))
+        os.mkdir(self.j(base, 'Calibre Library'))
+        os.mkdir(self.j(base, 'Calibre Settings'))
+
+        name = '%s-portable-%s.zip'%(__appname__, __version__)
+        with zipfile.ZipFile(self.j('dist', name), 'w', zipfile.ZIP_DEFLATED) as zf:
+            self.add_dir_to_zip(zf, base, 'Calibre Portable')
+
+    def add_dir_to_zip(self, zf, path, prefix=''):
+        '''
+        Add a directory recursively to the zip file with an optional prefix.
+        '''
+        if prefix:
+            zi = zipfile.ZipInfo(prefix+'/')
+            zi.external_attr = 16
+            zf.writestr(zi, '')
+        cwd = os.path.abspath(os.getcwd())
+        try:
+            os.chdir(path)
+            fp = (prefix + ('/' if prefix else '')).replace('//', '/')
+            for f in os.listdir('.'):
+                arcname = fp + f
+                if os.path.isdir(f):
+                    self.add_dir_to_zip(zf, f, prefix=arcname)
+                else:
+                    zf.write(f, arcname)
+        finally:
+            os.chdir(cwd)
+
     def build_launchers(self):
-        self.obj_dir = self.j(self.src_root, 'build', 'launcher')
         if not os.path.exists(self.obj_dir):
             os.makedirs(self.obj_dir)
         base = self.j(self.src_root, 'setup', 'installer', 'windows')
-        sources = [self.j(base, x) for x in ['util.c']]
+        sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']]
-        headers = [self.j(base, x) for x in ['util.h']]
+        headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']]
         objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources]
         cflags  = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split()
         cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver]
@@ -371,43 +477,49 @@ class Win32Freeze(Command, WixMixIn):
 
     def archive_lib_dir(self):
         self.info('Putting all python code into a zip file for performance')
-        if os.path.exists(self.pydlib):
-            shutil.rmtree(self.pydlib)
-        os.makedirs(self.pydlib)
         self.zf_timestamp = time.localtime(time.time())[:6]
         self.zf_names = set()
         with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf:
+            # Add the .pyds from python and calibre to the zip file
+            for x in (self.plugins_dir, self.dll_dir):
+                for pyd in os.listdir(x):
+                    if pyd.endswith('.pyd') and pyd != 'sqlite_custom.pyd':
+                        # sqlite_custom has to be a file for
+                        # sqlite_load_extension to work
+                        self.add_to_zipfile(zf, pyd, x)
+                        os.remove(self.j(x, pyd))
+
+            # Add everything in Lib except site-packages to the zip file
             for x in os.listdir(self.lib_dir):
                 if x == 'site-packages':
                     continue
                 self.add_to_zipfile(zf, x, self.lib_dir)
 
             sp = self.j(self.lib_dir, 'site-packages')
-            handled = set(['site.pyo'])
+            # Special handling for PIL and pywin32
-            for pth in ('PIL.pth', 'pywin32.pth'):
+            handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32'])
-                handled.add(pth)
+            self.add_to_zipfile(zf, 'PIL', sp)
-                shutil.copyfile(self.j(sp, pth), self.j(self.pydlib, pth))
+            base = self.j(sp, 'win32', 'lib')
-                for d in self.get_pth_dirs(self.j(sp, pth)):
+            for x in os.listdir(base):
-                    shutil.copytree(d, self.j(self.pydlib, self.b(d)), True)
+                if os.path.splitext(x)[1] not in ('.exe',):
-                    handled.add(self.b(d))
+                    self.add_to_zipfile(zf, x, base)
+            base = self.d(base)
+            for x in os.listdir(base):
+                if not os.path.isdir(self.j(base, x)):
+                    if os.path.splitext(x)[1] not in ('.exe',):
+                        self.add_to_zipfile(zf, x, base)
 
             handled.add('easy-install.pth')
             for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')):
                 handled.add(self.b(d))
-                zip_safe = self.is_zip_safe(d)
                 for x in os.listdir(d):
                     if x == 'EGG-INFO':
                         continue
-                    if zip_safe:
                     self.add_to_zipfile(zf, x, d)
-                    else:
-                        absp = self.j(d, x)
-                        dest = self.j(self.pydlib, x)
-                        if os.path.isdir(absp):
-                            shutil.copytree(absp, dest, True)
-                        else:
-                            shutil.copy2(absp, dest)
 
+            # The rest of site-packages
+            # We dont want the site.py from site-packages
+            handled.add('site.pyo')
             for x in os.listdir(sp):
                 if x in handled or x.endswith('.egg-info'):
                     continue
@@ -415,33 +527,18 @@ class Win32Freeze(Command, WixMixIn):
                 if os.path.isdir(absp):
                     if not os.listdir(absp):
                         continue
-                    if self.is_zip_safe(absp):
                     self.add_to_zipfile(zf, x, sp)
-                    else:
-                        shutil.copytree(absp, self.j(self.pydlib, x), True)
-                else:
-                    if x.endswith('.pyd'):
-                        shutil.copy2(absp, self.j(self.pydlib, x))
                 else:
                     self.add_to_zipfile(zf, x, sp)
 
         shutil.rmtree(self.lib_dir)
 
-    def is_zip_safe(self, path):
-        for f in walk(path):
-            ext = os.path.splitext(f)[1].lower()
-            if ext in ('.pyd', '.dll', '.exe'):
-                return False
-        return True
-
     def get_pth_dirs(self, pth):
         base = os.path.dirname(pth)
         for line in open(pth).readlines():
             line = line.strip()
             if not line or line.startswith('#') or line.startswith('import'):
                 continue
-            if line == 'win32\\lib':
-                continue
             candidate = self.j(base, line)
             if os.path.exists(candidate):
                 yield candidate
@@ -463,10 +560,10 @@ class Win32Freeze(Command, WixMixIn):
                     self.add_to_zipfile(zf, name + os.sep + x, base)
         else:
             ext = os.path.splitext(name)[1].lower()
-            if ext in ('.pyd', '.dll', '.exe'):
+            if ext in ('.dll',):
                 raise ValueError('Cannot add %r to zipfile'%abspath)
             zinfo.external_attr = 0600 << 16
-            if ext in ('.py', '.pyc', '.pyo'):
+            if ext in ('.py', '.pyc', '.pyo', '.pyd'):
                 with open(abspath, 'rb') as f:
                     zf.writestr(zinfo, f.read())
 

setup/installer/windows/notes.rst

@@ -88,7 +88,9 @@ Qt uses its own routine to locate and load "system libraries" including the open
 
 Now, run configure and make::
 
-    configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake
+-no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly
+
+    configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -no-plugin-manifests -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake
 
 SIP
 -----

setup/installer/windows/portable.c

@@ -0,0 +1,151 @@
+#ifndef UNICODE
+#define UNICODE
+#endif 
+
+#include <windows.h>
+#include <tchar.h>
+#include <stdio.h>
+
+#define BUFSIZE 4096
+
+void show_error(LPCTSTR msg) {
+    MessageBeep(MB_ICONERROR);
+    MessageBox(NULL, msg, TEXT("Error"), MB_OK|MB_ICONERROR);
+}
+
+void show_detailed_error(LPCTSTR preamble, LPCTSTR msg, int code) {
+    LPTSTR buf;
+    buf = (LPTSTR)LocalAlloc(LMEM_ZEROINIT, sizeof(TCHAR)*
+            (_tcslen(msg) + _tcslen(preamble) + 80));
+
+    _sntprintf_s(buf, 
+        LocalSize(buf) / sizeof(TCHAR), _TRUNCATE,
+        TEXT("%s\r\n  %s (Error Code: %d)\r\n"), 
+        preamble, msg, code);
+
+    show_error(buf);
+    LocalFree(buf);
+}
+
+void show_last_error_crt(LPCTSTR preamble) {
+    TCHAR buf[BUFSIZE];
+    int err = 0;
+
+    _get_errno(&err);
+    _wcserror_s(buf, BUFSIZE, err);
+    show_detailed_error(preamble, buf, err);
+}
+
+void show_last_error(LPCTSTR preamble) {
+    TCHAR *msg = NULL;
+    DWORD dw = GetLastError(); 
+
+    FormatMessage(
+        FORMAT_MESSAGE_ALLOCATE_BUFFER | 
+        FORMAT_MESSAGE_FROM_SYSTEM |
+        FORMAT_MESSAGE_IGNORE_INSERTS,
+        NULL,
+        dw,
+        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+        &msg,
+        0, NULL );
+
+    show_detailed_error(preamble, msg, (int)dw);
+}
+
+
+LPTSTR get_app_dir() {
+    LPTSTR buf, buf2, buf3;
+    DWORD sz;
+    TCHAR drive[4] = TEXT("\0\0\0");
+    errno_t err;
+
+    buf = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+    buf2 = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+    buf3 = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+
+    sz = GetModuleFileName(NULL, buf, BUFSIZE);
+
+    if (sz == 0 || sz > BUFSIZE-1) {
+        show_error(TEXT("Failed to get path to calibre-portable.exe"));
+        ExitProcess(1);
+    }
+
+    err = _tsplitpath_s(buf, drive, 4, buf2, BUFSIZE, NULL, 0, NULL, 0);
+
+    if (err != 0) {
+        show_last_error_crt(TEXT("Failed to split path to calibre-portable.exe"));
+        ExitProcess(1);
+    }
+
+    _sntprintf_s(buf3, BUFSIZE-1, _TRUNCATE, TEXT("%s%s"), drive, buf2);
+    free(buf); free(buf2);
+    return buf3;
+}
+
+void launch_calibre(LPCTSTR exe, LPCTSTR config_dir, LPCTSTR library_dir) {
+    DWORD dwFlags=0;
+    STARTUPINFO si;
+    PROCESS_INFORMATION pi;
+    BOOL fSuccess; 
+    TCHAR cmdline[BUFSIZE];
+
+    if (! SetEnvironmentVariable(TEXT("CALIBRE_CONFIG_DIRECTORY"), config_dir)) {
+        show_last_error(TEXT("Failed to set environment variables"));
+        ExitProcess(1);
+    }
+
+    if (! SetEnvironmentVariable(TEXT("CALIBRE_PORTABLE_BUILD"), exe)) {
+        show_last_error(TEXT("Failed to set environment variables"));
+        ExitProcess(1);
+    }
+
+    dwFlags = CREATE_UNICODE_ENVIRONMENT | CREATE_NEW_PROCESS_GROUP;
+    _sntprintf_s(cmdline, BUFSIZE, _TRUNCATE, TEXT(" \"--with-library=%s\""), library_dir);
+
+    ZeroMemory( &si, sizeof(si) );
+    si.cb = sizeof(si);
+    ZeroMemory( &pi, sizeof(pi) );
+
+    fSuccess = CreateProcess(exe, cmdline,
+        NULL,           // Process handle not inheritable
+        NULL,           // Thread handle not inheritable
+        FALSE,          // Set handle inheritance to FALSE
+        dwFlags,        // Creation flags http://msdn.microsoft.com/en-us/library/ms684863(v=vs.85).aspx
+        NULL,           // Use parent's environment block
+        NULL,           // Use parent's starting directory 
+        &si,            // Pointer to STARTUPINFO structure
+        &pi             // Pointer to PROCESS_INFORMATION structure
+    );
+
+    if (fSuccess == 0) {
+        show_last_error(TEXT("Failed to launch the calibre program"));
+    }
+
+    // Close process and thread handles.
+    CloseHandle( pi.hProcess );
+    CloseHandle( pi.hThread );
+
+}
+
+int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, PWSTR pCmdLine, int nCmdShow)
+{
+    LPTSTR app_dir, config_dir, exe, library_dir;
+
+    app_dir = get_app_dir();
+    config_dir = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+    library_dir = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+    exe = (LPTSTR)calloc(BUFSIZE, sizeof(TCHAR));
+
+    _sntprintf_s(config_dir, BUFSIZE, _TRUNCATE, TEXT("%sCalibre Settings"), app_dir);
+    _sntprintf_s(exe, BUFSIZE, _TRUNCATE, TEXT("%sCalibre\\calibre.exe"), app_dir);
+    _sntprintf_s(library_dir, BUFSIZE, _TRUNCATE, TEXT("%sCalibre Library"), app_dir);
+
+    launch_calibre(exe, config_dir, library_dir);
+
+    free(app_dir); free(config_dir); free(exe); free(library_dir);
+
+    return 0;
+}
+
+

setup/installer/windows/site.py

@@ -1,12 +1,72 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, os, linecache
+import sys
+import os
+import zipimport
+import _memimporter
+
+DEBUG_ZIPIMPORT = False
+
+class ZipExtensionImporter(zipimport.zipimporter):
+    '''
+    Taken, with thanks, from the py2exe source code
+    '''
+
+    def __init__(self, *args, **kwargs):
+        zipimport.zipimporter.__init__(self, *args, **kwargs)
+        # We know there are no dlls in the zip file, so dont set findproc
+        # (performance optimization)
+        #_memimporter.set_find_proc(self.locate_dll_image)
+
+    def find_module(self, fullname, path=None):
+        result = zipimport.zipimporter.find_module(self, fullname, path)
+        if result:
+            return result
+        fullname = fullname.replace(".", "\\")
+        if (fullname + '.pyd') in self._files:
+            return self
+        return None
+
+    def locate_dll_image(self, name):
+        # A callback function for_memimporter.import_module.  Tries to
+        # locate additional dlls.  Returns the image as Python string,
+        # or None if not found.
+        if name in self._files:
+            return self.get_data(name)
+        return None
+
+    def load_module(self, fullname):
+        if sys.modules.has_key(fullname):
+            mod = sys.modules[fullname]
+            if DEBUG_ZIPIMPORT:
+                sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive))
+            return mod
+        try:
+            return zipimport.zipimporter.load_module(self, fullname)
+        except zipimport.ZipImportError:
+            pass
+        initname = "init" + fullname.split(".")[-1] # name of initfunction
+        filename = fullname.replace(".", "\\")
+        path = filename + '.pyd'
+        if path in self._files:
+            if DEBUG_ZIPIMPORT:
+                sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive))
+            code = self.get_data(path)
+            mod = _memimporter.import_module(code, initname, fullname, path)
+            mod.__file__ = "%s\\%s" % (self.archive, path)
+            mod.__loader__ = self
+            if DEBUG_ZIPIMPORT:
+                sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__))
+            return mod
+        raise zipimport.ZipImportError, "can't find module %s" % fullname
+
+    def __repr__(self):
+        return "<%s object %r>" % (self.__class__.__name__, self.archive)
 
 
 def abs__file__():
@@ -32,7 +92,7 @@ def aliasmbcs():
 
 def add_calibre_vars():
     sys.resources_location = os.path.join(sys.app_dir, 'resources')
-    sys.extensions_location = os.path.join(sys.app_dir, 'plugins')
+    sys.extensions_location = os.path.join(sys.app_dir, 'plugins2')
 
     dv = os.environ.get('CALIBRE_DEVELOP_FROM', None)
     if dv and os.path.exists(dv):
@@ -42,42 +102,6 @@ def makepath(*paths):
     dir = os.path.abspath(os.path.join(*paths))
     return dir, os.path.normcase(dir)
 
-def addpackage(sitedir, name):
-    """Process a .pth file within the site-packages directory:
-       For each line in the file, either combine it with sitedir to a path,
-       or execute it if it starts with 'import '.
-    """
-    fullname = os.path.join(sitedir, name)
-    try:
-        f = open(fullname, "rU")
-    except IOError:
-        return
-    with f:
-        for line in f:
-            if line.startswith("#"):
-                continue
-            if line.startswith(("import ", "import\t")):
-                exec line
-                continue
-            line = line.rstrip()
-            dir, dircase = makepath(sitedir, line)
-            if os.path.exists(dir):
-                sys.path.append(dir)
-
-
-def addsitedir(sitedir):
-    """Add 'sitedir' argument to sys.path if missing and handle .pth files in
-    'sitedir'"""
-    sitedir, sitedircase = makepath(sitedir)
-    try:
-        names = os.listdir(sitedir)
-    except os.error:
-        return
-    dotpth = os.extsep + "pth"
-    names = [name for name in names if name.endswith(dotpth)]
-    for name in sorted(names):
-        addpackage(sitedir, name)
-
 def run_entry_point():
     bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function
     sys.argv[0] = bname+'.exe'
@@ -89,6 +113,10 @@ def main():
     sys.setdefaultencoding('utf-8')
     aliasmbcs()
 
+    sys.path_hooks.insert(0, ZipExtensionImporter)
+    sys.path_importer_cache.clear()
+
+    import linecache
     def fake_getline(filename, lineno, module_globals=None):
         return ''
     linecache.orig_getline = linecache.getline
@@ -96,10 +124,11 @@ def main():
 
     abs__file__()
 
-    addsitedir(os.path.join(sys.app_dir, 'pydlib'))
-
     add_calibre_vars()
 
+    # Needed for pywintypes to be able to load its DLL
+    sys.path.append(os.path.join(sys.app_dir, 'DLLs'))
+
     return run_entry_point()
 
 

setup/installer/windows/util.c

@@ -1,18 +1,130 @@
 /*
  * Copyright 2009 Kovid Goyal
+ * The memimporter code is taken from the py2exe project
  */
 
 #include "util.h"
+
 #include <delayimp.h>
 #include <io.h>
 #include <fcntl.h>
 
+
 static char GUI_APP = 0;
 static char python_dll[] = PYDLL;
 
 void set_gui_app(char yes) { GUI_APP = yes; }
 char is_gui_app() { return GUI_APP; }
 
+
+// memimporter {{{
+
+#include "MemoryModule.h"
+
+static char **DLL_Py_PackageContext = NULL;
+static PyObject **DLL_ImportError = NULL;
+static char module_doc[] =
+"Importer which can load extension modules from memory";
+
+
+static void *memdup(void *ptr, Py_ssize_t size)
+{
+	void *p = malloc(size);
+	if (p == NULL)
+		return NULL;
+	memcpy(p, ptr, size);
+	return p;
+}
+
+/*
+  Be sure to detect errors in FindLibrary - undetected errors lead to
+  very strange behaviour.
+*/
+static void* FindLibrary(char *name, PyObject *callback)
+{
+	PyObject *result;
+	char *p;
+	Py_ssize_t size;
+
+	if (callback == NULL)
+		return NULL;
+	result = PyObject_CallFunction(callback, "s", name);
+	if (result == NULL) {
+		PyErr_Clear();
+		return NULL;
+	}
+	if (-1 == PyString_AsStringAndSize(result, &p, &size)) {
+		PyErr_Clear();
+		Py_DECREF(result);
+		return NULL;
+	}
+	p = memdup(p, size);
+	Py_DECREF(result);
+	return p;
+}
+
+static PyObject *set_find_proc(PyObject *self, PyObject *args)
+{
+	PyObject *callback = NULL;
+	if (!PyArg_ParseTuple(args, "|O:set_find_proc", &callback))
+		return NULL;
+	Py_DECREF((PyObject *)findproc_data);
+	Py_INCREF(callback);
+	findproc_data = (void *)callback;
+    return Py_BuildValue("i", 1);
+}
+
+static PyObject *
+import_module(PyObject *self, PyObject *args)
+{
+	char *data;
+	int size;
+	char *initfuncname;
+	char *modname;
+	char *pathname;
+	HMEMORYMODULE hmem;
+	FARPROC do_init;
+
+	char *oldcontext;
+
+	/* code, initfuncname, fqmodulename, path */
+	if (!PyArg_ParseTuple(args, "s#sss:import_module",
+			      &data, &size,
+			      &initfuncname, &modname, &pathname))
+		return NULL;
+	hmem = MemoryLoadLibrary(pathname, data);
+	if (!hmem) {
+		PyErr_Format(*DLL_ImportError,
+			     "MemoryLoadLibrary() failed loading %s", pathname);
+		return NULL;
+	}
+	do_init = MemoryGetProcAddress(hmem, initfuncname);
+	if (!do_init) {
+		MemoryFreeLibrary(hmem);
+		PyErr_Format(*DLL_ImportError,
+			     "Could not find function %s in memory loaded pyd", initfuncname);
+		return NULL;
+	}
+
+    oldcontext = *DLL_Py_PackageContext;
+	*DLL_Py_PackageContext = modname;
+	do_init();
+	*DLL_Py_PackageContext = oldcontext;
+	if (PyErr_Occurred())
+		return NULL;
+	/* Retrieve from sys.modules */
+	return PyImport_ImportModule(modname);
+}
+
+static PyMethodDef methods[] = {
+	{ "import_module", import_module, METH_VARARGS,
+	  "import_module(code, initfunc, dllname[, finder]) -> module" },
+	{ "set_find_proc", set_find_proc, METH_VARARGS },
+	{ NULL, NULL },		/* Sentinel */
+};
+
+// }}}
+
 static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
     wchar_t *buf, *cbuf;
     buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)*
@@ -61,7 +173,7 @@ int show_last_error(wchar_t *preamble) {
         NULL,
         dw,
         MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-        msg,
+        &msg,
         0, NULL );
 
     return _show_error(preamble, msg, (int)dw);
@@ -185,7 +297,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     char *dummy_argv[1] = {""};
 
     buf  = (char*)calloc(MAX_PATH, sizeof(char));
-    path = (char*)calloc(3*MAX_PATH, sizeof(char));
+    path = (char*)calloc(MAX_PATH, sizeof(char));
     if (!buf || !path) ExitProcess(_show_error(L"Out of memory", L"", 1));
 
     sz = GetModuleFileNameA(NULL, buf, MAX_PATH);
@@ -198,8 +310,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     buf[strlen(buf)-1] = '\0';
 
     _snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf);
-    _snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\pylib.zip;%s\\pydlib;%s\\DLLs",
+    _snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\pylib.zip", buf);
-            buf, buf, buf);
     free(buf);
 
 
@@ -227,7 +338,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1));
     //*flag = 1;
 
-
+    DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext");
+    if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1));
+    DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError");
+    if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1));
 
     Py_SetProgramName(program_name);
     Py_SetPythonHome(python_home);
@@ -263,6 +377,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
         PyList_SetItem(argv, i, v);
     }
     PySys_SetObject("argv", argv);
+
+    findproc = FindLibrary;
+	Py_InitModule3("_memimporter", methods, module_doc);
+
 }
 
 

setup/installer/windows/wix-template.xml

@@ -11,6 +11,10 @@
 						SummaryCodepage='1252' />
 
                 <Media Id="1" Cabinet="{app}.cab" CompressionLevel="{compression}" EmbedCab="yes" />
+                <!-- The following line ensures that DLLs are replaced even if their version is the same as before. This
+                     is necessary because of the manifest nuking that was part of making calibre isolated. But I think it
+                     is more rigorous anyway. -->
+                <Property Id='REINSTALLMODE' Value='emus'/>
 
                 <Upgrade Id="{upgrade_code}">
                     <UpgradeVersion Maximum="{version}"
@@ -33,7 +37,6 @@
                </Property>
 
                 <Directory Id='TARGETDIR' Name='SourceDir'>
-                        <Merge Id="VCRedist" SourceFile="{crt_msm}" DiskId="1" Language="0"/>
 						<Directory Id='ProgramFilesFolder' Name='PFiles'>
 								<Directory Id='APPLICATIONFOLDER' Name='{app}' />
 						</Directory>
@@ -100,10 +103,6 @@
                             <ComponentRef Id="RememberInstallDir"/>
                         </Feature>
 
-                        <Feature Id="VCRedist" Title="Visual C++ 8.0 Runtime" AllowAdvertise="no" Display="hidden" Level="1">
-                            <MergeRef Id="VCRedist"/>
-                        </Feature>
-
                         <Feature Id="FSMS" Title="Start menu shortcuts" Level="1"
                             Description="Program shortcuts installed in the Start Menu">
                             <ComponentRef Id="StartMenuShortcuts"/>
@@ -149,12 +148,13 @@
                 Set default folder name and allow only per machine installs.
                 For a per-machine installation, the default installation location
                 will be [ProgramFilesFolder][ApplicationFolderName] and the user
-                will be able to change it in the setup UI. This is because the installer
+                will be able to change it in the setup UI. This is no longer necessary
-                has to install the VC90 merge module into the system winsxs folder for python
+                (i.e. per user installs should work) but left this way as I
-                to work, so per user installs are impossible anyway.
+                dont want to deal with the complications 
                 -->
                 <Property Id="ApplicationFolderName" Value="Calibre2" />
                 <Property Id="WixAppFolder" Value="WixPerMachineFolder" />
+                <Property Id="ALLUSERS" Value="1" />
                 <WixVariable Id="WixUISupportPerUser" Value="0" />
 
                 <!-- Add option to launch calibre after install -->
@@ -164,10 +164,6 @@
                 <CustomAction Id="LaunchApplication" BinaryKey="WixCA"
                     DllEntry="WixShellExec" Impersonate="yes"/>
 
-                <InstallUISequence>
-                    <FileCost Suppress="yes" />
-                </InstallUISequence>
-
 		</Product>
 </Wix>
 

setup/installer/windows/wix.py

@@ -35,7 +35,6 @@ class WixMixIn:
                 exe_map           = self.smap,
                 main_icon         = self.j(self.src_root, 'icons', 'library.ico'),
                 web_icon          = self.j(self.src_root, 'icons', 'web.ico'),
-                crt_msm           = self.j(self.SW, 'Microsoft_VC90_CRT_x86.msm')
                 )
         template = open(self.j(self.d(__file__), 'en-us.xml'),
                 'rb').read()

setup/pygettext.py

@@ -187,7 +187,6 @@ msgstr ""
 
 '''%dict(appname=__appname__, version=version, year=time.strftime('%Y'))
 
-
 def usage(code, msg=''):
     print >> sys.stderr, __doc__ % globals()
     if msg:

setup/translations.py

@@ -85,7 +85,7 @@ class Translations(POT):
 
     def mo_file(self, po_file):
         locale = os.path.splitext(os.path.basename(po_file))[0]
-        return locale, os.path.join(self.DEST, locale, 'LC_MESSAGES', 'messages.mo')
+        return locale, os.path.join(self.DEST, locale, 'messages.mo')
 
 
     def run(self, opts):
@@ -94,7 +94,6 @@ class Translations(POT):
             base = os.path.dirname(dest)
             if not os.path.exists(base):
                 os.makedirs(base)
-            if self.newer(dest, f):
             self.info('\tCompiling translations for', locale)
             subprocess.check_call(['msgfmt', '-o', dest, f])
             if locale in ('en_GB', 'nds', 'te', 'yi'):
@@ -123,6 +122,16 @@ class Translations(POT):
                 shutil.copy2(f, dest)
 
         self.write_stats()
+        self.freeze_locales()
+
+    def freeze_locales(self):
+        zf = self.DEST + '.zip'
+        from calibre import CurrentDir
+        from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED
+        with ZipFile(zf, 'w', ZIP_DEFLATED) as zf:
+            with CurrentDir(self.DEST):
+                zf.add_dir('.')
+        shutil.rmtree(self.DEST)
 
     @property
     def stats(self):

setup/upload.py

@@ -26,6 +26,7 @@ def installers():
     installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2')))
     installers.append(installer_name('tar.bz2', is64bit=True))
     installers.insert(0, 'dist/%s-%s.tar.gz'%(__appname__, __version__))
+    installers.append('dist/%s-portable-%s.zip'%(__appname__, __version__))
     return installers
 
 def installer_description(fname):
@@ -38,6 +39,8 @@ def installer_description(fname):
         return 'Windows installer'
     if fname.endswith('.dmg'):
         return 'OS X dmg'
+    if fname.endswith('.zip'):
+        return 'Calibre Portable'
     return 'Unknown file'
 
 class ReUpload(Command): # {{{
@@ -90,9 +93,11 @@ class UploadToGoogleCode(Command): # {{{
 
     def upload_one(self, fname):
         self.info('Uploading', fname)
-        typ = 'Type-Source' if fname.endswith('.gz') else 'Type-Installer'
+        typ = 'Type-' + ('Source' if fname.endswith('.gz') else 'Archive' if
+                fname.endswith('.zip') else 'Installer')
         ext = os.path.splitext(fname)[1][1:]
-        op  = 'OpSys-'+{'msi':'Windows','dmg':'OSX','bz2':'Linux','gz':'All'}[ext]
+        op  = 'OpSys-'+{'msi':'Windows','zip':'Windows',
+                'dmg':'OSX','bz2':'Linux','gz':'All'}[ext]
         desc = installer_description(fname)
         start = time.time()
         path = self.upload(os.path.abspath(fname), desc,

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 3)
+numeric_version = (0, 8, 5)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 
@@ -32,6 +32,7 @@ isbsd = isfreebsd or isnetbsd
 islinux   = not(iswindows or isosx or isbsd)
 isfrozen  = hasattr(sys, 'frozen')
 isunix = isosx or islinux
+isportable = os.environ.get('CALIBRE_PORTABLE_BUILD', None) is not None
 
 try:
     preferred_encoding = locale.getpreferredencoding()

src/calibre/customize/builtins.py

@@ -586,15 +586,15 @@ from calibre.devices.apple.driver import ITUNES
 from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA
 from calibre.devices.blackberry.driver import BLACKBERRY
 from calibre.devices.cybook.driver import CYBOOK, ORIZON
-from calibre.devices.eb600.driver import EB600, COOL_ER, SHINEBOOK, \
+from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK,
-                POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, \
+                POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK,
-                BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602, \
+                BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602,
-                POCKETBOOK701
+                POCKETBOOK701, POCKETBOOK360P)
 from calibre.devices.iliad.driver import ILIAD
 from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800
 from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI
 from calibre.devices.kindle.driver import KINDLE, KINDLE2, KINDLE_DX
-from calibre.devices.nook.driver import NOOK, NOOK_COLOR, NOOK_TSR
+from calibre.devices.nook.driver import NOOK, NOOK_COLOR
 from calibre.devices.prs505.driver import PRS505
 from calibre.devices.user_defined.driver import USER_DEFINED
 from calibre.devices.android.driver import ANDROID, S60
@@ -603,14 +603,15 @@ from calibre.devices.eslick.driver import ESLICK, EBK52
 from calibre.devices.nuut2.driver import NUUT2
 from calibre.devices.iriver.driver import IRIVER_STORY
 from calibre.devices.binatone.driver import README
-from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA, THEBOOK
+from calibre.devices.hanvon.driver import (N516, EB511, ALEX, AZBOOKA, THEBOOK,
+        LIBREAIR)
 from calibre.devices.edge.driver import EDGE
-from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
+from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS,
-        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER
+        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER)
 from calibre.devices.sne.driver import SNE
-from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, \
+from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL,
-        GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, \
+        GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR,
-        TREKSTOR, EEEREADER, NEXTBOOK
+        TREKSTOR, EEEREADER, NEXTBOOK, ADAM)
 from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 from calibre.devices.kobo.driver import KOBO
 from calibre.devices.bambook.driver import BAMBOOK
@@ -689,11 +690,11 @@ plugins += [
     JETBOOK_MINI,
     MIBUK,
     SHINEBOOK,
-    POCKETBOOK360, POCKETBOOK301, POCKETBOOK602, POCKETBOOK701,
+    POCKETBOOK360, POCKETBOOK301, POCKETBOOK602, POCKETBOOK701, POCKETBOOK360P,
     KINDLE,
     KINDLE2,
     KINDLE_DX,
-    NOOK, NOOK_COLOR, NOOK_TSR,
+    NOOK, NOOK_COLOR,
     PRS505,
     ANDROID,
     S60,
@@ -716,7 +717,7 @@ plugins += [
     EB600,
     README,
     N516,
-    THEBOOK,
+    THEBOOK, LIBREAIR,
     EB511,
     ELONEX,
     TECLAST_K3,
@@ -744,6 +745,7 @@ plugins += [
     TREKSTOR,
     EEEREADER,
     NEXTBOOK,
+    ADAM,
     ITUNES,
     BOEYE_BEX,
     BOEYE_BDX,
@@ -865,13 +867,20 @@ class ActionStore(InterfaceActionBase):
         from calibre.gui2.store.config.store import save_settings as save
         save(config_widget)
 
+class ActionPluginUpdates(InterfaceActionBase):
+    name = 'Plugin Updates'
+    author = 'Grant Drake'
+    description = 'Queries the MobileRead forums for updates to plugins to install'
+    actual_plugin = 'calibre.gui2.actions.plugin_updates:PluginUpdatesAction'
+
 plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
         ActionConvert, ActionDelete, ActionEditMetadata, ActionView,
         ActionFetchNews, ActionSaveToDisk, ActionShowBookDetails,
         ActionRestart, ActionOpenFolder, ActionConnectShare,
         ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks,
         ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary,
-        ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore]
+        ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore,
+        ActionPluginUpdates]
 
 # }}}
 
@@ -1417,6 +1426,15 @@ class StoreWoblinkStore(StoreBase):
     headquarters = 'PL'
     formats = ['EPUB']
 
+class StoreZixoStore(StoreBase):
+    name = 'Zixo'
+    author = u'Tomasz Długosz'
+    description = u'Księgarnia z ebookami oraz książkami audio. Aby otwierać książki w formacie Zixo należy zainstalować program dostępny na stronie księgarni. Umożliwia on m.in. dodawanie zakładek i dostosowywanie rozmiaru czcionki.'
+    actual_plugin = 'calibre.gui2.store.zixo_plugin:ZixoStore'
+
+    headquarters = 'PL'
+    formats = ['PDF, ZIXO']
+
 plugins += [
     StoreArchiveOrgStore,
     StoreAmazonKindleStore,
@@ -1451,7 +1469,8 @@ plugins += [
     StoreWeightlessBooksStore,
     StoreWHSmithUKStore,
     StoreWizardsTowerBooksStore,
-    StoreWoblinkStore
+    StoreWoblinkStore,
+    StoreZixoStore
 ]
 
 # }}}

src/calibre/customize/ui.py

@@ -355,11 +355,17 @@ def remove_plugin(plugin_or_name):
     name = getattr(plugin_or_name, 'name', plugin_or_name)
     plugins = config['plugins']
     removed = False
-    if name in plugins.keys():
+    if name in plugins:
         removed = True
+        try:
+            zfp = os.path.join(plugin_dir, name+'.zip')
+            if os.path.exists(zfp):
+                os.remove(zfp)
             zfp = plugins[name]
             if os.path.exists(zfp):
                 os.remove(zfp)
+        except:
+            pass
         plugins.pop(name)
     config['plugins'] = plugins
     initialize_plugins()
@@ -487,6 +493,8 @@ def initialize_plugin(plugin, path_to_zip_file):
         raise InvalidPlugin((_('Initialization of plugin %s failed with traceback:')
                             %tb) + '\n'+tb)
 
+def has_external_plugins():
+    return bool(config['plugins'])
 
 def initialize_plugins():
     global _initialized_plugins
@@ -495,8 +503,15 @@ def initialize_plugins():
             builtin_names]
     for p in conflicts:
         remove_plugin(p)
-    for zfp in list(config['plugins'].itervalues()) + builtin_plugins:
+    external_plugins = config['plugins']
+    for zfp in list(external_plugins) + builtin_plugins:
         try:
+            if not isinstance(zfp, type):
+                # We have a plugin name
+                pname = zfp
+                zfp = os.path.join(plugin_dir, zfp+'.zip')
+                if not os.path.exists(zfp):
+                    zfp = external_plugins[pname]
             try:
                 plugin = load_plugin(zfp) if not isinstance(zfp, type) else zfp
             except PluginNotFound:

src/calibre/debug.py

@@ -53,6 +53,8 @@ Run an embedded python interpreter.
             default=False, action='store_true')
     parser.add_option('-m', '--inspect-mobi',
             help='Inspect the MOBI file at the specified path', default=None)
+    parser.add_option('--test-build', help='Test binary modules in build',
+            action='store_true', default=False)
 
     return parser
 
@@ -232,6 +234,9 @@ def main(args=sys.argv):
     elif opts.inspect_mobi is not None:
         from calibre.ebooks.mobi.debug import inspect_mobi
         inspect_mobi(opts.inspect_mobi)
+    elif opts.test_build:
+        from calibre.test_build import test
+        test()
     else:
         from calibre import ipython
         ipython()

src/calibre/devices/android/driver.py

@@ -52,7 +52,9 @@ class ANDROID(USBMS):
             0x04e8 : { 0x681d : [0x0222, 0x0223, 0x0224, 0x0400],
                        0x681c : [0x0222, 0x0224, 0x0400],
                        0x6640 : [0x0100],
+                       0x685b : [0x0400],
                        0x685e : [0x0400],
+                       0x6860 : [0x0400],
                        0x6877 : [0x0400],
                      },
 
@@ -92,6 +94,9 @@ class ANDROID(USBMS):
             # CREEL?? Also Nextbook
             0x5e3 : { 0x726 : [0x222] },
 
+            # ZTE
+            0x19d2 : { 0x1353 : [0x226] },
+
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
     EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '
@@ -102,7 +107,7 @@ class ANDROID(USBMS):
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
             'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA',
-            'GENERIC-']
+            'GENERIC-', 'ZTE']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',

src/calibre/devices/apple/driver.py

@@ -2392,6 +2392,16 @@ class ITUNES(DriverBase):
                 self.iTunes.Windows[0].Minimized = True
             self.initial_status = 'launched'
 
+            try:
+                # Pre-emptive test to confirm functional iTunes automation interface
+                foo = self.iTunes.Version
+                foo
+            except:
+                self.iTunes = None
+                raise OpenFeedback('Unable to connect to iTunes.\n' +
+                             ' iTunes automation interface non-responsive, ' +
+                             'recommend reinstalling iTunes')
+
             # Read the current storage path for iTunes media from the XML file
             media_dir = ''
             string = None
@@ -2988,7 +2998,6 @@ class ITUNES(DriverBase):
             newmi = book
         return newmi
 
-
 class ITUNES_ASYNC(ITUNES):
     '''
     This subclass allows the user to interact directly with iTunes via a menu option

src/calibre/devices/eb600/driver.py

@@ -246,6 +246,16 @@ class POCKETBOOK602(USBMS):
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['PB602', 'PB603', 'PB902',
             'PB903', 'PB']
 
+class POCKETBOOK360P(POCKETBOOK602):
+
+    name = 'PocketBook 360+ Device Interface'
+    description    = _('Communicate with the PocketBook 360+ reader.')
+    BCD         = [0x0323]
+    EBOOK_DIR_MAIN = ''
+
+    VENDOR_NAME = '__POCKET'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'BOOK_USB_STORAGE'
+
 class POCKETBOOK701(USBMS):
 
     name = 'PocketBook 701 Device Interface'

src/calibre/devices/hanvon/driver.py

@@ -52,6 +52,18 @@ class THEBOOK(N516):
     EBOOK_DIR_MAIN = 'My books'
     WINDOWS_CARD_A_MEM = '_FILE-STOR_GADGE'
 
+class LIBREAIR(N516):
+    name = 'Libre Air Driver'
+    gui_name = 'Libre Air'
+    description    = _('Communicate with the Libre Air reader.')
+    author         = 'Kovid Goyal'
+    FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'rtf', 'txt', 'pdf']
+
+    BCD = [0x399]
+    VENDOR_NAME      = 'ALURATEK'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '_FILE-STOR_GADGET'
+    EBOOK_DIR_MAIN = 'Books'
+
 class ALEX(N516):
 
     name = 'Alex driver'

src/calibre/devices/kobo/driver.py

@@ -21,7 +21,7 @@ class KOBO(USBMS):
     name = 'Kobo Reader Device Interface'
     gui_name = 'Kobo Reader'
     description = _('Communicate with the Kobo Reader')
-    author = 'Timothy Legge and Kovid Goyal'
+    author = 'Timothy Legge'
     version = (1, 0, 9)
 
     dbversion = 0
@@ -37,8 +37,8 @@ class KOBO(USBMS):
     CAN_SET_METADATA = ['collections']
 
     VENDOR_ID   = [0x2237]
-    PRODUCT_ID  = [0x4161]
+    PRODUCT_ID  = [0x4161, 0x4163]
-    BCD         = [0x0110, 0x0323]
+    BCD         = [0x0110, 0x0323, 0x0326]
 
     VENDOR_NAME = ['KOBO_INC', 'KOBO']
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['.KOBOEREADER', 'EREADER']

src/calibre/devices/misc.py

@@ -224,13 +224,16 @@ class TREKSTOR(USBMS):
     FORMATS     = ['epub', 'txt', 'pdf']
 
     VENDOR_ID   = [0x1e68]
-    PRODUCT_ID  = [0x0041, 0x0042]
+    PRODUCT_ID  = [0x0041, 0x0042,
+            0x003e # This is for the EBOOK_PLAYER_5M https://bugs.launchpad.net/bugs/792091
+            ]
     BCD         = [0x0002]
 
     EBOOK_DIR_MAIN = 'Ebooks'
 
     VENDOR_NAME = 'TREKSTOR'
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_PLAYER_7'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['EBOOK_PLAYER_7',
+            'EBOOK_PLAYER_5M']
 
 class EEEREADER(USBMS):
 
@@ -252,6 +255,28 @@ class EEEREADER(USBMS):
     VENDOR_NAME = 'LINUX'
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET'
 
+class ADAM(USBMS):
+
+    name = 'Notion Ink Adam device interface'
+    gui_name = 'Adam'
+
+    description    = _('Communicate with the Adam tablet')
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['epub', 'pdf', 'doc']
+
+    VENDOR_ID   = [0x0955]
+    PRODUCT_ID  = [0x7100]
+    BCD         = [0x9999]
+
+    EBOOK_DIR_MAIN = 'eBooks'
+
+    VENDOR_NAME = 'NI'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['ADAM']
+    SUPPORTS_SUB_DIRS = True
+
 class NEXTBOOK(USBMS):
 
     name           = 'Nextbook device interface'

src/calibre/devices/nook/driver.py

@@ -77,44 +77,31 @@ class NOOK(USBMS):
         with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile:
             coverfile.write(coverdata)
 
-
     def sanitize_path_components(self, components):
         return [x.replace('#', '_') for x in components]
 
 class NOOK_COLOR(NOOK):
-    gui_name       = _('Nook Color')
+    description    = _('Communicate with the Nook Color and TSR eBook readers.')
-    description    = _('Communicate with the Nook Color eBook reader.')
 
-    PRODUCT_ID  = [0x002]
+    PRODUCT_ID  = [0x002, 0x003]
     BCD         = [0x216]
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_DISK'
 
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_DISK'
     EBOOK_DIR_MAIN = 'My Files'
 
+    def upload_cover(self, path, filename, metadata, filepath):
+        pass
+
+    def get_carda_ebook_dir(self, for_upload=False):
+        if for_upload:
+            return self.EBOOK_DIR_MAIN
+        return ''
+
     def create_upload_path(self, path, mdata, fname, create_dirs=True):
-        filepath = NOOK.create_upload_path(self, path, mdata, fname,
+        is_news = mdata.tags and _('News') in mdata.tags
-                create_dirs=False)
+        subdir = 'Magazines' if is_news else 'Books'
-        edm = self.EBOOK_DIR_MAIN
+        path = os.path.join(path, subdir)
-        subdir = 'Books'
+        return USBMS.create_upload_path(self, path, mdata, fname,
-        if mdata.tags:
+                create_dirs=create_dirs)
-            if _('News') in mdata.tags:
-                subdir = 'Magazines'
-        filepath = filepath.replace(os.sep+edm+os.sep,
-                os.sep+edm+os.sep+subdir+os.sep)
-        filedir = os.path.dirname(filepath)
-        if create_dirs and not os.path.exists(filedir):
-            os.makedirs(filedir)
-
-        return filepath
-
-class NOOK_TSR(NOOK):
-    gui_name       = _('Nook Simple')
-    description    = _('Communicate with the Nook TSR eBook reader.')
-
-    PRODUCT_ID  = [0x003]
-    BCD         = [0x216]
-
-    EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'My Files/Books'
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_DISK'
 
 

src/calibre/devices/usbms/books.py

@@ -204,7 +204,8 @@ class CollectionsBookList(BookList):
                 elif fm['datatype'] == 'text' and fm['is_multiple']:
                     val = orig_val
                 elif fm['datatype'] == 'composite' and fm['is_multiple']:
-                    val = [v.strip() for v in val.split(fm['is_multiple'])]
+                    val = [v.strip() for v in
+                           val.split(fm['is_multiple']['ui_to_list'])]
                 else:
                     val = [val]
 

src/calibre/devices/usbms/device.py

@@ -837,6 +837,9 @@ class Device(DeviceConfig, DevicePlugin):
     def get_main_ebook_dir(self, for_upload=False):
         return self.EBOOK_DIR_MAIN
 
+    def get_carda_ebook_dir(self, for_upload=False):
+        return self.EBOOK_DIR_CARD_A
+
     def _sanity_check(self, on_card, files):
         if on_card == 'carda' and not self._card_a_prefix:
             raise ValueError(_('The reader has no storage card in this slot.'))
@@ -847,7 +850,7 @@ class Device(DeviceConfig, DevicePlugin):
 
         if on_card == 'carda':
             path = os.path.join(self._card_a_prefix,
-                    *(self.EBOOK_DIR_CARD_A.split('/')))
+                    *(self.get_carda_ebook_dir(for_upload=True).split('/')))
         elif on_card == 'cardb':
             path = os.path.join(self._card_b_prefix,
                     *(self.EBOOK_DIR_CARD_B.split('/')))

src/calibre/devices/usbms/driver.py

@@ -132,7 +132,7 @@ class USBMS(CLI, Device):
                                      self._card_b_prefix if oncard == 'cardb' \
                                                          else self._main_prefix
 
-        ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \
+        ebook_dirs = self.get_carda_ebook_dir() if oncard == 'carda' else \
             self.EBOOK_DIR_CARD_B if oncard == 'cardb' else \
             self.get_main_ebook_dir()
 

src/calibre/ebooks/comic/input.py

@@ -11,7 +11,7 @@ import os, shutil, traceback, textwrap, time, codecs
 from Queue import Empty
 
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
-from calibre import extract, CurrentDir, prints
+from calibre import extract, CurrentDir, prints, walk
 from calibre.constants import filesystem_encoding
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.ipc.server import Server
@@ -27,6 +27,11 @@ def extract_comic(path_to_comic_file):
         # names
         tdir = tdir.decode(filesystem_encoding)
     extract(path_to_comic_file, tdir)
+    for x in walk(tdir):
+        bn = os.path.basename(x)
+        nbn = bn.replace('#', '_')
+        if nbn != bn:
+            os.rename(x, os.path.join(os.path.dirname(x), nbn))
     return tdir
 
 def find_pages(dir, sort_on_mtime=False, verbose=False):
@@ -362,6 +367,7 @@ class ComicInput(InputFormatPlugin):
                 if not line:
                     continue
                 fname, title = line.partition(':')[0], line.partition(':')[-1]
+                fname = fname.replace('#', '_')
                 fname = os.path.join(tdir, *fname.split('/'))
                 if not title:
                     title = os.path.basename(fname).rpartition('.')[0]

src/calibre/ebooks/epub/output.py

@@ -394,6 +394,13 @@ class EPUBOutput(OutputFormatPlugin):
             for tag in XPath('//h:img[@src]')(root):
                 tag.set('src', tag.get('src', '').replace('&', ''))
 
+            # ADE whimpers in fright when it encounters a <td> outside a
+            # <table>
+            in_table = XPath('ancestor::h:table')
+            for tag in XPath('//h:td|//h:tr|//h:th')(root):
+                if not in_table(tag):
+                    tag.tag = XHTML('div')
+
             special_chars = re.compile(u'[\u200b\u00ad]')
             for elem in root.iterdescendants():
                 if getattr(elem, 'text', False):
@@ -413,7 +420,7 @@ class EPUBOutput(OutputFormatPlugin):
                             rule.style.removeProperty('margin-left')
                             # padding-left breaks rendering in webkit and gecko
                             rule.style.removeProperty('padding-left')
-                # Change whitespace:pre to pre-line to accommodate readers that
+                # Change whitespace:pre to pre-wrap to accommodate readers that
                 # cannot scroll horizontally
                 for rule in stylesheet.data.cssRules.rulesOfType(CSSRule.STYLE_RULE):
                     style = rule.style

src/calibre/ebooks/html/input.py

@@ -455,13 +455,16 @@ class HTMLInput(InputFormatPlugin):
             bhref = os.path.basename(link)
             id, href = self.oeb.manifest.generate(id='added',
                     href=bhref)
+            guessed = self.guess_type(href)[0]
+            media_type = guessed or self.BINARY_MIME
+            if 'text' in media_type:
+                self.log.warn('Ignoring link to text file %r'%link_)
+                return None
+
             self.oeb.log.debug('Added', link)
             self.oeb.container = self.DirContainer(os.path.dirname(link),
                     self.oeb.log, ignore_opf=True)
             # Load into memory
-            guessed = self.guess_type(href)[0]
-            media_type = guessed or self.BINARY_MIME
-
             item = self.oeb.manifest.add(id, href, media_type)
             item.html_input_href = bhref
             if guessed in self.OEB_STYLES:

src/calibre/ebooks/metadata/book/base.py

@@ -621,10 +621,7 @@ class Metadata(object):
             orig_res = res
             datatype = cmeta['datatype']
             if datatype == 'text' and cmeta['is_multiple']:
-                if cmeta['display'].get('is_names', False):
+                res = cmeta['is_multiple']['list_to_ui'].join(res)
-                    res = u' & '.join(res)
-                else:
-                    res = u', '.join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 if self.get_extra(key) is not None:
                     res = res + \
@@ -668,7 +665,7 @@ class Metadata(object):
             elif datatype == 'text' and fmeta['is_multiple']:
                 if isinstance(res, dict):
                     res = [k + ':' + v for k,v in res.items()]
-                res = u', '.join(sorted(res, key=sort_key))
+                res = fmeta['is_multiple']['list_to_ui'].join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 res = res + ' [%s]'%self.format_series_index()
             elif datatype == 'datetime':

src/calibre/ebooks/metadata/book/json_codec.py

@@ -5,8 +5,7 @@ Created on 4 Jun 2010
 '''
 
 from base64 import b64encode, b64decode
-import json
+import json, traceback
-import traceback
 
 from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
 from calibre.constants import filesystem_encoding, preferred_encoding
@@ -69,6 +68,40 @@ def object_to_unicode(obj, enc=preferred_encoding):
         return ans
     return obj
 
+def encode_is_multiple(fm):
+    if fm.get('is_multiple', None):
+        # migrate is_multiple back to a character
+        fm['is_multiple2'] = fm.get('is_multiple', {})
+        dt = fm.get('datatype', None)
+        if dt == 'composite':
+            fm['is_multiple'] = ','
+        else:
+            fm['is_multiple'] =  '|'
+    else:
+        fm['is_multiple'] = None
+        fm['is_multiple2'] = {}
+
+def decode_is_multiple(fm):
+    im = fm.get('is_multiple2',  None)
+    if im:
+        fm['is_multiple'] = im
+        del fm['is_multiple2']
+    else:
+        # Must migrate the is_multiple from char to dict
+        im = fm.get('is_multiple',  {})
+        if im:
+            dt = fm.get('datatype', None)
+            if dt == 'composite':
+                im = {'cache_to_list': ',', 'ui_to_list': ',',
+                      'list_to_ui': ', '}
+            elif fm.get('display', {}).get('is_names', False):
+                im = {'cache_to_list': '|', 'ui_to_list': '&',
+                      'list_to_ui': ', '}
+            else:
+                im = {'cache_to_list': '|', 'ui_to_list': ',',
+                      'list_to_ui': ', '}
+        fm['is_multiple'] = im
+
 class JsonCodec(object):
 
     def __init__(self):
@@ -93,9 +126,10 @@ class JsonCodec(object):
     def encode_metadata_attr(self, book, key):
         if key == 'user_metadata':
             meta = book.get_all_user_metadata(make_copy=True)
-            for k in meta:
+            for fm in meta.itervalues():
-                if meta[k]['datatype'] == 'datetime':
+                if fm['datatype'] == 'datetime':
-                    meta[k]['#value#'] = datetime_to_string(meta[k]['#value#'])
+                    fm['#value#'] = datetime_to_string(fm['#value#'])
+                encode_is_multiple(fm)
             return meta
         if key in self.field_metadata:
             datatype = self.field_metadata[key]['datatype']
@@ -135,9 +169,10 @@ class JsonCodec(object):
         if key == 'classifiers':
             key = 'identifiers'
         if key == 'user_metadata':
-            for k in value:
+            for fm in value.itervalues():
-                if value[k]['datatype'] == 'datetime':
+                if fm['datatype'] == 'datetime':
-                    value[k]['#value#'] = string_to_datetime(value[k]['#value#'])
+                    fm['#value#'] = string_to_datetime(fm['#value#'])
+                decode_is_multiple(fm)
             return value
         elif key in self.field_metadata:
             if self.field_metadata[key]['datatype'] == 'datetime':

src/calibre/ebooks/metadata/opf2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 lxml based OPF parser.
 '''
 
-import re, sys, unittest, functools, os, uuid, glob, cStringIO, json
+import re, sys, unittest, functools, os, uuid, glob, cStringIO, json, copy
 from urllib import unquote
 from urlparse import urlparse
 
@@ -453,10 +453,13 @@ class TitleSortField(MetadataField):
 
 def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)):
     from calibre.utils.config import to_json
-    from calibre.ebooks.metadata.book.json_codec import object_to_unicode
+    from calibre.ebooks.metadata.book.json_codec import (object_to_unicode,
+                                                         encode_is_multiple)
 
     for name, fm in all_user_metadata.items():
         try:
+            fm = copy.copy(fm)
+            encode_is_multiple(fm)
             fm = object_to_unicode(fm)
             fm = json.dumps(fm, default=to_json, ensure_ascii=False)
         except:
@@ -575,6 +578,7 @@ class OPF(object): # {{{
         self._user_metadata_ = {}
         temp = Metadata('x', ['x'])
         from calibre.utils.config import from_json
+        from calibre.ebooks.metadata.book.json_codec import decode_is_multiple
         elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,'
                 '"calibre:user_metadata:") and @content]')
         for elem in elems:
@@ -585,6 +589,7 @@ class OPF(object): # {{{
             fm = elem.get('content')
             try:
                 fm = json.loads(fm, object_hook=from_json)
+                decode_is_multiple(fm)
                 temp.set_user_metadata(name, fm)
             except:
                 prints('Failed to read user metadata:', name)

src/calibre/ebooks/metadata/sources/amazon.py

@@ -42,6 +42,7 @@ class Worker(Thread): # Get details {{{
         months = {
                 'de': {
             1 : ['jän'],
+            2 : ['februar'],
             3 : ['märz'],
             5 : ['mai'],
             6 : ['juni'],

src/calibre/ebooks/mobi/mobiml.py

@@ -442,7 +442,10 @@ class MobiMLizer(object):
         if tag in TABLE_TAGS and self.ignore_tables:
             tag = 'span' if tag == 'td' else 'div'
 
-        # GR: Added 'width', 'border' and 'scope'
+        if tag == 'table':
+            css = style.cssdict()
+            if 'border' in css or 'border-width' in css:
+                elem.set('border', '1')
         if tag in TABLE_TAGS:
             for attr in ('rowspan', 'colspan', 'width', 'border', 'scope'):
                 if attr in elem.attrib:

src/calibre/ebooks/mobi/reader.py

@@ -348,7 +348,6 @@ class MobiReader(object):
                 self.processed_html = self.remove_random_bytes(self.processed_html)
                 root = soupparser.fromstring(self.processed_html)
 
-
         if root.tag != 'html':
             self.log.warn('File does not have opening <html> tag')
             nroot = html.fromstring('<html><head></head><body></body></html>')

src/calibre/ebooks/oeb/stylizer.py

@@ -13,7 +13,13 @@ from weakref import WeakKeyDictionary
 from xml.dom import SyntaxErr as CSSSyntaxError
 import cssutils
 from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration,
-    CSSValueList, CSSFontFaceRule, cssproperties)
+    CSSFontFaceRule, cssproperties)
+try:
+    from cssutils.css import CSSValueList
+    CSSValueList
+except ImportError:
+    # cssutils >= 0.9.8
+    from cssutils.css import PropertyValue as CSSValueList
 from cssutils import profile as cssprofiles
 from lxml import etree
 from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError

src/calibre/gui2/__init__.py

@@ -271,6 +271,9 @@ class Dispatcher(QObject):
     Convenience class to use Qt signals with arbitrary python callables.
     By default, ensures that a function call always happens in the
     thread this Dispatcher was created in.
+
+    Note that if you create the Dispatcher in a thread without an event loop of
+    its own, the function call will happen in the GUI thread (I think).
     '''
     dispatch_signal = pyqtSignal(object, object)
 
@@ -292,11 +295,20 @@ class FunctionDispatcher(QObject):
     '''
     Convenience class to use Qt signals with arbitrary python functions.
     By default, ensures that a function call always happens in the
-    thread this Dispatcher was created in.
+    thread this FunctionDispatcher was created in.
+
+    Note that you must create FunctionDispatcher objects in the GUI thread.
     '''
     dispatch_signal = pyqtSignal(object, object, object)
 
     def __init__(self, func, queued=True, parent=None):
+        global gui_thread
+        if gui_thread is None:
+            gui_thread = QThread.currentThread()
+        if not is_gui_thread():
+            raise ValueError(
+                'You can only create a FunctionDispatcher in the GUI thread')
+
         QObject.__init__(self, parent)
         self.func = func
         typ = Qt.QueuedConnection
@@ -307,6 +319,8 @@ class FunctionDispatcher(QObject):
         self.lock = threading.Lock()
 
     def __call__(self, *args, **kwargs):
+        if is_gui_thread():
+            return self.func(*args, **kwargs)
         with self.lock:
             self.dispatch_signal.emit(self.q, args, kwargs)
             res = self.q.get()

src/calibre/gui2/actions/choose_library.py

@@ -11,10 +11,11 @@ from functools import partial
 from PyQt4.Qt import QMenu, Qt, QInputDialog, QToolButton
 
 from calibre import isbytestring
-from calibre.constants import filesystem_encoding
+from calibre.constants import filesystem_encoding, iswindows
 from calibre.utils.config import prefs
-from calibre.gui2 import gprefs, warning_dialog, Dispatcher, error_dialog, \
+from calibre.gui2 import (gprefs, warning_dialog, Dispatcher, error_dialog,
-    question_dialog, info_dialog
+    question_dialog, info_dialog)
+from calibre.library.database2 import LibraryDatabase2
 from calibre.gui2.actions import InterfaceAction
 
 class LibraryUsageStats(object): # {{{
@@ -229,6 +230,12 @@ class ChooseLibraryAction(InterfaceAction):
             return error_dialog(self.gui, _('Already exists'),
                     _('The folder %s already exists. Delete it first.') %
                     newloc, show=True)
+        if (iswindows and len(newloc) >
+                LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+            return error_dialog(self.gui, _('Too long'),
+                    _('Path to library too long. Must be less than'
+                    ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+                    show=True)
         try:
             os.rename(loc, newloc)
         except:

src/calibre/gui2/actions/delete.py

@@ -94,6 +94,9 @@ class DeleteAction(InterfaceAction):
         self.delete_menu.addAction(
                 _('Remove all formats from selected books, except...'),
                 self.delete_all_but_selected_formats)
+        self.delete_menu.addAction(
+                _('Remove all formats from selected books'),
+                self.delete_all_formats)
         self.delete_menu.addAction(
                 _('Remove covers from selected books'), self.delete_covers)
         self.delete_menu.addSeparator()
@@ -174,6 +177,28 @@ class DeleteAction(InterfaceAction):
         if ids:
             self.gui.tags_view.recount()
 
+    def delete_all_formats(self, *args):
+        ids = self._get_selected_ids()
+        if not ids:
+            return
+        if not confirm('<p>'+_('<b>All formats</b> for the selected books will '
+                               'be <b>deleted</b> from your library.<br>'
+                               'The book metadata will be kept. Are you sure?')
+                            +'</p>', 'delete_all_formats', self.gui):
+            return
+        db = self.gui.library_view.model().db
+        for id in ids:
+            fmts = db.formats(id, index_is_id=True, verify_formats=False)
+            if fmts:
+                for fmt in fmts.split(','):
+                    self.gui.library_view.model().db.remove_format(id, fmt,
+                            index_is_id=True, notify=False)
+        self.gui.library_view.model().refresh_ids(ids)
+        self.gui.library_view.model().current_changed(self.gui.library_view.currentIndex(),
+                self.gui.library_view.currentIndex())
+        if ids:
+            self.gui.tags_view.recount()
+
     def remove_matching_books_from_device(self, *args):
         if not self.gui.device_manager.is_device_connected:
             d = error_dialog(self.gui, _('Cannot delete books'),

src/calibre/gui2/actions/edit_metadata.py

@@ -439,7 +439,8 @@ class EditMetadataAction(InterfaceAction):
             view.reset()
 
     # Apply bulk metadata changes {{{
-    def apply_metadata_changes(self, id_map, title=None, msg='', callback=None):
+    def apply_metadata_changes(self, id_map, title=None, msg='', callback=None,
+            merge_tags=True):
         '''
         Apply the metadata changes in id_map to the database synchronously
         id_map must be a mapping of ids to Metadata objects. Set any fields you
@@ -466,9 +467,9 @@ class EditMetadataAction(InterfaceAction):
                     cancelable=False)
             self.apply_pd.setModal(True)
             self.apply_pd.show()
+        self._am_merge_tags = True
         self.do_one_apply()
 
-
     def do_one_apply(self):
         if self.apply_current_idx >= len(self.apply_id_map):
             return self.finalize_apply()
@@ -484,6 +485,12 @@ class EditMetadataAction(InterfaceAction):
             mi.identifiers = idents
             if mi.is_null('series'):
                 mi.series_index = None
+            if self._am_merge_tags:
+                old_tags = db.tags(i, index_is_id=True)
+                if old_tags:
+                    tags = [x.strip() for x in old_tags.split(',')] + (
+                            mi.tags if mi.tags else [])
+                    mi.tags = list(set(tags))
             db.set_metadata(i, mi, commit=False, set_title=set_title,
                     set_authors=set_authors, notify=False)
             self.applied_ids.append(i)

src/calibre/gui2/actions/plugin_updates.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QApplication, Qt, QIcon
+from calibre.gui2.actions import InterfaceAction
+from calibre.gui2.dialogs.plugin_updater import (PluginUpdaterDialog,
+                                                 FILTER_ALL, FILTER_UPDATE_AVAILABLE)
+
+class PluginUpdatesAction(InterfaceAction):
+
+    name = 'Plugin Updates'
+    action_spec = (_('Plugin Updates'), None, None, None)
+    action_type = 'current'
+
+    def genesis(self):
+        self.qaction.setIcon(QIcon(I('plugins/plugin_updater.png')))
+        self.qaction.triggered.connect(self.check_for_plugin_updates)
+
+    def check_for_plugin_updates(self):
+        # Get the user to choose a plugin to install
+        initial_filter = FILTER_UPDATE_AVAILABLE
+        mods = QApplication.keyboardModifiers()
+        if mods & Qt.ControlModifier or mods & Qt.ShiftModifier:
+            initial_filter = FILTER_ALL
+
+        d = PluginUpdaterDialog(self.gui, initial_filter=initial_filter)
+        d.exec_()

src/calibre/gui2/actions/preferences.py

@@ -24,6 +24,8 @@ class PreferencesAction(InterfaceAction):
             pm.addAction(QIcon(I('config.png')), _('Change calibre behavior'), self.do_config)
         pm.addAction(QIcon(I('wizard.png')), _('Run welcome wizard'),
                 self.gui.run_wizard)
+        pm.addAction(QIcon(I('plugins/plugin_updater.png')),
+                _('Get plugins to enhance calibre'), self.get_plugins)
         if not DEBUG:
             pm.addSeparator()
             ac = pm.addAction(QIcon(I('debug.png')), _('Restart in debug mode'),
@@ -36,6 +38,12 @@ class PreferencesAction(InterfaceAction):
         for x in (self.gui.preferences_action, self.qaction):
             x.triggered.connect(self.do_config)
 
+    def get_plugins(self):
+        from calibre.gui2.dialogs.plugin_updater import (PluginUpdaterDialog,
+                FILTER_NOT_INSTALLED)
+        d = PluginUpdaterDialog(self.gui,
+                initial_filter=FILTER_NOT_INSTALLED)
+        d.exec_()
 
     def do_config(self, checked=False, initial_plugin=None,
             close_after_initial=False):

src/calibre/gui2/actions/save_to_disk.py

@@ -33,7 +33,6 @@ class SaveMenu(QMenu): # {{{
 
 # }}}
 
-
 class SaveToDiskAction(InterfaceAction):
 
     name = "Save To Disk"

src/calibre/gui2/complete.py

@@ -12,16 +12,23 @@ from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
 from calibre.gui2.widgets import EnComboBox, LineEditECM
+from calibre.utils.config_base import tweaks
 
 class CompleteModel(QAbstractListModel):
 
     def __init__(self, parent=None):
         QAbstractListModel.__init__(self, parent)
         self.items = []
+        self.sorting = QCompleter.UnsortedModel
 
     def set_items(self, items):
         items = [unicode(x.strip()) for x in items]
-        self.items = list(sorted(items, key=lambda x: sort_key(x)))
+        if len(items) < tweaks['completion_change_to_ascii_sorting']:
+            self.items = sorted(items, key=lambda x: sort_key(x))
+            self.sorting = QCompleter.UnsortedModel
+        else:
+            self.items = sorted(items, key=lambda x:x.lower())
+            self.sorting = QCompleter.CaseInsensitivelySortedModel
         self.lowered_items = [lower(x) for x in self.items]
         self.reset()
 
@@ -62,7 +69,7 @@ class MultiCompleteLineEdit(QLineEdit, LineEditECM):
         c.setWidget(self)
         c.setCompletionMode(QCompleter.PopupCompletion)
         c.setCaseSensitivity(Qt.CaseInsensitive)
-        c.setModelSorting(QCompleter.UnsortedModel)
+        c.setModelSorting(self._model.sorting)
         c.setCompletionRole(Qt.DisplayRole)
         p = c.popup()
         p.setMouseTracking(True)
@@ -146,6 +153,7 @@ class MultiCompleteLineEdit(QLineEdit, LineEditECM):
             return self._model.items
         def fset(self, items):
             self._model.set_items(items)
+            self._completer.setModelSorting(self._model.sorting)
         return property(fget=fget, fset=fset)
 
 class MultiCompleteComboBox(EnComboBox):

src/calibre/gui2/convert/epub_output.ui

@@ -58,7 +58,7 @@
       <string> KB</string>
      </property>
      <property name="minimum">
-      <number>100</number>
+      <number>25</number>
      </property>
      <property name="maximum">
       <number>1000000</number>

src/calibre/gui2/custom_column_widgets.py

@@ -226,16 +226,14 @@ class Comments(Base):
 class Text(Base):
 
     def setup_ui(self, parent):
-        if self.col_metadata['display'].get('is_names', False):
+        self.sep = self.col_metadata['multiple_seps']
-            self.sep = u' & '
-        else:
-            self.sep = u', '
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
         values.sort(key=sort_key)
+
         if self.col_metadata['is_multiple']:
             w = MultiCompleteLineEdit(parent)
-            w.set_separator(self.sep.strip())
+            w.set_separator(self.sep['ui_to_list'])
-            if self.sep == u' & ':
+            if self.sep['ui_to_list'] == '&':
                 w.set_space_before_sep(True)
                 w.set_add_separator(tweaks['authors_completer_append_separator'])
             w.update_items_cache(values)
@@ -269,12 +267,12 @@ class Text(Base):
         if self.col_metadata['is_multiple']:
             if not val:
                 val = []
-            self.widgets[1].setText(self.sep.join(val))
+            self.widgets[1].setText(self.sep['list_to_ui'].join(val))
 
     def getter(self):
         if self.col_metadata['is_multiple']:
             val = unicode(self.widgets[1].text()).strip()
-            ans = [x.strip() for x in val.split(self.sep.strip()) if x.strip()]
+            ans = [x.strip() for x in val.split(self.sep['ui_to_list']) if x.strip()]
             if not ans:
                 ans = None
             return ans
@@ -899,9 +897,10 @@ class BulkText(BulkBase):
         if not self.a_c_checkbox.isChecked():
             return
         if self.col_metadata['is_multiple']:
+            ism = self.col_metadata['multiple_seps']
             if self.col_metadata['display'].get('is_names', False):
                 val = self.gui_val
-                add = [v.strip() for v in val.split('&') if v.strip()]
+                add = [v.strip() for v in val.split(ism['ui_to_list']) if v.strip()]
                 self.db.set_custom_bulk(book_ids, add, num=self.col_id)
             else:
                 remove_all, adding, rtext = self.gui_val
@@ -911,10 +910,10 @@ class BulkText(BulkBase):
                 else:
                     txt = rtext
                     if txt:
-                        remove = set([v.strip() for v in txt.split(',')])
+                        remove = set([v.strip() for v in txt.split(ism['ui_to_list'])])
                 txt = adding
                 if txt:
-                    add = set([v.strip() for v in txt.split(',')])
+                    add = set([v.strip() for v in txt.split(ism['ui_to_list'])])
                 else:
                     add = set()
                 self.db.set_custom_bulk_multiple(book_ids, add=add,

src/calibre/gui2/device.py

@@ -6,18 +6,18 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import os, traceback, Queue, time, cStringIO, re, sys
 from threading import Thread
 
-from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, \
+from PyQt4.Qt import (QMenu, QAction, QActionGroup, QIcon, SIGNAL,
-                     Qt, pyqtSignal, QDialog, QObject
+                     Qt, pyqtSignal, QDialog, QObject)
 
-from calibre.customize.ui import available_input_formats, available_output_formats, \
+from calibre.customize.ui import (available_input_formats, available_output_formats,
-    device_plugins
+    device_plugins)
 from calibre.devices.interface import DevicePlugin
 from calibre.devices.errors import UserFeedback, OpenFeedback
 from calibre.gui2.dialogs.choose_format_device import ChooseFormatDeviceDialog
 from calibre.utils.ipc.job import BaseJob
 from calibre.devices.scanner import DeviceScanner
-from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \
+from calibre.gui2 import (config, error_dialog, Dispatcher, dynamic,
-        warning_dialog, info_dialog, choose_dir
+        warning_dialog, info_dialog, choose_dir, FunctionDispatcher)
 from calibre.ebooks.metadata import authors_to_string
 from calibre import preferred_encoding, prints, force_unicode, as_unicode
 from calibre.utils.filenames import ascii_filename
@@ -35,8 +35,13 @@ class DeviceJob(BaseJob): # {{{
 
     def __init__(self, func, done, job_manager, args=[], kwargs={},
             description=''):
-        BaseJob.__init__(self, description, done=done)
+        BaseJob.__init__(self, description)
         self.func = func
+        self.callback_on_done = done
+        if not isinstance(self.callback_on_done, (Dispatcher,
+            FunctionDispatcher)):
+            self.callback_on_done = FunctionDispatcher(self.callback_on_done)
+
         self.args, self.kwargs = args, kwargs
         self.exception = None
         self.job_manager = job_manager
@@ -50,6 +55,10 @@ class DeviceJob(BaseJob): # {{{
     def job_done(self):
         self.duration = time.time() - self.start_time
         self.percent = 1
+        try:
+            self.callback_on_done(self)
+        except:
+            pass
         self.job_manager.changed_queue.put(self)
 
     def report_progress(self, percent, msg=''):
@@ -254,6 +263,7 @@ class DeviceManager(Thread): # {{{
                 job = self.next()
                 if job is not None:
                     self.current_job = job
+                    if self.device is not None:
                         self.device.set_progress_reporter(job.report_progress)
                     self.current_job.run()
                     self.current_job = None
@@ -587,7 +597,7 @@ class DeviceMenu(QMenu): # {{{
 
     # }}}
 
-class DeviceSignals(QObject):
+class DeviceSignals(QObject): # {{{
     #: This signal is emitted once, after metadata is downloaded from the
     #: connected device.
     #: The sequence: gui.device_manager.is_device_connected will become True,
@@ -604,6 +614,7 @@ class DeviceSignals(QObject):
     device_connection_changed = pyqtSignal(object)
 
 device_signals = DeviceSignals()
+# }}}
 
 class DeviceMixin(object): # {{{
 
@@ -611,7 +622,7 @@ class DeviceMixin(object): # {{{
         self.device_error_dialog = error_dialog(self, _('Error'),
                 _('Error communicating with device'), ' ')
         self.device_error_dialog.setModal(Qt.NonModal)
-        self.device_manager = DeviceManager(Dispatcher(self.device_detected),
+        self.device_manager = DeviceManager(FunctionDispatcher(self.device_detected),
                 self.job_manager, Dispatcher(self.status_bar.show_message),
                 Dispatcher(self.show_open_feedback))
         self.device_manager.start()
@@ -736,7 +747,7 @@ class DeviceMixin(object): # {{{
         self.set_device_menu_items_state(connected)
         if connected:
             self.device_manager.get_device_information(\
-                    Dispatcher(self.info_read))
+                    FunctionDispatcher(self.info_read))
             self.set_default_thumbnail(\
                     self.device_manager.device.THUMBNAIL_HEIGHT)
             self.status_bar.show_message(_('Device: ')+\
@@ -767,7 +778,7 @@ class DeviceMixin(object): # {{{
                 self.device_manager.device.icon)
         self.bars_manager.update_bars()
         self.status_bar.device_connected(info[0])
-        self.device_manager.books(Dispatcher(self.metadata_downloaded))
+        self.device_manager.books(FunctionDispatcher(self.metadata_downloaded))
 
     def metadata_downloaded(self, job):
         '''
@@ -810,7 +821,7 @@ class DeviceMixin(object): # {{{
 
     def remove_paths(self, paths):
         return self.device_manager.delete_books(
-                Dispatcher(self.books_deleted), paths)
+                FunctionDispatcher(self.books_deleted), paths)
 
     def books_deleted(self, job):
         '''
@@ -1187,7 +1198,7 @@ class DeviceMixin(object): # {{{
         Upload metadata to device.
         '''
         plugboards = self.library_view.model().db.prefs.get('plugboards', {})
-        self.device_manager.sync_booklists(Dispatcher(self.metadata_synced),
+        self.device_manager.sync_booklists(FunctionDispatcher(self.metadata_synced),
                                            self.booklists(), plugboards)
 
     def metadata_synced(self, job):
@@ -1222,7 +1233,7 @@ class DeviceMixin(object): # {{{
         titles = [i.title for i in metadata]
         plugboards = self.library_view.model().db.prefs.get('plugboards', {})
         job = self.device_manager.upload_books(
-                Dispatcher(self.books_uploaded),
+                FunctionDispatcher(self.books_uploaded),
                 files, names, on_card=on_card,
                 metadata=metadata, titles=titles, plugboards=plugboards
               )
@@ -1475,7 +1486,7 @@ class DeviceMixin(object): # {{{
                                     self.cover_to_thumbnail(open(book.cover, 'rb').read())
                 plugboards = self.library_view.model().db.prefs.get('plugboards', {})
                 self.device_manager.sync_booklists(
-                                    Dispatcher(self.metadata_synced), booklists,
+                                    FunctionDispatcher(self.metadata_synced), booklists,
                                     plugboards)
         return update_metadata
     # }}}

src/calibre/gui2/dialogs/choose_library.py

@@ -11,10 +11,11 @@ from PyQt4.Qt import QDialog
 
 from calibre.gui2.dialogs.choose_library_ui import Ui_Dialog
 from calibre.gui2 import error_dialog, choose_dir
-from calibre.constants import filesystem_encoding
+from calibre.constants import filesystem_encoding, iswindows
 from calibre import isbytestring, patheq
 from calibre.utils.config import prefs
 from calibre.gui2.wizard import move_library
+from calibre.library.database2 import LibraryDatabase2
 
 class ChooseLibrary(QDialog, Ui_Dialog):
 
@@ -57,12 +58,20 @@ class ChooseLibrary(QDialog, Ui_Dialog):
                     _('There is no existing calibre library at %s')%loc,
                     show=True)
             return False
-        if ac in ('new', 'move') and not empty:
+        if ac in ('new', 'move'):
+            if not empty:
                 error_dialog(self, _('Not empty'),
                     _('The folder %s is not empty. Please choose an empty'
                        ' folder')%loc,
                     show=True)
                 return False
+            if (iswindows and len(loc) >
+                    LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+                error_dialog(self, _('Too long'),
+                    _('Path to library too long. Must be less than'
+                    ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+                    show=True)
+                return False
 
         return True
 

src/calibre/gui2/dialogs/message_box.py

@@ -148,19 +148,21 @@ class ViewLog(QDialog): # {{{
         QApplication.clipboard().setText(txt)
 # }}}
 
-
 _proceed_memory = []
 
 class ProceedNotification(MessageBox): # {{{
 
     def __init__(self, callback, payload, html_log, log_viewer_title, title, msg,
-            det_msg='', show_copy_button=False, parent=None):
+            det_msg='', show_copy_button=False, parent=None,
+            cancel_callback=None):
         '''
         A non modal popup that notifies the user that a background task has
         been completed.
 
         :param callback: A callable that is called with payload if the user
-        asks to proceed. Note that this is always called in the GUI thread
+        asks to proceed. Note that this is always called in the GUI thread.
+        :param cancel_callback: A callable that is called with the payload if
+        the users asks not to proceed.
         :param payload: Arbitrary object, passed to callback
         :param html_log: An HTML or plain text log
         :param log_viewer_title: The title for the log viewer window
@@ -181,7 +183,7 @@ class ProceedNotification(MessageBox): # {{{
         self.vlb.clicked.connect(self.show_log)
         self.det_msg_toggle.setVisible(bool(det_msg))
         self.setModal(False)
-        self.callback = callback
+        self.callback, self.cancel_callback = callback, cancel_callback
         _proceed_memory.append(self)
 
     def show_log(self):
@@ -192,9 +194,51 @@ class ProceedNotification(MessageBox): # {{{
         try:
             if result == self.Accepted:
                 self.callback(self.payload)
+            elif self.cancel_callback is not None:
+                self.cancel_callback(self.payload)
         finally:
             # Ensure this notification is garbage collected
-            self.callback = None
+            self.callback = self.cancel_callback = None
+            self.setParent(None)
+            self.finished.disconnect()
+            self.vlb.clicked.disconnect()
+            _proceed_memory.remove(self)
+# }}}
+
+class ErrorNotification(MessageBox): # {{{
+
+    def __init__(self, html_log, log_viewer_title, title, msg,
+            det_msg='', show_copy_button=False, parent=None):
+        '''
+        A non modal popup that notifies the user that a background task has
+        errored.
+
+        :param html_log: An HTML or plain text log
+        :param log_viewer_title: The title for the log viewer window
+        :param title: The title for this popup
+        :param msg: The msg to display
+        :param det_msg: Detailed message
+        '''
+        MessageBox.__init__(self, MessageBox.ERROR, title, msg,
+                det_msg=det_msg, show_copy_button=show_copy_button,
+                parent=parent)
+        self.html_log = html_log
+        self.log_viewer_title = log_viewer_title
+        self.finished.connect(self.do_close, type=Qt.QueuedConnection)
+
+        self.vlb = self.bb.addButton(_('View log'), self.bb.ActionRole)
+        self.vlb.setIcon(QIcon(I('debug.png')))
+        self.vlb.clicked.connect(self.show_log)
+        self.det_msg_toggle.setVisible(bool(det_msg))
+        self.setModal(False)
+        _proceed_memory.append(self)
+
+    def show_log(self):
+        self.log_viewer = ViewLog(self.log_viewer_title, self.html_log,
+                parent=self)
+
+    def do_close(self, result):
+        # Ensure this notification is garbage collected
         self.setParent(None)
         self.finished.disconnect()
         self.vlb.clicked.disconnect()

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -520,7 +520,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             elif not fm['is_multiple']:
                 val = [val]
             elif fm['datatype'] == 'composite':
-                val = [v.strip() for v in val.split(fm['is_multiple'])]
+                val = [v.strip() for v in val.split(fm['is_multiple']['ui_to_list'])]
             elif field == 'authors':
                 val = [v.replace('|', ',') for v in val]
         else:
@@ -655,19 +655,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
 
         if self.destination_field_fm['is_multiple']:
             if self.comma_separated.isChecked():
-                if dest == 'authors' or \
+                splitter = self.destination_field_fm['is_multiple']['ui_to_list']
-                        (self.destination_field_fm['is_custom'] and
-                         self.destination_field_fm['datatype'] == 'text' and
-                         self.destination_field_fm['display'].get('is_names', False)):
-                    splitter = ' & '
-                else:
-                    splitter = ','
-
                 res = []
                 for v in val:
-                    for x in v.split(splitter):
+                    res.extend([x.strip() for x in v.split(splitter) if x.strip()])
-                        if x.strip():
-                            res.append(x.strip())
                 val = res
             else:
                 val = [v.replace(',', '') for v in val]

src/calibre/gui2/dialogs/plugin_updater.py

@@ -0,0 +1,869 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import re, datetime, traceback
+from lxml import html
+from PyQt4.Qt import (Qt, QUrl, QFrame, QVBoxLayout, QLabel, QBrush, QTextEdit,
+                      QComboBox, QAbstractItemView, QHBoxLayout, QDialogButtonBox,
+                      QAbstractTableModel, QVariant, QTableView, QModelIndex,
+                      QSortFilterProxyModel, QAction, QIcon, QDialog,
+                      QFont, QPixmap, QSize)
+from calibre import browser, prints
+from calibre.constants import numeric_version, iswindows, isosx, DEBUG
+from calibre.customize.ui import (initialized_plugins, is_disabled, remove_plugin,
+                                  add_plugin, enable_plugin, disable_plugin,
+                                  NameConflict, has_external_plugins)
+from calibre.gui2 import error_dialog, question_dialog, info_dialog, NONE, open_url, gprefs
+from calibre.gui2.preferences.plugins import ConfigWidget
+from calibre.utils.date import UNDEFINED_DATE, format_date
+
+
+MR_URL = 'http://www.mobileread.com/forums/'
+MR_INDEX_URL = MR_URL + 'showpost.php?p=1362767&postcount=1'
+
+FILTER_ALL = 0
+FILTER_INSTALLED = 1
+FILTER_UPDATE_AVAILABLE = 2
+FILTER_NOT_INSTALLED = 3
+
+def get_plugin_updates_available():
+    '''
+    API exposed to read whether there are updates available for any
+    of the installed user plugins.
+    Returns None if no updates found
+    Returns list(DisplayPlugin) of plugins installed that have a new version
+    '''
+    if not has_external_plugins():
+        return None
+    display_plugins = read_available_plugins()
+    if display_plugins:
+        update_plugins = filter(filter_upgradeable_plugins, display_plugins)
+        if len(update_plugins) > 0:
+            return update_plugins
+    return None
+
+def filter_upgradeable_plugins(display_plugin):
+    return display_plugin.is_upgrade_available()
+
+def filter_not_installed_plugins(display_plugin):
+    return not display_plugin.is_installed()
+
+def read_available_plugins():
+    display_plugins = []
+    br = browser()
+    br.set_handle_gzip(True)
+    try:
+        raw = br.open_novisit(MR_INDEX_URL).read()
+        if not raw:
+            return
+    except:
+        traceback.print_exc()
+        return
+    raw = raw.decode('utf-8', errors='replace')
+    root = html.fromstring(raw)
+    list_nodes = root.xpath('//div[@id="post_message_1362767"]/ul/li')
+    # Add our deprecated plugins which are nested in a grey span
+    list_nodes.extend(root.xpath('//div[@id="post_message_1362767"]/span/ul/li'))
+    for list_node in list_nodes:
+        try:
+            display_plugin = DisplayPlugin(list_node)
+            get_installed_plugin_status(display_plugin)
+            display_plugins.append(display_plugin)
+        except:
+            if DEBUG:
+                prints('======= MobileRead Parse Error =======')
+                traceback.print_exc()
+                prints(html.tostring(list_node))
+    display_plugins = sorted(display_plugins, key=lambda k: k.name)
+    return display_plugins
+
+def get_installed_plugin_status(display_plugin):
+    display_plugin.installed_version = None
+    display_plugin.plugin = None
+    for plugin in initialized_plugins():
+        if plugin.name == display_plugin.name:
+            display_plugin.plugin = plugin
+            display_plugin.installed_version = plugin.version
+            break
+    if display_plugin.uninstall_plugins:
+        # Plugin requires a specific plugin name to be uninstalled first
+        # This could occur when a plugin is renamed (Kindle Collections)
+        # or multiple plugins deprecated into a newly named one.
+        # Check whether user has the previous version(s) installed
+        plugins_to_remove = list(display_plugin.uninstall_plugins)
+        for plugin_to_uninstall in plugins_to_remove:
+            found = False
+            for plugin in initialized_plugins():
+                if plugin.name == plugin_to_uninstall:
+                    found = True
+                    break
+            if not found:
+                display_plugin.uninstall_plugins.remove(plugin_to_uninstall)
+
+
+class ImageTitleLayout(QHBoxLayout):
+    '''
+    A reusable layout widget displaying an image followed by a title
+    '''
+    def __init__(self, parent, icon_name, title):
+        QHBoxLayout.__init__(self)
+        title_font = QFont()
+        title_font.setPointSize(16)
+        title_image_label = QLabel(parent)
+        pixmap = QPixmap()
+        pixmap.load(I(icon_name))
+        if pixmap is None:
+            error_dialog(parent, _('Restart required'),
+                         _('You must restart Calibre before using this plugin!'), show=True)
+        else:
+            title_image_label.setPixmap(pixmap)
+        title_image_label.setMaximumSize(32, 32)
+        title_image_label.setScaledContents(True)
+        self.addWidget(title_image_label)
+        shelf_label = QLabel(title, parent)
+        shelf_label.setFont(title_font)
+        self.addWidget(shelf_label)
+        self.insertStretch(-1)
+
+
+class SizePersistedDialog(QDialog):
+    '''
+    This dialog is a base class for any dialogs that want their size/position
+    restored when they are next opened.
+    '''
+
+    initial_extra_size = QSize(0, 0)
+
+    def __init__(self, parent, unique_pref_name):
+        QDialog.__init__(self, parent)
+        self.unique_pref_name = unique_pref_name
+        self.geom = gprefs.get(unique_pref_name, None)
+        self.finished.connect(self.dialog_closing)
+
+    def resize_dialog(self):
+        if self.geom is None:
+            self.resize(self.sizeHint()+self.initial_extra_size)
+        else:
+            self.restoreGeometry(self.geom)
+
+    def dialog_closing(self, result):
+        geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = geom
+
+
+class VersionHistoryDialog(SizePersistedDialog):
+
+    def __init__(self, parent, plugin_name, html):
+        SizePersistedDialog.__init__(self, parent, 'Plugin Updater plugin:version history dialog')
+        self.setWindowTitle(_('Version History for %s')%plugin_name)
+
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+
+        self.notes = QTextEdit(html, self)
+        self.notes.setReadOnly(True)
+        layout.addWidget(self.notes)
+
+        self.button_box = QDialogButtonBox(QDialogButtonBox.Close)
+        self.button_box.rejected.connect(self.reject)
+        layout.addWidget(self.button_box)
+
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+
+
+class PluginFilterComboBox(QComboBox):
+    def __init__(self, parent):
+        QComboBox.__init__(self, parent)
+        items = [_('All'), _('Installed'), _('Update available'), _('Not installed')]
+        self.addItems(items)
+
+
+class DisplayPlugin(object):
+
+    def __init__(self, list_node):
+        # The html from the index web page looks like this:
+        '''
+<li><a href="http://www.mobileread.com/forums/showthread.php?t=121787">Book Sync</a><br />
+<i>Add books to a list to be automatically sent to your device the next time it is connected.<br />
+<span class="resize_1">Version: 1.1; Released: 02-22-2011; Calibre: 0.7.42; Author: kiwidude; <br />
+Platforms: Windows, OSX, Linux; History: Yes;</span></i></li>
+        '''
+        self.name = list_node.xpath('a')[0].text_content().strip()
+        self.forum_link = list_node.xpath('a/@href')[0].strip()
+        self.installed_version = None
+
+        description_text = list_node.xpath('i')[0].text_content()
+        description_parts = description_text.partition('Version:')
+        self.description = description_parts[0].strip()
+
+        details_text = description_parts[1] + description_parts[2].replace('\r\n','')
+        details_pairs = details_text.split(';')
+        details = {}
+        for details_pair in details_pairs:
+            pair = details_pair.split(':')
+            if len(pair) == 2:
+                key = pair[0].strip().lower()
+                value = pair[1].strip()
+                details[key] = value
+
+        donation_node = list_node.xpath('i/span/a/@href')
+        self.donation_link = donation_node[0] if donation_node else None
+
+        self.available_version = self._version_text_to_tuple(details.get('version', None))
+
+        release_date = details.get('released', '01-01-0101').split('-')
+        date_parts = [int(re.search(r'(\d+)', x).group(1)) for x in release_date]
+        self.release_date = datetime.date(date_parts[2], date_parts[0], date_parts[1])
+
+        self.calibre_required_version = self._version_text_to_tuple(details.get('calibre', None))
+        self.author = details.get('author', '')
+        self.platforms = [p.strip().lower() for p in details.get('platforms', '').split(',')]
+        # Optional pairing just for plugins which require checking for uninstall first
+        self.uninstall_plugins = []
+        uninstall = details.get('uninstall', None)
+        if uninstall:
+            self.uninstall_plugins = [i.strip() for i in uninstall.split(',')]
+        self.has_changelog = details.get('history', 'No').lower() in ['yes', 'true']
+        self.is_deprecated = details.get('deprecated', 'No').lower() in ['yes', 'true']
+
+    def _version_text_to_tuple(self, version_text):
+        if version_text:
+            ver = version_text.split('.')
+            while len(ver) < 3:
+                ver.append('0')
+            ver = [int(re.search(r'(\d+)', x).group(1)) for x in ver]
+            return tuple(ver)
+        else:
+            return None
+
+    def is_disabled(self):
+        if self.plugin is None:
+            return False
+        return is_disabled(self.plugin)
+
+    def is_installed(self):
+        return self.installed_version is not None
+
+    def is_upgrade_available(self):
+        return self.is_installed() and (self.installed_version < self.available_version \
+                or self.is_deprecated)
+
+    def is_valid_platform(self):
+        if iswindows:
+            return 'windows' in self.platforms
+        if isosx:
+            return 'osx' in self.platforms
+        return 'linux' in self.platforms
+
+    def is_valid_calibre(self):
+        return numeric_version >= self.calibre_required_version
+
+    def is_valid_to_install(self):
+        return self.is_valid_platform() and self.is_valid_calibre() and not self.is_deprecated
+
+
+class DisplayPluginSortFilterModel(QSortFilterProxyModel):
+
+    def __init__(self, parent):
+        QSortFilterProxyModel.__init__(self, parent)
+        self.setSortRole(Qt.UserRole)
+        self.filter_criteria = FILTER_ALL
+
+    def filterAcceptsRow(self, sourceRow, sourceParent):
+        index = self.sourceModel().index(sourceRow, 0, sourceParent)
+        display_plugin = self.sourceModel().display_plugins[index.row()]
+        if self.filter_criteria == FILTER_ALL:
+            return not (display_plugin.is_deprecated and not display_plugin.is_installed())
+        if self.filter_criteria == FILTER_INSTALLED:
+            return display_plugin.is_installed()
+        if self.filter_criteria == FILTER_UPDATE_AVAILABLE:
+            return display_plugin.is_upgrade_available()
+        if self.filter_criteria == FILTER_NOT_INSTALLED:
+            return not display_plugin.is_installed() and not display_plugin.is_deprecated
+        return False
+
+    def set_filter_criteria(self, filter_value):
+        self.filter_criteria = filter_value
+        self.invalidateFilter()
+
+
+class DisplayPluginModel(QAbstractTableModel):
+
+    def __init__(self, display_plugins):
+        QAbstractTableModel.__init__(self)
+        self.display_plugins = display_plugins
+        self.headers = map(QVariant, [_('Plugin Name'), _('Donate'), _('Status'), _('Installed'),
+                                      _('Available'), _('Released'), _('Calibre'), _('Author')])
+
+    def rowCount(self, *args):
+        return len(self.display_plugins)
+
+    def columnCount(self, *args):
+        return len(self.headers)
+
+    def headerData(self, section, orientation, role):
+        if role == Qt.DisplayRole and orientation == Qt.Horizontal:
+            return self.headers[section]
+        return NONE
+
+    def data(self, index, role):
+        if not index.isValid():
+            return NONE;
+        row, col = index.row(), index.column()
+        if row < 0 or row >= self.rowCount():
+            return NONE
+        display_plugin = self.display_plugins[row]
+        if role in [Qt.DisplayRole, Qt.UserRole]:
+            if col == 0:
+                return QVariant(display_plugin.name)
+            if col == 1:
+                if display_plugin.donation_link:
+                    return QVariant(_('PayPal'))
+            if col == 2:
+                return self._get_status(display_plugin)
+            if col == 3:
+                return QVariant(self._get_display_version(display_plugin.installed_version))
+            if col == 4:
+                return QVariant(self._get_display_version(display_plugin.available_version))
+            if col == 5:
+                if role == Qt.UserRole:
+                    return self._get_display_release_date(display_plugin.release_date, 'yyyyMMdd')
+                else:
+                    return self._get_display_release_date(display_plugin.release_date)
+            if col == 6:
+                return QVariant(self._get_display_version(display_plugin.calibre_required_version))
+            if col == 7:
+                return QVariant(display_plugin.author)
+        elif role == Qt.DecorationRole:
+            if col == 0:
+                return self._get_status_icon(display_plugin)
+            if col == 1:
+                if display_plugin.donation_link:
+                    return QIcon(I('donate.png'))
+        elif role == Qt.ToolTipRole:
+            if col == 1 and display_plugin.donation_link:
+                return QVariant(_('This plugin is FREE but you can reward the developer for their effort\n'
+                                  'by donating to them via PayPal.\n\n'
+                                  'Right-click and choose Donate to reward: ')+display_plugin.author)
+            else:
+                return self._get_status_tooltip(display_plugin)
+        elif role == Qt.ForegroundRole:
+            if col != 1: # Never change colour of the donation column
+                if display_plugin.is_deprecated:
+                    return QVariant(QBrush(Qt.blue))
+                if display_plugin.is_disabled():
+                    return QVariant(QBrush(Qt.gray))
+        return NONE
+
+    def plugin_to_index(self, display_plugin):
+        for i, p in enumerate(self.display_plugins):
+            if display_plugin == p:
+                return self.index(i, 0, QModelIndex())
+        return QModelIndex()
+
+    def refresh_plugin(self, display_plugin):
+        idx = self.plugin_to_index(display_plugin)
+        self.dataChanged.emit(idx, idx)
+
+    def _get_display_release_date(self, date_value, format='dd MMM yyyy'):
+        if date_value and date_value != UNDEFINED_DATE:
+            return QVariant(format_date(date_value, format))
+        return NONE
+
+    def _get_display_version(self, version):
+        if version is None:
+            return ''
+        return '.'.join([str(v) for v in list(version)])
+
+    def _get_status(self, display_plugin):
+        if not display_plugin.is_valid_platform():
+            return _('Platform unavailable')
+        if not display_plugin.is_valid_calibre():
+            return _('Calibre upgrade required')
+        if display_plugin.is_installed():
+            if display_plugin.is_deprecated:
+                return _('Plugin deprecated')
+            elif display_plugin.is_upgrade_available():
+                return _('New version available')
+            else:
+                return _('Latest version installed')
+        return _('Not installed')
+
+    def _get_status_icon(self, display_plugin):
+        if display_plugin.is_deprecated:
+            icon_name = 'plugin_deprecated.png'
+        elif display_plugin.is_disabled():
+            if display_plugin.is_upgrade_available():
+                if display_plugin.is_valid_to_install():
+                    icon_name = 'plugin_disabled_valid.png'
+                else:
+                    icon_name = 'plugin_disabled_invalid.png'
+            else:
+                icon_name = 'plugin_disabled_ok.png'
+        elif display_plugin.is_installed():
+            if display_plugin.is_upgrade_available():
+                if display_plugin.is_valid_to_install():
+                    icon_name = 'plugin_upgrade_valid.png'
+                else:
+                    icon_name = 'plugin_upgrade_invalid.png'
+            else:
+                icon_name = 'plugin_upgrade_ok.png'
+        else: # A plugin available not currently installed
+            if display_plugin.is_valid_to_install():
+                icon_name = 'plugin_new_valid.png'
+            else:
+                icon_name = 'plugin_new_invalid.png'
+        return QIcon(I('plugins/' + icon_name))
+
+    def _get_status_tooltip(self, display_plugin):
+        if display_plugin.is_deprecated:
+            return QVariant(_('This plugin has been deprecated and should be uninstalled')+'\n\n'+
+                            _('Right-click to see more options'))
+        if not display_plugin.is_valid_platform():
+            return QVariant(_('This plugin can only be installed on: %s') % \
+                            ', '.join(display_plugin.platforms)+'\n\n'+
+                            _('Right-click to see more options'))
+        if numeric_version < display_plugin.calibre_required_version:
+            return QVariant(_('You must upgrade to at least Calibre %s before installing this plugin') % \
+                            self._get_display_version(display_plugin.calibre_required_version)+'\n\n'+
+                            _('Right-click to see more options'))
+        if display_plugin.installed_version < display_plugin.available_version:
+            if display_plugin.installed_version is None:
+                return QVariant(_('You can install this plugin')+'\n\n'+
+                                _('Right-click to see more options'))
+            else:
+                return QVariant(_('A new version of this plugin is available')+'\n\n'+
+                                _('Right-click to see more options'))
+        return QVariant(_('This plugin is installed and up-to-date')+'\n\n'+
+                        _('Right-click to see more options'))
+
+
+class PluginUpdaterDialog(SizePersistedDialog):
+
+    initial_extra_size = QSize(350, 100)
+
+    def __init__(self, gui, initial_filter=FILTER_UPDATE_AVAILABLE):
+        SizePersistedDialog.__init__(self, gui, 'Plugin Updater plugin:plugin updater dialog')
+        self.gui = gui
+        self.forum_link = None
+        self.model = None
+        self._initialize_controls()
+        self._create_context_menu()
+
+        display_plugins = read_available_plugins()
+
+        if display_plugins:
+            self.model = DisplayPluginModel(display_plugins)
+            self.proxy_model = DisplayPluginSortFilterModel(self)
+            self.proxy_model.setSourceModel(self.model)
+            self.plugin_view.setModel(self.proxy_model)
+            self.plugin_view.resizeColumnsToContents()
+            self.plugin_view.selectionModel().currentRowChanged.connect(self._plugin_current_changed)
+            self.plugin_view.doubleClicked.connect(self.install_button.click)
+            self.filter_combo.setCurrentIndex(initial_filter)
+            self._select_and_focus_view()
+        else:
+            error_dialog(self.gui, _('Update Check Failed'),
+                        _('Unable to reach the MobileRead plugins forum index page.'),
+                        det_msg=MR_INDEX_URL, show=True)
+            self.filter_combo.setEnabled(False)
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+
+    def _initialize_controls(self):
+        self.setWindowTitle(_('User plugins'))
+        self.setWindowIcon(QIcon(I('plugins/plugin_updater.png')))
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, 'plugins/plugin_updater.png',
+                _('User Plugins'))
+        layout.addLayout(title_layout)
+
+        header_layout = QHBoxLayout()
+        layout.addLayout(header_layout)
+        self.filter_combo = PluginFilterComboBox(self)
+        self.filter_combo.setMinimumContentsLength(20)
+        self.filter_combo.currentIndexChanged[int].connect(self._filter_combo_changed)
+        header_layout.addWidget(QLabel(_('Filter list of plugins')+':', self))
+        header_layout.addWidget(self.filter_combo)
+        header_layout.addStretch(10)
+
+        self.plugin_view = QTableView(self)
+        self.plugin_view.horizontalHeader().setStretchLastSection(True)
+        self.plugin_view.setSelectionBehavior(QAbstractItemView.SelectRows)
+        self.plugin_view.setSelectionMode(QAbstractItemView.SingleSelection)
+        self.plugin_view.setAlternatingRowColors(True)
+        self.plugin_view.setSortingEnabled(True)
+        self.plugin_view.setIconSize(QSize(28, 28))
+        layout.addWidget(self.plugin_view)
+
+        details_layout = QHBoxLayout()
+        layout.addLayout(details_layout)
+        forum_label = QLabel('<a href="http://www.foo.com/">Plugin Forum Thread</a>', self)
+        forum_label.setTextInteractionFlags(Qt.LinksAccessibleByMouse | Qt.LinksAccessibleByKeyboard)
+        forum_label.linkActivated.connect(self._forum_label_activated)
+        details_layout.addWidget(QLabel(_('Description')+':', self), 0, Qt.AlignLeft)
+        details_layout.addWidget(forum_label, 1, Qt.AlignRight)
+
+        self.description = QLabel(self)
+        self.description.setFrameStyle(QFrame.Panel | QFrame.Sunken)
+        self.description.setAlignment(Qt.AlignTop | Qt.AlignLeft)
+        self.description.setMinimumHeight(40)
+        layout.addWidget(self.description)
+
+        self.button_box = QDialogButtonBox(QDialogButtonBox.Close)
+        self.button_box.rejected.connect(self._close_clicked)
+        self.install_button = self.button_box.addButton(_('&Install'), QDialogButtonBox.AcceptRole)
+        self.install_button.setToolTip(_('Install the selected plugin'))
+        self.install_button.clicked.connect(self._install_clicked)
+        self.install_button.setEnabled(False)
+        self.configure_button = self.button_box.addButton(' '+_('&Customize plugin ')+' ', QDialogButtonBox.ResetRole)
+        self.configure_button.setToolTip(_('Customize the options for this plugin'))
+        self.configure_button.clicked.connect(self._configure_clicked)
+        self.configure_button.setEnabled(False)
+        layout.addWidget(self.button_box)
+
+    def _create_context_menu(self):
+        self.plugin_view.setContextMenuPolicy(Qt.ActionsContextMenu)
+        self.install_action = QAction(QIcon(I('plugins/plugin_upgrade_ok.png')), _('&Install'), self)
+        self.install_action.setToolTip(_('Install the selected plugin'))
+        self.install_action.triggered.connect(self._install_clicked)
+        self.install_action.setEnabled(False)
+        self.plugin_view.addAction(self.install_action)
+        self.history_action = QAction(QIcon(I('chapters.png')), _('Version &History'), self)
+        self.history_action.setToolTip(_('Show history of changes to this plugin'))
+        self.history_action.triggered.connect(self._history_clicked)
+        self.history_action.setEnabled(False)
+        self.plugin_view.addAction(self.history_action)
+        self.forum_action = QAction(QIcon(I('plugins/mobileread.png')), _('Plugin &Forum Thread'), self)
+        self.forum_action.triggered.connect(self._forum_label_activated)
+        self.forum_action.setEnabled(False)
+        self.plugin_view.addAction(self.forum_action)
+
+        sep1 = QAction(self)
+        sep1.setSeparator(True)
+        self.plugin_view.addAction(sep1)
+
+        self.toggle_enabled_action = QAction(_('Enable/&Disable plugin'), self)
+        self.toggle_enabled_action.setToolTip(_('Enable or disable this plugin'))
+        self.toggle_enabled_action.triggered.connect(self._toggle_enabled_clicked)
+        self.toggle_enabled_action.setEnabled(False)
+        self.plugin_view.addAction(self.toggle_enabled_action)
+        self.uninstall_action = QAction(_('&Remove plugin'), self)
+        self.uninstall_action.setToolTip(_('Uninstall the selected plugin'))
+        self.uninstall_action.triggered.connect(self._uninstall_clicked)
+        self.uninstall_action.setEnabled(False)
+        self.plugin_view.addAction(self.uninstall_action)
+
+        sep2 = QAction(self)
+        sep2.setSeparator(True)
+        self.plugin_view.addAction(sep2)
+
+        self.donate_enabled_action = QAction(QIcon(I('donate.png')), _('Donate to developer'), self)
+        self.donate_enabled_action.setToolTip(_('Donate to the developer of this plugin'))
+        self.donate_enabled_action.triggered.connect(self._donate_clicked)
+        self.donate_enabled_action.setEnabled(False)
+        self.plugin_view.addAction(self.donate_enabled_action)
+
+        sep3 = QAction(self)
+        sep3.setSeparator(True)
+        self.plugin_view.addAction(sep3)
+
+        self.configure_action = QAction(QIcon(I('config.png')), _('&Customize plugin'), self)
+        self.configure_action.setToolTip(_('Customize the options for this plugin'))
+        self.configure_action.triggered.connect(self._configure_clicked)
+        self.configure_action.setEnabled(False)
+        self.plugin_view.addAction(self.configure_action)
+
+    def _close_clicked(self):
+        # Force our toolbar/action to be updated based on uninstalled updates
+        if self.model:
+            update_plugins = filter(filter_upgradeable_plugins, self.model.display_plugins)
+            self.gui.recalc_update_label(len(update_plugins))
+        self.reject()
+
+    def _plugin_current_changed(self, current, previous):
+        if current.isValid():
+            actual_idx = self.proxy_model.mapToSource(current)
+            display_plugin = self.model.display_plugins[actual_idx.row()]
+            self.description.setText(display_plugin.description)
+            self.forum_link = display_plugin.forum_link
+            self.forum_action.setEnabled(bool(self.forum_link))
+            self.install_button.setEnabled(display_plugin.is_valid_to_install())
+            self.install_action.setEnabled(self.install_button.isEnabled())
+            self.uninstall_action.setEnabled(display_plugin.is_installed())
+            self.history_action.setEnabled(display_plugin.has_changelog)
+            self.configure_button.setEnabled(display_plugin.is_installed())
+            self.configure_action.setEnabled(self.configure_button.isEnabled())
+            self.toggle_enabled_action.setEnabled(display_plugin.is_installed())
+            self.donate_enabled_action.setEnabled(bool(display_plugin.donation_link))
+        else:
+            self.description.setText('')
+            self.forum_link = None
+            self.forum_action.setEnabled(False)
+            self.install_button.setEnabled(False)
+            self.install_action.setEnabled(False)
+            self.uninstall_action.setEnabled(False)
+            self.history_action.setEnabled(False)
+            self.configure_button.setEnabled(False)
+            self.configure_action.setEnabled(False)
+            self.toggle_enabled_action.setEnabled(False)
+            self.donate_enabled_action.setEnabled(False)
+
+    def _donate_clicked(self):
+        plugin = self._selected_display_plugin()
+        if plugin and plugin.donation_link:
+            open_url(QUrl(plugin.donation_link))
+
+    def _select_and_focus_view(self, change_selection=True):
+        if change_selection and self.plugin_view.model().rowCount() > 0:
+            self.plugin_view.selectRow(0)
+        else:
+            idx = self.plugin_view.selectionModel().currentIndex()
+            self._plugin_current_changed(idx, 0)
+        self.plugin_view.setFocus()
+
+    def _filter_combo_changed(self, idx):
+        self.proxy_model.set_filter_criteria(idx)
+        if idx == FILTER_NOT_INSTALLED:
+            self.plugin_view.sortByColumn(5, Qt.DescendingOrder)
+        else:
+            self.plugin_view.sortByColumn(0, Qt.AscendingOrder)
+        self._select_and_focus_view()
+
+    def _forum_label_activated(self):
+        if self.forum_link:
+            open_url(QUrl(self.forum_link))
+
+    def _selected_display_plugin(self):
+        idx = self.plugin_view.selectionModel().currentIndex()
+        actual_idx = self.proxy_model.mapToSource(idx)
+        return self.model.display_plugins[actual_idx.row()]
+
+    def _uninstall_plugin(self, name_to_remove):
+        if DEBUG:
+            prints('Removing plugin: ', name_to_remove)
+        remove_plugin(name_to_remove)
+        # Make sure that any other plugins that required this plugin
+        # to be uninstalled first have the requirement removed
+        for display_plugin in self.model.display_plugins:
+            # Make sure we update the status and display of the
+            # plugin we just uninstalled
+            if name_to_remove in display_plugin.uninstall_plugins:
+                if DEBUG:
+                    prints('Removing uninstall dependency for: ', display_plugin.name)
+                display_plugin.uninstall_plugins.remove(name_to_remove)
+            if display_plugin.name == name_to_remove:
+                if DEBUG:
+                    prints('Resetting plugin to uninstalled status: ', display_plugin.name)
+                display_plugin.installed_version = None
+                display_plugin.plugin = None
+                display_plugin.uninstall_plugins = []
+                if self.proxy_model.filter_criteria not in [FILTER_INSTALLED, FILTER_UPDATE_AVAILABLE]:
+                    self.model.refresh_plugin(display_plugin)
+
+    def _uninstall_clicked(self):
+        display_plugin = self._selected_display_plugin()
+        if not question_dialog(self, _('Are you sure?'), '<p>'+
+                   _('Are you sure you want to uninstall the <b>%s</b> plugin?')%display_plugin.name,
+                   show_copy_button=False):
+            return
+        self._uninstall_plugin(display_plugin.name)
+        if self.proxy_model.filter_criteria in [FILTER_INSTALLED, FILTER_UPDATE_AVAILABLE]:
+            self.model.reset()
+            self._select_and_focus_view()
+        else:
+            self._select_and_focus_view(change_selection=False)
+
+    def _install_clicked(self):
+        display_plugin = self._selected_display_plugin()
+        if not question_dialog(self, _('Install %s')%display_plugin.name, '<p>' + \
+                _('Installing plugins is a <b>security risk</b>. '
+                'Plugins can contain a virus/malware. '
+                    'Only install it if you got it from a trusted source.'
+                    ' Are you sure you want to proceed?'),
+                show_copy_button=False):
+            return
+
+        if display_plugin.uninstall_plugins:
+            uninstall_names = list(display_plugin.uninstall_plugins)
+            if DEBUG:
+                prints('Uninstalling plugin: ', ', '.join(uninstall_names))
+            for name_to_remove in uninstall_names:
+                self._uninstall_plugin(name_to_remove)
+
+        if DEBUG:
+            prints('Locating zip file for %s: %s'% (display_plugin.name, display_plugin.forum_link))
+        self.gui.status_bar.showMessage(_('Locating zip file for %s: %s') % (display_plugin.name, display_plugin.forum_link))
+        plugin_zip_url = self._read_zip_attachment_url(display_plugin.forum_link)
+        if not plugin_zip_url:
+            return error_dialog(self.gui, _('Install Plugin Failed'),
+                        _('Unable to locate a plugin zip file for <b>%s</b>') % display_plugin.name,
+                        det_msg=display_plugin.forum_link, show=True)
+
+        if DEBUG:
+            prints('Downloading plugin zip attachment: ', plugin_zip_url)
+        self.gui.status_bar.showMessage(_('Downloading plugin zip attachment: %s') % plugin_zip_url)
+        zip_path = self._download_zip(plugin_zip_url)
+
+        if DEBUG:
+            prints('Installing plugin: ', zip_path)
+        self.gui.status_bar.showMessage(_('Installing plugin: %s') % zip_path)
+
+        try:
+            try:
+                plugin = add_plugin(zip_path)
+            except NameConflict as e:
+                return error_dialog(self.gui, _('Already exists'),
+                        unicode(e), show=True)
+            # Check for any toolbars to add to.
+            widget = ConfigWidget(self.gui)
+            widget.gui = self.gui
+            widget.check_for_add_to_toolbars(plugin)
+            self.gui.status_bar.showMessage(_('Plugin installed: %s') % display_plugin.name)
+            info_dialog(self.gui, _('Success'),
+                    _('Plugin <b>{0}</b> successfully installed under <b>'
+                        ' {1} plugins</b>. You may have to restart calibre '
+                        'for the plugin to take effect.').format(plugin.name, plugin.type),
+                    show=True, show_copy_button=False)
+
+            display_plugin.plugin = plugin
+            # We cannot read the 'actual' version information as the plugin will not be loaded yet
+            display_plugin.installed_version = display_plugin.available_version
+        except:
+            if DEBUG:
+                prints('ERROR occurred while installing plugin: %s'%display_plugin.name)
+                traceback.print_exc()
+            error_dialog(self.gui, _('Install Plugin Failed'),
+                         _('A problem occurred while installing this plugin.'
+                           ' This plugin will now be uninstalled.'
+                           ' Please post the error message in details below into'
+                           ' the forum thread for this plugin and restart Calibre.'),
+                         det_msg=traceback.format_exc(), show=True)
+            if DEBUG:
+                prints('Due to error now uninstalling plugin: %s'%display_plugin.name)
+                remove_plugin(display_plugin.name)
+                display_plugin.plugin = None
+
+        display_plugin.uninstall_plugins = []
+        if self.proxy_model.filter_criteria in [FILTER_NOT_INSTALLED, FILTER_UPDATE_AVAILABLE]:
+            self.model.reset()
+            self._select_and_focus_view()
+        else:
+            self.model.refresh_plugin(display_plugin)
+            self._select_and_focus_view(change_selection=False)
+
+    def _history_clicked(self):
+        display_plugin = self._selected_display_plugin()
+        text = self._read_version_history_html(display_plugin.forum_link)
+        if text:
+            dlg = VersionHistoryDialog(self, display_plugin.name, text)
+            dlg.exec_()
+        else:
+            return error_dialog(self, _('Version history missing'),
+                _('Unable to find the version history for %s')%display_plugin.name,
+                show=True)
+
+    def _configure_clicked(self):
+        display_plugin = self._selected_display_plugin()
+        plugin = display_plugin.plugin
+        if not plugin.is_customizable():
+            return info_dialog(self, _('Plugin not customizable'),
+                _('Plugin: %s does not need customization')%plugin.name, show=True)
+        from calibre.customize import InterfaceActionBase
+        if isinstance(plugin, InterfaceActionBase) and not getattr(plugin,
+                'actual_iaction_plugin_loaded', False):
+            return error_dialog(self, _('Must restart'),
+                    _('You must restart calibre before you can'
+                        ' configure the <b>%s</b> plugin')%plugin.name, show=True)
+        plugin.do_user_config(self.parent())
+
+    def _toggle_enabled_clicked(self):
+        display_plugin = self._selected_display_plugin()
+        plugin = display_plugin.plugin
+        if not plugin.can_be_disabled:
+            return error_dialog(self,_('Plugin cannot be disabled'),
+                         _('The plugin: %s cannot be disabled')%plugin.name, show=True)
+        if is_disabled(plugin):
+            enable_plugin(plugin)
+        else:
+            disable_plugin(plugin)
+        self.model.refresh_plugin(display_plugin)
+
+    def _read_version_history_html(self, forum_link):
+        br = browser()
+        br.set_handle_gzip(True)
+        try:
+            raw = br.open_novisit(forum_link).read()
+            if not raw:
+                return None
+        except:
+            traceback.print_exc()
+            return None
+        raw = raw.decode('utf-8', errors='replace')
+        root = html.fromstring(raw)
+        spoiler_nodes = root.xpath('//div[@class="smallfont" and strong="Spoiler"]')
+        for spoiler_node in spoiler_nodes:
+            try:
+                if spoiler_node.getprevious() is None:
+                    # This is a spoiler node that has been indented using [INDENT]
+                    # Need to go up to parent div, then previous node to get header
+                    heading_node = spoiler_node.getparent().getprevious()
+                else:
+                    # This is a spoiler node after a BR tag from the heading
+                    heading_node = spoiler_node.getprevious().getprevious()
+                if heading_node is None:
+                    continue
+                if heading_node.text_content().lower().find('version history') != -1:
+                    div_node = spoiler_node.xpath('div')[0]
+                    text = html.tostring(div_node, method='html', encoding=unicode)
+                    return re.sub('<div\s.*?>', '<div>', text)
+            except:
+                if DEBUG:
+                    prints('======= MobileRead Parse Error =======')
+                    traceback.print_exc()
+                    prints(html.tostring(spoiler_node))
+        return None
+
+    def _read_zip_attachment_url(self, forum_link):
+        br = browser()
+        br.set_handle_gzip(True)
+        try:
+            raw = br.open_novisit(forum_link).read()
+            if not raw:
+                return None
+        except:
+            traceback.print_exc()
+            return None
+        raw = raw.decode('utf-8', errors='replace')
+        root = html.fromstring(raw)
+        attachment_nodes = root.xpath('//fieldset/table/tr/td/a')
+        for attachment_node in attachment_nodes:
+            try:
+                filename = attachment_node.text_content().lower()
+                if filename.find('.zip') != -1:
+                    full_url = MR_URL + attachment_node.attrib['href']
+                    return full_url
+            except:
+                if DEBUG:
+                    prints('======= MobileRead Parse Error =======')
+                    traceback.print_exc()
+                    prints(html.tostring(attachment_node))
+        return None
+
+    def _download_zip(self, plugin_zip_url):
+        from calibre.ptempfile import PersistentTemporaryFile
+        br = browser()
+        br.set_handle_gzip(True)
+        raw = br.open_novisit(plugin_zip_url).read()
+        pt = PersistentTemporaryFile('.zip')
+        pt.write(raw)
+        pt.close()
+        return pt.name

src/calibre/gui2/dialogs/template_dialog.py

@@ -254,6 +254,15 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
         self.textbox_changed()
         self.rule = (None, '')
 
+        tt = _('Template language tutorial')
+        self.template_tutorial.setText(
+                '<a href="http://manual.calibre-ebook.com/template_lang.html">'
+                '%s</a>'%tt)
+        tt = _('Template function reference')
+        self.template_func_reference.setText(
+                '<a href="http://manual.calibre-ebook.com/template_ref.html">'
+                '%s</a>'%tt)
+
     def textbox_changed(self):
         cur_text = unicode(self.textbox.toPlainText())
         if self.last_text != cur_text:

src/calibre/gui2/dialogs/template_dialog.ui

@@ -125,6 +125,20 @@
      <item row="9" column="1">
       <widget class="QPlainTextEdit" name="source_code"/>
      </item>
+     <item row="10" column="1">
+      <widget class="QLabel" name="template_tutorial">
+       <property name="openExternalLinks">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="11" column="1">
+      <widget class="QLabel" name="template_func_reference">
+       <property name="openExternalLinks">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
     </layout>
    </item>
   </layout>