merge with John's branch

Changelog.yaml

@@ -19,6 +19,74 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.4
+  date: 2011-06-03
+
+  new features:
+    - title: "New and much simpler interface for specifying column coloring via Preferences->Look & Feel->Column Coloring"
+
+    - title: "Driver for Trekstor eBook Player 5M, Samsung Galaxy SII I9100, Motorola Defy and miBuk GAMMA 6.2"
+      tickets: [792091, 791216]
+
+    - title: "Get Books: Add EpubBud, WH Smits and E-book Shoppe stores"
+
+    - title: "When deleting 'all formats except ...', do not delete if it leaves a book with no formats"
+
+    - title: "Change default toolbar to make it a little more new user friendly. The icons have been re-arranged and now the text is always visiblke by default. You can change that in Preferences->Look & Feeel and Preferences->Toolbar"
+
+    - title: "Windows installer: Remember and use previous settings for installing desktop icons, adding to path, etc. This makes the installer a little slower, complaints should go to Microsoft."
+
+    - title: "Template language: Add str_in_list and on_device formatter functions. Make debugging templates a little easier"
+
+    - title: "Allow the user to specify formatting for number type custom columns"
+  
+  bug fixes:
+    - title: "Fix typo in NOOK TSR driver that prevented it from working on windows"
+
+    - title: "Fix quotes in identifiers causing Tag Browser to be blank."
+      tickets: [791044]
+
+    - title: "Speedup auto complete when there are lots of items (>2500) the downside being that non ASCII characters are not sorted correctly. The threshold can be controlled via Preferences->Tweaks"
+      tickets: [792191]
+
+    - title: "RTF Output: Fix handling of curly brackets"
+      tickets: [791805]
+
+    - title: "Fix searching in Get Books not working with non ASCII characters"
+      tickets: [791788]
+
+    - title: "Fix excessive memory consumption when moving very large files during a metadata change"
+      tickets: [791806]
+
+    - title: "Fix series index being overwritten even when series is turned off in bulk metadata download"
+      tickets: [789990]
+
+    - title: "Fix regression in templates where id and other non standard fields no longer worked."
+
+    - title: "EPUB Output: Fix crash caused by ids with non-ascii characters in them"
+
+    - title: "Try to preserve the timestamps of files in a ZIP container"
+
+    - title: "After adding books always select the most recently added book."
+      tickets: [789343]
+
+  improved recipes:
+    - bild.de
+    - CNN
+    - BBC News (fast)
+    - Dilema Veche
+
+  new recipes:
+    - title: Metro UK
+      author: Dave Asbury
+
+    - title: Alt om Herning and Version2.dk
+      author: Rasmus Lauritsen
+
+    - title: Observatorul cultural
+      author: song2
+
+
 - version: 0.8.3
   date: 2011-05-27
 

recipes/bild_de.recipe

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 from calibre.web.feeds.recipes import BasicNewsRecipe
 class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
     title          = u'Bild.de'
     __author__  = 'schuster'
     oldest_article = 1
-    max_articles_per_feed = 50
+    max_articles_per_feed = 100
     no_stylesheets         = True
     use_embedded_content   = False
     language               = 'de'
@@ -12,11 +13,25 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
 # get cover from myspace
     cover_url = 'http://a3.l3-images.myspacecdn.com/images02/56/0232f842170b4d349779f8379c27e073/l.jpg'
+    masthead_url = 'http://a3.l3-images.myspacecdn.com/images02/56/0232f842170b4d349779f8379c27e073/l.jpg'
 
 # set what to fetch on the site
     remove_tags_before =  dict(name = 'h2', attrs={'id':'cover'})
     remove_tags_after = dict(name ='div', attrs={'class':'back'})
 
+
+# remove things on the site that we don't want
+    remove_tags = [dict(name='div', attrs={'class':'credit'}),
+        dict(name='div', attrs={'class':'index'}),
+        dict(name='div', attrs={'id':'zstart31'}),
+        dict(name='div', attrs={'class':'hentry'}),
+        dict(name='div', attrs={'class':'back'}),
+        dict(name='div', attrs={'class':'pagination'}),
+        dict(name='div', attrs={'class':'header'}),
+        dict(name='div', attrs={'class':'element floatL'}),
+        dict(name='div', attrs={'class':'stWrap'})
+]
+
 #  thanx to kiklop74 for code (see sticky thread -> Recipes - Re-usable code)
 # this one removes a lot of direct-link's
     def preprocess_html(self, soup):
@@ -42,5 +57,18 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
                           (u'Unterhaltung', u'http://rss.bild.de/bild-unterhaltung.xml'),
                           (u'Sport', u'http://rss.bild.de/bild-sport.xml'),
                           (u'Lifestyle', u'http://rss.bild.de/bild-lifestyle.xml'),
-                          (u'Ratgeber', u'http://rss.bild.de/bild-ratgeber.xml')
+                          (u'Ratgeber', u'http://rss.bild.de/bild-ratgeber.xml'),
+                          (u'Reg. - Berlin', u'http://rss.bild.de/bild-berlin.xml'),
+                          (u'Reg. - Bremen', u'http://rss.bild.de/bild-bremen.xml'),
+                          (u'Reg. - Dresden', u'http://rss.bild.de/bild-dresden.xml'),
+                          (u'Reg. - Düsseldorf', u'http://rss.bild.de/bild-duesseldorf.xml'),
+                          (u'Reg. - Frankfurt-Main', u'http://rss.bild.de/bild-frankfurt-main.xml'),
+                          (u'Reg. - Hamburg', u'http://rss.bild.de/bild-hamburg.xml'),
+                          (u'Reg. - Hannover', u'http://rss.bild.de/bild-hannover.xml'),
+                          (u'Reg. - Köln', u'http://rss.bild.de/bild-koeln.xml'),
+                          (u'Reg. - Leipzig', u'http://rss.bild.de/bild-leipzig.xml'),
+                          (u'Reg. - München', u'http://rss.bild.de/bild-muenchen.xml'),
+                          (u'Reg. - Ruhrgebiet', u'http://rss.bild.de/bild-ruhrgebiet.xml'),
+                          (u'Reg. - Stuttgart', u'http://rss.bild.de/bild-stuttgart.xml')
 ]
+

recipes/brigitte_de.recipe

@@ -0,0 +1,36 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+    title          = u'Brigitte.de'
+    __author__  = 'schuster'
+    oldest_article = 14
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    remove_empty_feeds = True
+    timeout = 10
+    cover_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg'
+    masthead_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg'
+
+
+    remove_tags = [dict(attrs={'class':['linklist', 'head', 'indent right relatedContent', 'artikel-meta segment', 'segment', 'comment commentFormWrapper segment borderBG', 'segment borderBG comments', 'segment borderBG box', 'center', 'segment nextPageLink', 'inCar']}),
+                dict(id=['header', 'artTools', 'context', 'interact', 'footer-navigation', 'bwNet', 'copy', 'keyboardNavigationHint']),
+                dict(name=['hjtrs', 'kud'])]
+
+    feeds          = [(u'Mode', u'http://www.brigitte.de/mode/feed.rss'),
+                          (u'Beauty', u'http://www.brigitte.de/beauty/feed.rss'),
+                          (u'Luxus', u'http://www.brigitte.de/luxus/feed.rss'),
+                          (u'Figur', u'http://www.brigitte.de/figur/feed.rss'),
+                          (u'Gesundheit', u'http://www.brigitte.de/gesundheit/feed.rss'),
+                          (u'Liebe&Sex', u'http://www.brigitte.de/liebe-sex/feed.rss'),
+                          (u'Gesellschaft', u'http://www.brigitte.de/gesellschaft/feed.rss'),
+                          (u'Kultur', u'http://www.brigitte.de/kultur/feed.rss'),
+                          (u'Reise', u'http://www.brigitte.de/reise/feed.rss'),
+                          (u'Kochen', u'http://www.brigitte.de/kochen/feed.rss'),
+                          (u'Wohnen', u'http://www.brigitte.de/wohnen/feed.rss'),
+                          (u'Job', u'http://www.brigitte.de/job/feed.rss'),
+                          (u'Erfahrungen', u'http://www.brigitte.de/erfahrungen/feed.rss'),
+]

recipes/express_de.recipe

@@ -1,5 +1,4 @@
 from calibre.web.feeds.news import BasicNewsRecipe
-
 class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
     title          = u'Express.de'
@@ -12,7 +11,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
     extra_css = '''
                     h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small;}
                     h1{ font-family:Arial,Helvetica,sans-serif;  font-size:x-large; font-weight:bold;}
-
                 '''
     remove_javascript      = True
     remove_tags_befor = [dict(name='div', attrs={'class':'Datum'})]
@@ -25,6 +23,7 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
            dict(id='Logo'),
            dict(id='MainLinkSpacer'),
            dict(id='MainLinks'),
+           dict(id='ContainerPfad'), #neu
            dict(title='Diese Seite Bookmarken'),
 
                                            dict(name='span'),
@@ -44,7 +43,8 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
            dict(name='div', attrs={'class':'HeaderSearch'}),
            dict(name='div', attrs={'class':'sbutton'}),
            dict(name='div', attrs={'class':'active'}),
-
+           dict(name='div', attrs={'class':'MoreNews'}), #neu
+           dict(name='div', attrs={'class':'ContentBoxSubline'}) #neu
 ]
 
 
@@ -68,7 +68,5 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
                           (u'Fortuna D~Dorf', u'http://www.express.de/sport/fussball/fortuna/-/3292/3292/-/view/asFeed/-/index.xml'),
                           (u'Basketball News', u'http://www.express.de/sport/basketball/-/3190/3190/-/view/asFeed/-/index.xml'),
                           (u'Big Brother', u'http://www.express.de/news/promi-show/big-brother/-/2402/2402/-/view/asFeed/-/index.xml'),
+                    ]
 
-
-
-]

recipes/guardian.recipe

@@ -28,7 +28,7 @@ class Guardian(BasicNewsRecipe):
     # List of section titles to ignore
     # For example: ['Sport']
     ignore_sections = []
-
+    
     timefmt = ' [%a, %d %b %Y]'
     keep_only_tags = [
                       dict(name='div', attrs={'id':["content","article_header","main-article-info",]}),
@@ -87,8 +87,14 @@ class Guardian(BasicNewsRecipe):
         idx = soup.find('div', id='book-index')
         for s in idx.findAll('strong', attrs={'class':'book'}):
             a = s.find('a', href=True)
-            yield (self.tag_to_string(a), a['href'])
+            section_title = self.tag_to_string(a)
-
+            if not section_title in self.ignore_sections:
+                prefix = ''
+                if section_title != 'Main section':
+                    prefix = section_title + ': '
+                for subsection in s.parent.findAll('a', attrs={'class':'book-section'}):
+                    yield (prefix + self.tag_to_string(subsection), subsection['href'])
+    
     def find_articles(self, url):
         soup = self.index_to_soup(url)
         div = soup.find('div', attrs={'class':'book-index'})
@@ -109,15 +115,12 @@ class Guardian(BasicNewsRecipe):
                         'title': title, 'url':url, 'description':desc,
                         'date' : strftime('%a, %d %b'),
                         }
-
+    
     def parse_index(self):
         try:
             feeds = []
             for title, href in self.find_sections():
-                if not title in self.ignore_sections:
+                feeds.append((title, list(self.find_articles(href))))
-                    feeds.append((title, list(self.find_articles(href))))
             return feeds
         except:
             raise NotImplementedError
-
-

recipes/heise_online.recipe

@@ -0,0 +1,52 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+    title = 'Heise-online'
+    description = 'News vom Heise-Verlag'
+    __author__ = 'schuster'
+    use_embedded_content   = False
+    language = 'de'
+    oldest_article = 2
+    max_articles_per_feed = 35
+    rescale_images = True
+    remove_empty_feeds = True
+    timeout = 5
+    no_stylesheets = True
+
+
+    remove_tags_after = dict(name ='p', attrs={'class':'editor'})
+    remove_tags = [dict(id='navi_top_container'),
+                            dict(id='navi_bottom'),
+                            dict(id='mitte_rechts'),
+                            dict(id='navigation'),
+                            dict(id='subnavi'),
+                            dict(id='social_bookmarks'),
+                            dict(id='permalink'),
+                            dict(id='content_foren'),
+                            dict(id='seiten_navi'),
+                            dict(id='adbottom'),
+                            dict(id='sitemap')]
+
+    feeds =  [
+                   ('Newsticker', 'http://www.heise.de/newsticker/heise.rdf'),
+                   ('Auto', 'http://www.heise.de/autos/rss/news.rdf'),
+                   ('Foto ', 'http://www.heise.de/foto/rss/news-atom.xml'),
+                   ('Mac&i', 'http://www.heise.de/mac-and-i/news.rdf'),
+                   ('Mobile ', 'http://www.heise.de/mobil/newsticker/heise-atom.xml'),
+                   ('Netz ', 'http://www.heise.de/netze/rss/netze-atom.xml'),
+                   ('Open ', 'http://www.heise.de/open/news/news-atom.xml'),
+                   ('Resale ', 'http://www.heise.de/resale/rss/resale.rdf'),
+                   ('Security ', 'http://www.heise.de/security/news/news-atom.xml'),
+                   ('C`t', 'http://www.heise.de/ct/rss/artikel-atom.xml'),
+                   ('iX', 'http://www.heise.de/ix/news/news.rdf'),
+                   ('Mach-flott', 'http://www.heise.de/mach-flott/rss/mach-flott-atom.xml'),
+                   ('Blog: Babel-Bulletin', 'http://www.heise.de/developer/rss/babel-bulletin/blog.rdf'),
+                   ('Blog: Der Dotnet-Doktor', 'http://www.heise.de/developer/rss/dotnet-doktor/blog.rdf'),
+                   ('Blog: Bernds Management-Welt', 'http://www.heise.de/developer/rss/bernds-management-welt/blog.rdf'),
+                   ('Blog: IT conversation', 'http://www.heise.de/developer/rss/world-of-it/blog.rdf'),
+                   ('Blog: Kais bewegtes Web', 'http://www.heise.de/developer/rss/kais-bewegtes-web/blog.rdf')
+]
+
+    def print_version(self, url):
+        return url + '?view=print'
+

recipes/max_planck.recipe

@@ -3,9 +3,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
     title          = u'Max-Planck-Inst.'
     __author__  = 'schuster'
-    remove_tags = [dict(attrs={'class':['clearfix', 'lens', 'col2_box_list', 'col2_box_teaser group_ext no_print', 'dotted_line', 'col2_box_teaser', 'box_image small', 'bold', 'col2_box_teaser no_print', 'print_kontakt']}),
-                dict(id=['ie_clearing', 'col2', 'col2_content']),
-                dict(name=['script', 'noscript', 'style'])]
     oldest_article = 30
     max_articles_per_feed = 100
     no_stylesheets         = True
@@ -13,6 +10,11 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
     language               = 'de'
     remove_javascript      = True
 
+    remove_tags = [dict(attrs={'class':['box_url', 'print_kontakt']}),
+                dict(id=['skiplinks'])]
+
+
+
     def print_version(self, url):
         split_url = url.split("/")
         print_url = 'http://www.mpg.de/print/' +  split_url[3]

recipes/newsweek.recipe

@@ -69,7 +69,11 @@ class Newsweek(BasicNewsRecipe):
         for section, shref in self.newsweek_sections():
             self.log('Processing section', section, shref)
             articles = []
-            soups = [self.index_to_soup(shref)]
+            try:
+                soups = [self.index_to_soup(shref)]
+            except:
+                self.log.warn('Section %s not found, skipping'%section)
+                continue
             na = soups[0].find('a', rel='next')
             if na:
                 soups.append(self.index_to_soup(self.BASE_URL+na['href']))

recipes/polizeipress_de.recipe

@@ -0,0 +1,35 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+    title          = u'Polizeipresse - Deutschland'
+    __author__  = 'schuster'
+    description            =    'Tagesaktuelle "Polizeiberichte" aus ganz Deutschland (bis auf Ortsebene).' 'Um deinen Ort/Stadt/Kreis usw. einzubinden, gehe auf "http://www.presseportal.de/polizeipresse/" und suche im oberen "Suchfeld" nach dem Namen.'   'Oberhalb der Suchergebnisse (Folgen:) auf den üblichen link zu den RSS-Feeds klicken und den RSS-link im Rezept unter "feeds" eintragen wie üblich.'   'Die Auswahl von Orten kann vereinfacht werden wenn man den Suchbegriff wie folgt eingibt:' '"Stadt-Ort".'
+    oldest_article = 21
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    masthead_url = 'http://www.alt-heliservice.de/images/34_BPOL_Logo_4C_g_schutzbereich.jpg'
+    cover_url = 'http://berlinstadtservice.de/buerger/Bundespolizei-Logo.png'
+
+    remove_tags = [
+        dict(name='div', attrs={'id':'logo'}),
+        dict(name='div', attrs={'id':'origin'}),
+        dict(name='pre', attrs={'class':'xml_contact'})]
+
+    def print_version(self,url):
+        segments = url.split('/')
+        printURL = 'http://www.presseportal.de/print.htx?nr=' + '/'.join(segments[5:6]) + '&type=polizei'
+        return printURL
+
+    feeds          = [(u'Frimmerdorf', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-frimmersdorf&w=public_service'),
+          (u'Neurath', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-neurath&w=public_service'),
+          (u'Gustorf', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-gustorf&w=public_service'),
+          (u'Neuenhausen', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-neuenhausen&w=public_service'),
+          (u'Wevelinghoven', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-Wevelinghoven&w=public_service'),
+          (u'Grevenbroich ges.', u'http://www.presseportal.de/rss/rss2_vts.htx?q=grevenbroich&w=public_service'),
+          (u'Kreis Neuss ges.', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Rhein-Kreis+Neuss&w=public_service'),
+          ]
+

recipes/telegraph_uk.recipe

@@ -49,6 +49,7 @@ class TelegraphUK(BasicNewsRecipe):
                          (u'UK News'        , u'http://www.telegraph.co.uk/news/uknews/rss'                                      )
                         ,(u'World News'     , u'http://www.telegraph.co.uk/news/worldnews/rss'                                   )
                         ,(u'Politics'       , u'http://www.telegraph.co.uk/news/newstopics/politics/rss'                         )
+                        ,(u'Finance'        , u'http://www.telegraph.co.uk/finance/rss'                                          )
                         ,(u'Technology News', u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologynews/rss'   )
                         ,(u'UK News'        , u'http://www.telegraph.co.uk/scienceandtechnology/technology/technologyreviews/rss')
                         ,(u'Science News'   , u'http://www.telegraph.co.uk/scienceandtechnology/science/sciencenews/rss'         )

recipes/time_magazine.recipe

@@ -10,8 +10,8 @@ import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Time(BasicNewsRecipe):
-    recipe_disabled = ('This recipe has been disabled as TIME no longer'
+    #recipe_disabled = ('This recipe has been disabled as TIME no longer'
-            ' publish complete articles on the web.')
+    #        ' publish complete articles on the web.')
     title                 = u'Time'
     __author__            = 'Kovid Goyal and Sujata Raman'
     description           = 'Weekly magazine'

recipes/zaobao.recipe

@@ -82,7 +82,7 @@ class ZAOBAO(BasicNewsRecipe):
         return soup
 
     def parse_feeds(self):
-        self.log_debug(_('ZAOBAO overrided parse_feeds()'))
+        self.log(_('ZAOBAO overrided parse_feeds()'))
         parsed_feeds = BasicNewsRecipe.parse_feeds(self)
 
         for id, obj in enumerate(self.INDEXES):
@@ -99,7 +99,7 @@ class ZAOBAO(BasicNewsRecipe):
                     a_title = self.tag_to_string(a)
                     date = ''
                     description = ''
-                    self.log_debug(_('adding %s at %s')%(a_title,a_url))
+                    self.log(_('adding %s at %s')%(a_title,a_url))
                     articles.append({
                                     'title':a_title,
                                     'date':date,
@@ -110,23 +110,23 @@ class ZAOBAO(BasicNewsRecipe):
             pfeeds = feeds_from_index([(title, articles)], oldest_article=self.oldest_article,
                                      max_articles_per_feed=self.max_articles_per_feed)
 
-            self.log_debug(_('adding %s to feed')%(title))
+            self.log(_('adding %s to feed')%(title))
             for feed in pfeeds:
-                self.log_debug(_('adding feed: %s')%(feed.title))
+                self.log(_('adding feed: %s')%(feed.title))
                 feed.description = self.DESC_SENSE
                 parsed_feeds.append(feed)
                 for a, article in enumerate(feed):
-                    self.log_debug(_('added article %s from %s')%(article.title, article.url))
+                    self.log(_('added article %s from %s')%(article.title, article.url))
-                self.log_debug(_('added feed %s')%(feed.title))
+                self.log(_('added feed %s')%(feed.title))
 
         for i, feed in enumerate(parsed_feeds):
             # workaorund a strange problem: Somethimes the xml encoding is not apllied correctly by parse()
             weired_encoding_detected = False
             if not isinstance(feed.description, unicode) and self.encoding and feed.description:
-                self.log_debug(_('Feed %s is not encoded correctly, manually replace it')%(feed.title))
+                self.log(_('Feed %s is not encoded correctly, manually replace it')%(feed.title))
                 feed.description = feed.description.decode(self.encoding, 'replace')
             elif feed.description.find(self.DESC_SENSE) == -1 and self.encoding and feed.description:
-                self.log_debug(_('Feed %s is weired encoded, manually redo all')%(feed.title))
+                self.log(_('Feed %s is weired encoded, manually redo all')%(feed.title))
                 feed.description = feed.description.encode('cp1252', 'replace').decode(self.encoding, 'replace')
                 weired_encoding_detected = True
 
@@ -148,7 +148,7 @@ class ZAOBAO(BasicNewsRecipe):
                         article.text_summary = article.text_summary.encode('cp1252', 'replace').decode(self.encoding, 'replace')
 
                 if article.title == "Untitled article":
-                    self.log_debug(_('Removing empty article %s from %s')%(article.title, article.url))
+                    self.log(_('Removing empty article %s from %s')%(article.title, article.url))
                     # remove the article
                     feed.articles[a:a+1] = []
         return parsed_feeds

resources/default_tweaks.py

@@ -37,7 +37,6 @@ series_index_auto_increment = 'next'
 # Can be either True or False
 authors_completer_append_separator = False
 
-
 #: Author sort name algorithm
 # The algorithm used to copy author to author_sort
 # Possible values are:
@@ -71,6 +70,15 @@ author_name_suffixes = ('Jr', 'Sr', 'Inc', 'Ph.D', 'Phd',
 #   categories_use_field_for_author_name = 'author_sort'
 categories_use_field_for_author_name = 'author'
 
+#: Completion sort order: choose when to change from lexicographic to ASCII-like
+# Calibre normally uses locale-dependent lexicographic ordering when showing
+# completion values. This means that the sort order is correct for the user's
+# language. However, this can be slow. Performance is improved by switching to
+# ascii ordering. This tweak controls when that switch happens. Set it to zero
+# to always use ascii ordering. Set it to something larger than zero to switch
+# to ascii ordering for performance reasons.
+completion_change_to_ascii_sorting = 2500
+
 #: Control partitioning of Tag Browser
 # When partitioning the tags browser, the format of the subcategory label is
 # controlled by a template: categories_collapsed_name_template if sorting by
@@ -93,7 +101,6 @@ categories_collapsed_name_template = r'{first.sort:shorten(4,,0)} - {last.sort:s
 categories_collapsed_rating_template = r'{first.avg_rating:4.2f:ifempty(0)} - {last.avg_rating:4.2f:ifempty(0)}'
 categories_collapsed_popularity_template = r'{first.count:d} - {last.count:d}'
 
-
 #: Specify columns to sort the booklist by on startup
 # Provide a set of columns to be sorted on when calibre starts
 #  The argument is None if saved sort history is to be used
@@ -244,17 +251,14 @@ sony_collection_name_template='{value}{category:| (|)}'
 # Default: empty (no rules), so no collection attributes are named.
 sony_collection_sorting_rules = []
 
-
 #: Control how tags are applied when copying books to another library
 # Set this to True to ensure that tags in 'Tags to add when adding
 # a book' are added when copying books to another library
 add_new_book_tags_when_importing_books = False
 
-
 #: Set the maximum number of tags to show per book in the content server
 max_content_server_tags_shown=5
 
-
 #: Set custom metadata fields that the content server will or will not display.
 # content_server_will_display is a list of custom fields to be displayed.
 # content_server_wont_display is a list of custom fields not to be displayed.
@@ -296,7 +300,6 @@ generate_cover_foot_font = None
 # Example: doubleclick_on_library_view = 'do_nothing'
 doubleclick_on_library_view = 'open_viewer'
 
-
 #: Language to use when sorting.
 # Setting this tweak will force sorting to use the
 # collating order for the specified language. This might be useful if you run

resources/template-functions.json

@@ -5,7 +5,7 @@
     "uppercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.upper()\n", 
     "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    res = ''\n    for i in range(0, len(args)):\n        res += args[i]\n    return res\n", 
     "in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, sep, pat, fv, nfv):\n    l = [v.strip() for v in val.split(sep) if v.strip()]\n    if l:\n        for v in l:\n            if re.search(pat, v, flags=re.I):\n                return fv\n    return nfv\n", 
-    "multiply": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x * y)\n", 
+    "not": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return '1'\n        i += 1\n    return ''\n", 
     "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n    if val:\n        return val\n    else:\n        return value_if_empty\n", 
     "booksize": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.book_size is not None:\n        try:\n            return str(mi.book_size)\n        except:\n            pass\n    return ''\n", 
     "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n    if not val:\n        return ''\n    vals = [v.strip() for v in val.split(',')]\n    for v in vals:\n        if v.startswith(key+':'):\n            return v[len(key)+1:]\n    return ''\n", 
@@ -27,9 +27,10 @@
     "sublist": "def evaluate(self, formatter, kwargs, mi, locals, val, start_index, end_index, sep):\n    if not val:\n        return ''\n    si = int(start_index)\n    ei = int(end_index)\n    val = val.split(sep)\n    try:\n        if ei == 0:\n            return sep.join(val[si:])\n        else:\n            return sep.join(val[si:ei])\n    except:\n        return ''\n", 
     "test": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_set, value_not_set):\n    if val:\n        return value_if_set\n    else:\n        return value_not_set\n", 
     "eval": "def evaluate(self, formatter, kwargs, mi, locals, template):\n    from formatter import eval_formatter\n    template = template.replace('[[', '{').replace(']]', '}')\n    return eval_formatter.safe_format(template, locals, 'EVAL', None)\n", 
-    "not": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    while i < len(args):\n        if args[i]:\n            return '1'\n        i += 1\n    return ''\n", 
+    "multiply": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x * y)\n", 
-    "format_date": "def evaluate(self, formatter, kwargs, mi, locals, val, format_string):\n    if not val:\n        return ''\n    try:\n        dt = parse_date(val)\n        s = format_date(dt, format_string)\n    except:\n        s = 'BAD DATE'\n    return s\n", 
+    "format_date": "def evaluate(self, formatter, kwargs, mi, locals, val, format_string):\n    if not val or val == 'None':\n        return ''\n    try:\n        dt = parse_date(val)\n        s = format_date(dt, format_string)\n    except:\n        s = 'BAD DATE'\n    return s\n", 
     "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
+    "identifier_in_list": "def evaluate(self, formatter, kwargs, mi, locals, val, ident, fv, nfv):\n    l = [v.strip() for v in val.split(',') if v.strip()]\n    (id, _, regexp) = ident.partition(':')\n    if not id:\n        return nfv\n    id += ':'\n    if l:\n        for v in l:\n            if v.startswith(id):\n                if not regexp or re.search(regexp, v[len(id):], flags=re.I):\n                    return fv\n    return nfv\n", 
     "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n    return unicode(len(val.split(sep)))\n", 
     "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.lower()\n", 
     "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n    return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", 
@@ -38,5 +39,5 @@
     "ondevice": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.ondevice_col:\n        return _('Yes')\n    return ''\n", 
     "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
     "raw_field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return unicode(getattr(mi, name, None))\n", 
-    "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    if x < y:\n        return lt\n    if x == y:\n        return eq\n    return gt\n"
+    "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    x = float(x if x and x != 'None' else 0)\n    y = float(y if y and y != 'None' else 0)\n    if x < y:\n        return lt\n    if x == y:\n        return eq\n    return gt\n"
 }

setup/installer/windows/MemoryModule.c

@@ -0,0 +1,689 @@
+/*
+ * Memory DLL loading code
+ * Version 0.0.2 with additions from Thomas Heller
+ *
+ * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
+ * http://www.joachim-bauch.de
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is MemoryModule.c
+ *
+ * The Initial Developer of the Original Code is Joachim Bauch.
+ *
+ * Portions created by Joachim Bauch are Copyright (C) 2004-2005
+ * Joachim Bauch. All Rights Reserved.
+ *
+ * Portions Copyright (C) 2005 Thomas Heller.
+ *
+ */
+
+// disable warnings about pointer <-> DWORD conversions
+#pragma warning( disable : 4311 4312 )
+
+#include <Windows.h>
+#include <winnt.h>
+#if DEBUG_OUTPUT
+#include <stdio.h>
+#endif
+
+#ifndef IMAGE_SIZEOF_BASE_RELOCATION
+// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!?
+# define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION))
+#endif
+#include "MemoryModule.h"
+
+/*
+  XXX We need to protect at least walking the 'loaded' linked list with a lock!
+*/
+
+/******************************************************************/
+FINDPROC findproc;
+void *findproc_data = NULL;
+
+struct NAME_TABLE {
+	char *name;
+	DWORD ordinal;
+};
+
+typedef struct tagMEMORYMODULE {
+	PIMAGE_NT_HEADERS headers;
+	unsigned char *codeBase;
+	HMODULE *modules;
+	int numModules;
+	int initialized;
+
+	struct NAME_TABLE *name_table;
+
+	char *name;
+	int refcount;
+	struct tagMEMORYMODULE *next, *prev;
+} MEMORYMODULE, *PMEMORYMODULE;
+
+typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved);
+
+#define GET_HEADER_DICTIONARY(module, idx)	&(module)->headers->OptionalHeader.DataDirectory[idx]
+
+MEMORYMODULE *loaded; /* linked list of loaded memory modules */
+
+/* private - insert a loaded library in a linked list */
+static void _Register(char *name, MEMORYMODULE *module)
+{
+	module->next = loaded;
+	if (loaded)
+		loaded->prev = module;
+	module->prev = NULL;
+	loaded = module;
+}
+
+/* private - remove a loaded library from a linked list */
+static void _Unregister(MEMORYMODULE *module)
+{
+	free(module->name);
+	if (module->prev)
+		module->prev->next = module->next;
+	if (module->next)
+		module->next->prev = module->prev;
+	if (module == loaded)
+		loaded = module->next;
+}
+
+/* public - replacement for GetModuleHandle() */
+HMODULE MyGetModuleHandle(LPCTSTR lpModuleName)
+{
+	MEMORYMODULE *p = loaded;
+	while (p) {
+		// If already loaded, only increment the reference count
+		if (0 == stricmp(lpModuleName, p->name)) {
+			return (HMODULE)p;
+		}
+		p = p->next;
+	}
+	return GetModuleHandle(lpModuleName);
+}
+
+/* public - replacement for LoadLibrary, but searches FIRST for memory
+   libraries, then for normal libraries.  So, it will load libraries AS memory
+   module if they are found by findproc().
+*/
+HMODULE MyLoadLibrary(char *lpFileName)
+{
+	MEMORYMODULE *p = loaded;
+	HMODULE hMod;
+
+	while (p) {
+		// If already loaded, only increment the reference count
+		if (0 == stricmp(lpFileName, p->name)) {
+			p->refcount++;
+			return (HMODULE)p;
+		}
+		p = p->next;
+	}
+	if (findproc && findproc_data) {
+		void *pdata = findproc(lpFileName, findproc_data);
+		if (pdata) {
+			hMod = MemoryLoadLibrary(lpFileName, pdata);
+			free(p);
+			return hMod;
+		}
+	}
+	hMod = LoadLibrary(lpFileName);
+	return hMod;
+}
+
+/* public - replacement for GetProcAddress() */
+FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName)
+{
+	MEMORYMODULE *p = loaded;
+	while (p) {
+		if ((HMODULE)p == hModule)
+			return MemoryGetProcAddress(p, lpProcName);
+		p = p->next;
+	}
+	return GetProcAddress(hModule, lpProcName);
+}
+
+/* public - replacement for FreeLibrary() */
+BOOL MyFreeLibrary(HMODULE hModule)
+{
+	MEMORYMODULE *p = loaded;
+	while (p) {
+		if ((HMODULE)p == hModule) {
+			if (--p->refcount == 0) {
+				_Unregister(p);
+				MemoryFreeLibrary(p);
+			}
+			return TRUE;
+		}
+		p = p->next;
+	}
+	return FreeLibrary(hModule);
+}
+
+#if DEBUG_OUTPUT
+static void
+OutputLastError(const char *msg)
+{
+	LPVOID tmp;
+	char *tmpmsg;
+	FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+		NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL);
+	tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3);
+	sprintf(tmpmsg, "%s: %s", msg, tmp);
+	OutputDebugString(tmpmsg);
+	LocalFree(tmpmsg);
+	LocalFree(tmp);
+}
+#endif
+
+/*
+static int dprintf(char *fmt, ...)
+{
+	char Buffer[4096];
+	va_list marker;
+	int result;
+	
+	va_start(marker, fmt);
+	result = vsprintf(Buffer, fmt, marker);
+	OutputDebugString(Buffer);
+	return result;
+}
+*/
+
+static void
+CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module)
+{
+	int i, size;
+	unsigned char *codeBase = module->codeBase;
+	unsigned char *dest;
+	PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
+	for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++)
+	{
+		if (section->SizeOfRawData == 0)
+		{
+			// section doesn't contain data in the dll itself, but may define
+			// uninitialized data
+			size = old_headers->OptionalHeader.SectionAlignment;
+			if (size > 0)
+			{
+				dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
+					size,
+					MEM_COMMIT,
+					PAGE_READWRITE);
+
+				section->Misc.PhysicalAddress = (DWORD)dest;
+				memset(dest, 0, size);
+			}
+
+			// section is empty
+			continue;
+		}
+
+		// commit memory block and copy data from dll
+		dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
+							section->SizeOfRawData,
+							MEM_COMMIT,
+							PAGE_READWRITE);
+		memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData);
+		section->Misc.PhysicalAddress = (DWORD)dest;
+	}
+}
+
+// Protection flags for memory pages (Executable, Readable, Writeable)
+static int ProtectionFlags[2][2][2] = {
+	{
+		// not executable
+		{PAGE_NOACCESS, PAGE_WRITECOPY},
+		{PAGE_READONLY, PAGE_READWRITE},
+	}, {
+		// executable
+		{PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY},
+		{PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE},
+	},
+};
+
+static void
+FinalizeSections(PMEMORYMODULE module)
+{
+	int i;
+	PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
+	
+	// loop through all sections and change access flags
+	for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++)
+	{
+		DWORD protect, oldProtect, size;
+		int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0;
+		int readable =   (section->Characteristics & IMAGE_SCN_MEM_READ) != 0;
+		int writeable =  (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0;
+
+		if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE)
+		{
+			// section is not needed any more and can safely be freed
+			VirtualFree((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, MEM_DECOMMIT);
+			continue;
+		}
+
+		// determine protection flags based on characteristics
+		protect = ProtectionFlags[executable][readable][writeable];
+		if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED)
+			protect |= PAGE_NOCACHE;
+
+		// determine size of region
+		size = section->SizeOfRawData;
+		if (size == 0)
+		{
+			if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA)
+				size = module->headers->OptionalHeader.SizeOfInitializedData;
+			else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)
+				size = module->headers->OptionalHeader.SizeOfUninitializedData;
+		}
+
+		if (size > 0)
+		{
+			// change memory access flags
+			if (VirtualProtect((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, protect, &oldProtect) == 0)
+#if DEBUG_OUTPUT
+				OutputLastError("Error protecting memory page")
+#endif
+			;
+		}
+	}
+}
+
+static void
+PerformBaseRelocation(PMEMORYMODULE module, DWORD delta)
+{
+	DWORD i;
+	unsigned char *codeBase = module->codeBase;
+
+	PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC);
+	if (directory->Size > 0)
+	{
+		PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION)(codeBase + directory->VirtualAddress);
+		for (; relocation->VirtualAddress > 0; )
+		{
+			unsigned char *dest = (unsigned char *)(codeBase + relocation->VirtualAddress);
+			unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION);
+			for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++)
+			{
+				DWORD *patchAddrHL;
+				int type, offset;
+
+				// the upper 4 bits define the type of relocation
+				type = *relInfo >> 12;
+				// the lower 12 bits define the offset
+				offset = *relInfo & 0xfff;
+				
+				switch (type)
+				{
+				case IMAGE_REL_BASED_ABSOLUTE:
+					// skip relocation
+					break;
+
+				case IMAGE_REL_BASED_HIGHLOW:
+					// change complete 32 bit address
+					patchAddrHL = (DWORD *)(dest + offset);
+					*patchAddrHL += delta;
+					break;
+
+				default:
+					//printf("Unknown relocation: %d\n", type);
+					break;
+				}
+			}
+
+			// advance to next relocation block
+			relocation = (PIMAGE_BASE_RELOCATION)(((DWORD)relocation) + relocation->SizeOfBlock);
+		}
+	}
+}
+
+static int
+BuildImportTable(PMEMORYMODULE module)
+{
+	int result=1;
+	unsigned char *codeBase = module->codeBase;
+
+	PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT);
+	if (directory->Size > 0)
+	{
+		PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR)(codeBase + directory->VirtualAddress);
+		for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++)
+		{
+			DWORD *thunkRef, *funcRef;
+			HMODULE handle;
+
+			handle = MyLoadLibrary(codeBase + importDesc->Name);
+			if (handle == INVALID_HANDLE_VALUE)
+			{
+				//LastError should already be set
+#if DEBUG_OUTPUT
+				OutputLastError("Can't load library");
+#endif
+				result = 0;
+				break;
+			}
+
+			module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE)));
+			if (module->modules == NULL)
+			{
+				SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+				result = 0;
+				break;
+			}
+
+			module->modules[module->numModules++] = handle;
+			if (importDesc->OriginalFirstThunk)
+			{
+				thunkRef = (DWORD *)(codeBase + importDesc->OriginalFirstThunk);
+				funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+			} else {
+				// no hint table
+				thunkRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+				funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+			}
+			for (; *thunkRef; thunkRef++, funcRef++)
+			{
+				if IMAGE_SNAP_BY_ORDINAL(*thunkRef) {
+					*funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef));
+				} else {
+					PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME)(codeBase + *thunkRef);
+					*funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)&thunkData->Name);
+				}
+				if (*funcRef == 0)
+				{
+					SetLastError(ERROR_PROC_NOT_FOUND);
+					result = 0;
+					break;
+				}
+			}
+
+			if (!result)
+				break;
+		}
+	}
+
+	return result;
+}
+
+/*
+  MemoryLoadLibrary - load a library AS MEMORY MODULE, or return
+  existing MEMORY MODULE with increased refcount.
+
+  This allows to load a library AGAIN as memory module which is
+  already loaded as HMODULE!
+
+*/
+HMEMORYMODULE MemoryLoadLibrary(char *name, const void *data)
+{
+	PMEMORYMODULE result;
+	PIMAGE_DOS_HEADER dos_header;
+	PIMAGE_NT_HEADERS old_header;
+	unsigned char *code, *headers;
+	DWORD locationDelta;
+	DllEntryProc DllEntry;
+	BOOL successfull;
+	MEMORYMODULE *p = loaded;
+
+	while (p) {
+		// If already loaded, only increment the reference count
+		if (0 == stricmp(name, p->name)) {
+			p->refcount++;
+			return (HMODULE)p;
+		}
+		p = p->next;
+	}
+
+	/* Do NOT check for GetModuleHandle here! */
+
+	dos_header = (PIMAGE_DOS_HEADER)data;
+	if (dos_header->e_magic != IMAGE_DOS_SIGNATURE)
+	{
+		SetLastError(ERROR_BAD_FORMAT);
+#if DEBUG_OUTPUT
+		OutputDebugString("Not a valid executable file.\n");
+#endif
+		return NULL;
+	}
+
+	old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew];
+	if (old_header->Signature != IMAGE_NT_SIGNATURE)
+	{
+		SetLastError(ERROR_BAD_FORMAT);
+#if DEBUG_OUTPUT
+		OutputDebugString("No PE header found.\n");
+#endif
+		return NULL;
+	}
+
+	// reserve memory for image of library
+	code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase),
+		old_header->OptionalHeader.SizeOfImage,
+		MEM_RESERVE,
+		PAGE_READWRITE);
+
+    if (code == NULL)
+        // try to allocate memory at arbitrary position
+        code = (unsigned char *)VirtualAlloc(NULL,
+            old_header->OptionalHeader.SizeOfImage,
+            MEM_RESERVE,
+            PAGE_READWRITE);
+    
+	if (code == NULL)
+	{
+		SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+#if DEBUG_OUTPUT
+		OutputLastError("Can't reserve memory");
+#endif
+		return NULL;
+	}
+
+	result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE));
+	result->codeBase = code;
+	result->numModules = 0;
+	result->modules = NULL;
+	result->initialized = 0;
+	result->next = result->prev = NULL;
+	result->refcount = 1;
+	result->name = strdup(name);
+	result->name_table = NULL;
+
+	// XXX: is it correct to commit the complete memory region at once?
+    //      calling DllEntry raises an exception if we don't...
+	VirtualAlloc(code,
+		old_header->OptionalHeader.SizeOfImage,
+		MEM_COMMIT,
+		PAGE_READWRITE);
+
+	// commit memory for headers
+	headers = (unsigned char *)VirtualAlloc(code,
+		old_header->OptionalHeader.SizeOfHeaders,
+		MEM_COMMIT,
+		PAGE_READWRITE);
+	
+	// copy PE header to code
+	memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders);
+	result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew];
+
+	// update position
+	result->headers->OptionalHeader.ImageBase = (DWORD)code;
+
+	// copy sections from DLL file block to new memory location
+	CopySections(data, old_header, result);
+
+	// adjust base address of imported data
+	locationDelta = (DWORD)(code - old_header->OptionalHeader.ImageBase);
+	if (locationDelta != 0)
+		PerformBaseRelocation(result, locationDelta);
+
+	// load required dlls and adjust function table of imports
+	if (!BuildImportTable(result))
+		goto error;
+
+	// mark memory pages depending on section headers and release
+	// sections that are marked as "discardable"
+	FinalizeSections(result);
+
+	// get entry point of loaded library
+	if (result->headers->OptionalHeader.AddressOfEntryPoint != 0)
+	{
+		DllEntry = (DllEntryProc)(code + result->headers->OptionalHeader.AddressOfEntryPoint);
+		if (DllEntry == 0)
+		{
+			SetLastError(ERROR_BAD_FORMAT); /* XXX ? */
+#if DEBUG_OUTPUT
+			OutputDebugString("Library has no entry point.\n");
+#endif
+			goto error;
+		}
+
+		// notify library about attaching to process
+		successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0);
+		if (!successfull)
+		{
+#if DEBUG_OUTPUT
+			OutputDebugString("Can't attach library.\n");
+#endif
+			goto error;
+		}
+		result->initialized = 1;
+	}
+
+	_Register(name, result);
+
+	return (HMEMORYMODULE)result;
+
+error:
+	// cleanup
+	free(result->name);
+	MemoryFreeLibrary(result);
+	return NULL;
+}
+
+int _compare(const struct NAME_TABLE *p1, const struct NAME_TABLE *p2)
+{
+	return stricmp(p1->name, p2->name);
+}
+
+int _find(const char **name, const struct NAME_TABLE *p)
+{
+	return stricmp(*name, p->name);
+}
+
+struct NAME_TABLE *GetNameTable(PMEMORYMODULE module)
+{
+	unsigned char *codeBase;
+	PIMAGE_EXPORT_DIRECTORY exports;
+	PIMAGE_DATA_DIRECTORY directory;
+	DWORD i, *nameRef;
+	WORD *ordinal;
+	struct NAME_TABLE *p, *ptab;
+
+	if (module->name_table)
+		return module->name_table;
+
+	codeBase = module->codeBase;
+	directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXPORT);
+	exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
+
+	nameRef = (DWORD *)(codeBase + exports->AddressOfNames);
+	ordinal = (WORD *)(codeBase + exports->AddressOfNameOrdinals);
+
+	p = ((PMEMORYMODULE)module)->name_table = (struct NAME_TABLE *)malloc(sizeof(struct NAME_TABLE)
+									      * exports->NumberOfNames);
+	if (p == NULL)
+		return NULL;
+	ptab = p;
+	for (i=0; i<exports->NumberOfNames; ++i) {
+		p->name = (char *)(codeBase + *nameRef++);
+		p->ordinal = *ordinal++;
+		++p;
+	}
+	qsort(ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _compare);
+	return ptab;
+}
+
+FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name)
+{
+	unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase;
+	int idx=-1;
+	PIMAGE_EXPORT_DIRECTORY exports;
+	PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT);
+
+	if (directory->Size == 0)
+		// no export table found
+		return NULL;
+
+	exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
+	if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0)
+		// DLL doesn't export anything
+		return NULL;
+
+	if (HIWORD(name)) {
+		struct NAME_TABLE *ptab;
+		struct NAME_TABLE *found;
+		ptab = GetNameTable((PMEMORYMODULE)module);
+		if (ptab == NULL)
+			// some failure
+			return NULL;
+		found = bsearch(&name, ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _find);
+		if (found == NULL)
+			// exported symbol not found
+			return NULL;
+	
+		idx = found->ordinal;
+	}
+	else
+		idx = LOWORD(name) - exports->Base;
+
+	if ((DWORD)idx > exports->NumberOfFunctions)
+		// name <-> ordinal number don't match
+		return NULL;
+	
+	// AddressOfFunctions contains the RVAs to the "real" functions
+	return (FARPROC)(codeBase + *(DWORD *)(codeBase + exports->AddressOfFunctions + (idx*4)));
+}
+
+void MemoryFreeLibrary(HMEMORYMODULE mod)
+{
+	int i;
+	PMEMORYMODULE module = (PMEMORYMODULE)mod;
+
+	if (module != NULL)
+	{
+		if (module->initialized != 0)
+		{
+			// notify library about detaching from process
+			DllEntryProc DllEntry = (DllEntryProc)(module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint);
+			(*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0);
+			module->initialized = 0;
+		}
+
+		if (module->modules != NULL)
+		{
+			// free previously opened libraries
+			for (i=0; i<module->numModules; i++)
+				if (module->modules[i] != INVALID_HANDLE_VALUE)
+					MyFreeLibrary(module->modules[i]);
+
+			free(module->modules);
+		}
+
+		if (module->codeBase != NULL)
+			// release memory of library
+			VirtualFree(module->codeBase, 0, MEM_RELEASE);
+
+		if (module->name_table != NULL)
+			free(module->name_table);
+
+		HeapFree(GetProcessHeap(), 0, module);
+	}
+}

setup/installer/windows/MemoryModule.h

@@ -0,0 +1,58 @@
+/*
+ * Memory DLL loading code
+ * Version 0.0.2
+ *
+ * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
+ * http://www.joachim-bauch.de
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is MemoryModule.h
+ *
+ * The Initial Developer of the Original Code is Joachim Bauch.
+ *
+ * Portions created by Joachim Bauch are Copyright (C) 2004-2005
+ * Joachim Bauch. All Rights Reserved.
+ *
+ */
+
+#ifndef __MEMORY_MODULE_HEADER
+#define __MEMORY_MODULE_HEADER
+
+#include <Windows.h>
+
+typedef void *HMEMORYMODULE;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void *(*FINDPROC)();
+
+extern FINDPROC findproc;
+extern void *findproc_data;
+
+HMEMORYMODULE MemoryLoadLibrary(char *, const void *);
+
+FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *);
+
+void MemoryFreeLibrary(HMEMORYMODULE);
+
+BOOL MyFreeLibrary(HMODULE hModule);
+HMODULE MyLoadLibrary(char *lpFileName);
+FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName);
+HMODULE MyGetModuleHandle(LPCTSTR lpModuleName);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // __MEMORY_MODULE_HEADER

setup/installer/windows/freeze.py

@@ -16,7 +16,6 @@ from setup.installer.windows.wix import WixMixIn
 OPENSSL_DIR = r'Q:\openssl'
 QT_DIR = 'Q:\\Qt\\4.7.3'
 QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
-LIBUSB_DIR       = 'C:\\libusb'
 LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
 SW               = r'C:\cygwin\home\kovid\sw'
 IMAGEMAGICK      = os.path.join(SW, 'build', 'ImageMagick-6.6.6',
@@ -71,11 +70,13 @@ class Win32Freeze(Command, WixMixIn):
         self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc')
         self.py_ver = ''.join(map(str, sys.version_info[:2]))
         self.lib_dir = self.j(self.base, 'Lib')
-        self.pydlib = self.j(self.base, 'pydlib')
         self.pylib = self.j(self.base, 'pylib.zip')
+        self.dll_dir = self.j(self.base, 'DLLs')
+        self.plugins_dir = os.path.join(self.base, 'plugins')
 
         self.initbase()
         self.build_launchers()
+        self.add_plugins()
         self.freeze()
         self.embed_manifests()
         self.install_site_py()
@@ -87,18 +88,21 @@ class Win32Freeze(Command, WixMixIn):
             shutil.rmtree(self.base)
         os.makedirs(self.base)
 
+    def add_plugins(self):
+        self.info('Adding plugins...')
+        tgt = self.plugins_dir
+        if os.path.exists(tgt):
+            shutil.rmtree(tgt)
+        os.mkdir(tgt)
+        base = self.j(self.SRC, 'calibre', 'plugins')
+        for f in glob.glob(self.j(base, '*.pyd')):
+            # We dont want the manifests as the manifest in the exe will be
+            # used instead
+            shutil.copy2(f, tgt)
+
     def freeze(self):
         shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
 
-        self.info('Adding plugins...')
-        tgt = os.path.join(self.base, 'plugins')
-        if not os.path.exists(tgt):
-            os.mkdir(tgt)
-        base = self.j(self.SRC, 'calibre', 'plugins')
-        for pat in ('*.pyd', '*.manifest'):
-            for f in glob.glob(self.j(base, pat)):
-                shutil.copy2(f, tgt)
-
         self.info('Adding resources...')
         tgt = self.j(self.base, 'resources')
         if os.path.exists(tgt):
@@ -106,7 +110,6 @@ class Win32Freeze(Command, WixMixIn):
         shutil.copytree(self.j(self.src_root, 'resources'), tgt)
 
         self.info('Adding Qt and python...')
-        self.dll_dir = self.j(self.base, 'DLLs')
         shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir,
                 ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*'))
         for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')):
@@ -197,11 +200,6 @@ class Win32Freeze(Command, WixMixIn):
 
         print
         print 'Adding third party dependencies'
-        tdir = os.path.join(self.base, 'driver')
-        os.makedirs(tdir)
-        for pat in ('*.dll', '*.sys', '*.cat', '*.inf'):
-            for f in glob.glob(os.path.join(LIBUSB_DIR, pat)):
-                shutil.copyfile(f, os.path.join(tdir, os.path.basename(f)))
         print '\tAdding unrar'
         shutil.copyfile(LIBUNRAR,
                 os.path.join(self.dll_dir, os.path.basename(LIBUNRAR)))
@@ -318,8 +316,8 @@ class Win32Freeze(Command, WixMixIn):
         if not os.path.exists(self.obj_dir):
             os.makedirs(self.obj_dir)
         base = self.j(self.src_root, 'setup', 'installer', 'windows')
-        sources = [self.j(base, x) for x in ['util.c']]
+        sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']]
-        headers = [self.j(base, x) for x in ['util.h']]
+        headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']]
         objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources]
         cflags  = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split()
         cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver]
@@ -371,43 +369,49 @@ class Win32Freeze(Command, WixMixIn):
 
     def archive_lib_dir(self):
         self.info('Putting all python code into a zip file for performance')
-        if os.path.exists(self.pydlib):
-            shutil.rmtree(self.pydlib)
-        os.makedirs(self.pydlib)
         self.zf_timestamp = time.localtime(time.time())[:6]
         self.zf_names = set()
         with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf:
+            # Add the .pyds from python and calibre to the zip file
+            for x in (self.plugins_dir, self.dll_dir):
+                for pyd in os.listdir(x):
+                    if pyd.endswith('.pyd') and pyd != 'sqlite_custom.pyd':
+                        # sqlite_custom has to be a file for
+                        # sqlite_load_extension to work
+                        self.add_to_zipfile(zf, pyd, x)
+                        os.remove(self.j(x, pyd))
+
+            # Add everything in Lib except site-packages to the zip file
             for x in os.listdir(self.lib_dir):
                 if x == 'site-packages':
                     continue
                 self.add_to_zipfile(zf, x, self.lib_dir)
 
             sp = self.j(self.lib_dir, 'site-packages')
-            handled = set(['site.pyo'])
+            # Special handling for PIL and pywin32
-            for pth in ('PIL.pth', 'pywin32.pth'):
+            handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32'])
-                handled.add(pth)
+            self.add_to_zipfile(zf, 'PIL', sp)
-                shutil.copyfile(self.j(sp, pth), self.j(self.pydlib, pth))
+            base = self.j(sp, 'win32', 'lib')
-                for d in self.get_pth_dirs(self.j(sp, pth)):
+            for x in os.listdir(base):
-                    shutil.copytree(d, self.j(self.pydlib, self.b(d)), True)
+                if os.path.splitext(x)[1] not in ('.exe',):
-                    handled.add(self.b(d))
+                    self.add_to_zipfile(zf, x, base)
+            base = self.d(base)
+            for x in os.listdir(base):
+                if not os.path.isdir(self.j(base, x)):
+                    if os.path.splitext(x)[1] not in ('.exe',):
+                        self.add_to_zipfile(zf, x, base)
 
             handled.add('easy-install.pth')
             for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')):
                 handled.add(self.b(d))
-                zip_safe = self.is_zip_safe(d)
                 for x in os.listdir(d):
                     if x == 'EGG-INFO':
                         continue
-                    if zip_safe:
+                    self.add_to_zipfile(zf, x, d)
-                        self.add_to_zipfile(zf, x, d)
-                    else:
-                        absp = self.j(d, x)
-                        dest = self.j(self.pydlib, x)
-                        if os.path.isdir(absp):
-                            shutil.copytree(absp, dest, True)
-                        else:
-                            shutil.copy2(absp, dest)
 
+            # The rest of site-packages
+            # We dont want the site.py from site-packages
+            handled.add('site.pyo')
             for x in os.listdir(sp):
                 if x in handled or x.endswith('.egg-info'):
                     continue
@@ -415,33 +419,18 @@ class Win32Freeze(Command, WixMixIn):
                 if os.path.isdir(absp):
                     if not os.listdir(absp):
                         continue
-                    if self.is_zip_safe(absp):
+                    self.add_to_zipfile(zf, x, sp)
-                        self.add_to_zipfile(zf, x, sp)
-                    else:
-                        shutil.copytree(absp, self.j(self.pydlib, x), True)
                 else:
-                    if x.endswith('.pyd'):
+                    self.add_to_zipfile(zf, x, sp)
-                        shutil.copy2(absp, self.j(self.pydlib, x))
-                    else:
-                        self.add_to_zipfile(zf, x, sp)
 
         shutil.rmtree(self.lib_dir)
 
-    def is_zip_safe(self, path):
-        for f in walk(path):
-            ext = os.path.splitext(f)[1].lower()
-            if ext in ('.pyd', '.dll', '.exe'):
-                return False
-        return True
-
     def get_pth_dirs(self, pth):
         base = os.path.dirname(pth)
         for line in open(pth).readlines():
             line = line.strip()
             if not line or line.startswith('#') or line.startswith('import'):
                 continue
-            if line == 'win32\\lib':
-                continue
             candidate = self.j(base, line)
             if os.path.exists(candidate):
                 yield candidate
@@ -463,10 +452,10 @@ class Win32Freeze(Command, WixMixIn):
                     self.add_to_zipfile(zf, name + os.sep + x, base)
         else:
             ext = os.path.splitext(name)[1].lower()
-            if ext in ('.pyd', '.dll', '.exe'):
+            if ext in ('.dll',):
                 raise ValueError('Cannot add %r to zipfile'%abspath)
             zinfo.external_attr = 0600 << 16
-            if ext in ('.py', '.pyc', '.pyo'):
+            if ext in ('.py', '.pyc', '.pyo', '.pyd'):
                 with open(abspath, 'rb') as f:
                     zf.writestr(zinfo, f.read())
 

setup/installer/windows/notes.rst

@@ -88,7 +88,9 @@ Qt uses its own routine to locate and load "system libraries" including the open
 
 Now, run configure and make::
 
-    configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake
+-no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly
+
+    configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -no-plugin-manifests -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake
 
 SIP
 -----

setup/installer/windows/site.py

@@ -1,12 +1,72 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, os, linecache
+import sys
+import os
+import zipimport
+import _memimporter
+
+DEBUG_ZIPIMPORT = False
+
+class ZipExtensionImporter(zipimport.zipimporter):
+    '''
+    Taken, with thanks, from the py2exe source code
+    '''
+
+    def __init__(self, *args, **kwargs):
+        zipimport.zipimporter.__init__(self, *args, **kwargs)
+        # We know there are no dlls in the zip file, so dont set findproc
+        # (performance optimization)
+        #_memimporter.set_find_proc(self.locate_dll_image)
+
+    def find_module(self, fullname, path=None):
+        result = zipimport.zipimporter.find_module(self, fullname, path)
+        if result:
+            return result
+        fullname = fullname.replace(".", "\\")
+        if (fullname + '.pyd') in self._files:
+            return self
+        return None
+
+    def locate_dll_image(self, name):
+        # A callback function for_memimporter.import_module.  Tries to
+        # locate additional dlls.  Returns the image as Python string,
+        # or None if not found.
+        if name in self._files:
+            return self.get_data(name)
+        return None
+
+    def load_module(self, fullname):
+        if sys.modules.has_key(fullname):
+            mod = sys.modules[fullname]
+            if DEBUG_ZIPIMPORT:
+                sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive))
+            return mod
+        try:
+            return zipimport.zipimporter.load_module(self, fullname)
+        except zipimport.ZipImportError:
+            pass
+        initname = "init" + fullname.split(".")[-1] # name of initfunction
+        filename = fullname.replace(".", "\\")
+        path = filename + '.pyd'
+        if path in self._files:
+            if DEBUG_ZIPIMPORT:
+                sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive))
+            code = self.get_data(path)
+            mod = _memimporter.import_module(code, initname, fullname, path)
+            mod.__file__ = "%s\\%s" % (self.archive, path)
+            mod.__loader__ = self
+            if DEBUG_ZIPIMPORT:
+                sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__))
+            return mod
+        raise zipimport.ZipImportError, "can't find module %s" % fullname
+
+    def __repr__(self):
+        return "<%s object %r>" % (self.__class__.__name__, self.archive)
 
 
 def abs__file__():
@@ -42,42 +102,6 @@ def makepath(*paths):
     dir = os.path.abspath(os.path.join(*paths))
     return dir, os.path.normcase(dir)
 
-def addpackage(sitedir, name):
-    """Process a .pth file within the site-packages directory:
-       For each line in the file, either combine it with sitedir to a path,
-       or execute it if it starts with 'import '.
-    """
-    fullname = os.path.join(sitedir, name)
-    try:
-        f = open(fullname, "rU")
-    except IOError:
-        return
-    with f:
-        for line in f:
-            if line.startswith("#"):
-                continue
-            if line.startswith(("import ", "import\t")):
-                exec line
-                continue
-            line = line.rstrip()
-            dir, dircase = makepath(sitedir, line)
-            if os.path.exists(dir):
-                sys.path.append(dir)
-
-
-def addsitedir(sitedir):
-    """Add 'sitedir' argument to sys.path if missing and handle .pth files in
-    'sitedir'"""
-    sitedir, sitedircase = makepath(sitedir)
-    try:
-        names = os.listdir(sitedir)
-    except os.error:
-        return
-    dotpth = os.extsep + "pth"
-    names = [name for name in names if name.endswith(dotpth)]
-    for name in sorted(names):
-        addpackage(sitedir, name)
-
 def run_entry_point():
     bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function
     sys.argv[0] = bname+'.exe'
@@ -89,6 +113,10 @@ def main():
     sys.setdefaultencoding('utf-8')
     aliasmbcs()
 
+    sys.path_hooks.insert(0, ZipExtensionImporter)
+    sys.path_importer_cache.clear()
+
+    import linecache
     def fake_getline(filename, lineno, module_globals=None):
         return ''
     linecache.orig_getline = linecache.getline
@@ -96,10 +124,11 @@ def main():
 
     abs__file__()
 
-    addsitedir(os.path.join(sys.app_dir, 'pydlib'))
-
     add_calibre_vars()
 
+    # Needed for pywintypes to be able to load its DLL
+    sys.path.append(os.path.join(sys.app_dir, 'DLLs'))
+
     return run_entry_point()
 
 

setup/installer/windows/util.c

@@ -1,18 +1,130 @@
 /*
  * Copyright 2009 Kovid Goyal
+ * The memimporter code is taken from the py2exe project
  */
 
 #include "util.h"
+
 #include <delayimp.h>
 #include <io.h>
 #include <fcntl.h>
 
+
 static char GUI_APP = 0;
 static char python_dll[] = PYDLL;
 
 void set_gui_app(char yes) { GUI_APP = yes; }
 char is_gui_app() { return GUI_APP; }
 
+
+// memimporter {{{
+
+#include "MemoryModule.h"
+
+static char **DLL_Py_PackageContext = NULL;
+static PyObject **DLL_ImportError = NULL;
+static char module_doc[] =
+"Importer which can load extension modules from memory";
+
+
+static void *memdup(void *ptr, Py_ssize_t size)
+{
+	void *p = malloc(size);
+	if (p == NULL)
+		return NULL;
+	memcpy(p, ptr, size);
+	return p;
+}
+
+/*
+  Be sure to detect errors in FindLibrary - undetected errors lead to
+  very strange behaviour.
+*/
+static void* FindLibrary(char *name, PyObject *callback)
+{
+	PyObject *result;
+	char *p;
+	Py_ssize_t size;
+
+	if (callback == NULL)
+		return NULL;
+	result = PyObject_CallFunction(callback, "s", name);
+	if (result == NULL) {
+		PyErr_Clear();
+		return NULL;
+	}
+	if (-1 == PyString_AsStringAndSize(result, &p, &size)) {
+		PyErr_Clear();
+		Py_DECREF(result);
+		return NULL;
+	}
+	p = memdup(p, size);
+	Py_DECREF(result);
+	return p;
+}
+
+static PyObject *set_find_proc(PyObject *self, PyObject *args)
+{
+	PyObject *callback = NULL;
+	if (!PyArg_ParseTuple(args, "|O:set_find_proc", &callback))
+		return NULL;
+	Py_DECREF((PyObject *)findproc_data);
+	Py_INCREF(callback);
+	findproc_data = (void *)callback;
+    return Py_BuildValue("i", 1);
+}
+
+static PyObject *
+import_module(PyObject *self, PyObject *args)
+{
+	char *data;
+	int size;
+	char *initfuncname;
+	char *modname;
+	char *pathname;
+	HMEMORYMODULE hmem;
+	FARPROC do_init;
+
+	char *oldcontext;
+
+	/* code, initfuncname, fqmodulename, path */
+	if (!PyArg_ParseTuple(args, "s#sss:import_module",
+			      &data, &size,
+			      &initfuncname, &modname, &pathname))
+		return NULL;
+	hmem = MemoryLoadLibrary(pathname, data);
+	if (!hmem) {
+		PyErr_Format(*DLL_ImportError,
+			     "MemoryLoadLibrary() failed loading %s", pathname);
+		return NULL;
+	}
+	do_init = MemoryGetProcAddress(hmem, initfuncname);
+	if (!do_init) {
+		MemoryFreeLibrary(hmem);
+		PyErr_Format(*DLL_ImportError,
+			     "Could not find function %s in memory loaded pyd", initfuncname);
+		return NULL;
+	}
+
+    oldcontext = *DLL_Py_PackageContext;
+	*DLL_Py_PackageContext = modname;
+	do_init();
+	*DLL_Py_PackageContext = oldcontext;
+	if (PyErr_Occurred())
+		return NULL;
+	/* Retrieve from sys.modules */
+	return PyImport_ImportModule(modname);
+}
+
+static PyMethodDef methods[] = {
+	{ "import_module", import_module, METH_VARARGS,
+	  "import_module(code, initfunc, dllname[, finder]) -> module" },
+	{ "set_find_proc", set_find_proc, METH_VARARGS },
+	{ NULL, NULL },		/* Sentinel */
+};
+
+// }}}
+
 static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
     wchar_t *buf, *cbuf;
     buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)*
@@ -185,7 +297,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     char *dummy_argv[1] = {""};
 
     buf  = (char*)calloc(MAX_PATH, sizeof(char));
-    path = (char*)calloc(3*MAX_PATH, sizeof(char));
+    path = (char*)calloc(MAX_PATH, sizeof(char));
     if (!buf || !path) ExitProcess(_show_error(L"Out of memory", L"", 1));
 
     sz = GetModuleFileNameA(NULL, buf, MAX_PATH);
@@ -198,8 +310,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     buf[strlen(buf)-1] = '\0';
 
     _snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf);
-    _snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\pylib.zip;%s\\pydlib;%s\\DLLs",
+    _snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\pylib.zip", buf);
-            buf, buf, buf);
     free(buf);
 
 
@@ -227,7 +338,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1));
     //*flag = 1;
 
-
+    DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext");
+    if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1));
+    DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError");
+    if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1));
 
     Py_SetProgramName(program_name);
     Py_SetPythonHome(python_home);
@@ -263,6 +377,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
         PyList_SetItem(argv, i, v);
     }
     PySys_SetObject("argv", argv);
+
+    findproc = FindLibrary;
+	Py_InitModule3("_memimporter", methods, module_doc);
+
 }
 
 

setup/installer/windows/wix-template.xml

@@ -164,10 +164,6 @@
                 <CustomAction Id="LaunchApplication" BinaryKey="WixCA"
                     DllEntry="WixShellExec" Impersonate="yes"/>
 
-                <InstallUISequence>
-                    <FileCost Suppress="yes" />
-                </InstallUISequence>
-
 		</Product>
 </Wix>
 

setup/pygettext.py

@@ -187,7 +187,6 @@ msgstr ""
 
 '''%dict(appname=__appname__, version=version, year=time.strftime('%Y'))
 
-
 def usage(code, msg=''):
     print >> sys.stderr, __doc__ % globals()
     if msg:

setup/translations.py

@@ -85,7 +85,7 @@ class Translations(POT):
 
     def mo_file(self, po_file):
         locale = os.path.splitext(os.path.basename(po_file))[0]
-        return locale, os.path.join(self.DEST, locale, 'LC_MESSAGES', 'messages.mo')
+        return locale, os.path.join(self.DEST, locale, 'messages.mo')
 
 
     def run(self, opts):
@@ -94,9 +94,8 @@ class Translations(POT):
             base = os.path.dirname(dest)
             if not os.path.exists(base):
                 os.makedirs(base)
-            if self.newer(dest, f):
+            self.info('\tCompiling translations for', locale)
-                self.info('\tCompiling translations for', locale)
+            subprocess.check_call(['msgfmt', '-o', dest, f])
-                subprocess.check_call(['msgfmt', '-o', dest, f])
             if locale in ('en_GB', 'nds', 'te', 'yi'):
                 continue
             pycountry = self.j(sysconfig.get_python_lib(), 'pycountry',
@@ -123,6 +122,16 @@ class Translations(POT):
                 shutil.copy2(f, dest)
 
         self.write_stats()
+        self.freeze_locales()
+
+    def freeze_locales(self):
+        zf = self.DEST + '.zip'
+        from calibre import CurrentDir
+        from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED
+        with ZipFile(zf, 'w', ZIP_DEFLATED) as zf:
+            with CurrentDir(self.DEST):
+                zf.add_dir('.')
+        shutil.rmtree(self.DEST)
 
     @property
     def stats(self):

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 3)
+numeric_version = (0, 8, 4)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -608,9 +608,9 @@ from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
         SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER
 from calibre.devices.sne.driver import SNE
-from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, \
+from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL,
-        GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, \
+        GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR,
-        TREKSTOR, EEEREADER, NEXTBOOK
+        TREKSTOR, EEEREADER, NEXTBOOK, ADAM)
 from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 from calibre.devices.kobo.driver import KOBO
 from calibre.devices.bambook.driver import BAMBOOK
@@ -744,6 +744,7 @@ plugins += [
     TREKSTOR,
     EEEREADER,
     NEXTBOOK,
+    ADAM,
     ITUNES,
     BOEYE_BEX,
     BOEYE_BDX,
@@ -1231,7 +1232,7 @@ class StoreEpubBudStore(StoreBase):
     name = 'ePub Bud'
     description = 'Well, it\'s pretty much just "YouTube for Children\'s eBooks. A not-for-profit organization devoted to brining self published childrens books to the world.'
     actual_plugin = 'calibre.gui2.store.epubbud_plugin:EpubBudStore'
-    
+
     drm_free_only = True
     headquarters = 'US'
     formats = ['EPUB']

src/calibre/debug.py

@@ -53,6 +53,8 @@ Run an embedded python interpreter.
             default=False, action='store_true')
     parser.add_option('-m', '--inspect-mobi',
             help='Inspect the MOBI file at the specified path', default=None)
+    parser.add_option('--test-build', help='Test binary modules in build',
+            action='store_true', default=False)
 
     return parser
 
@@ -232,6 +234,9 @@ def main(args=sys.argv):
     elif opts.inspect_mobi is not None:
         from calibre.ebooks.mobi.debug import inspect_mobi
         inspect_mobi(opts.inspect_mobi)
+    elif opts.test_build:
+        from calibre.test_build import test
+        test()
     else:
         from calibre import ipython
         ipython()

src/calibre/devices/android/driver.py

@@ -53,6 +53,7 @@ class ANDROID(USBMS):
                        0x681c : [0x0222, 0x0224, 0x0400],
                        0x6640 : [0x0100],
                        0x685e : [0x0400],
+                       0x6860 : [0x0400],
                        0x6877 : [0x0400],
                      },
 

src/calibre/devices/apple/driver.py

@@ -2392,6 +2392,16 @@ class ITUNES(DriverBase):
                 self.iTunes.Windows[0].Minimized = True
             self.initial_status = 'launched'
 
+            try:
+                # Pre-emptive test to confirm functional iTunes automation interface
+                foo = self.iTunes.Version
+                foo
+            except:
+                self.iTunes = None
+                raise OpenFeedback('Unable to connect to iTunes.\n' +
+                             ' iTunes automation interface non-responsive, ' +
+                             'recommend reinstalling iTunes')
+
             # Read the current storage path for iTunes media from the XML file
             media_dir = ''
             string = None
@@ -2988,7 +2998,6 @@ class ITUNES(DriverBase):
             newmi = book
         return newmi
 
-
 class ITUNES_ASYNC(ITUNES):
     '''
     This subclass allows the user to interact directly with iTunes via a menu option

src/calibre/devices/misc.py

@@ -224,13 +224,16 @@ class TREKSTOR(USBMS):
     FORMATS     = ['epub', 'txt', 'pdf']
 
     VENDOR_ID   = [0x1e68]
-    PRODUCT_ID  = [0x0041, 0x0042]
+    PRODUCT_ID  = [0x0041, 0x0042,
+            0x003e # This is for the EBOOK_PLAYER_5M https://bugs.launchpad.net/bugs/792091
+            ]
     BCD         = [0x0002]
 
     EBOOK_DIR_MAIN = 'Ebooks'
 
     VENDOR_NAME = 'TREKSTOR'
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_PLAYER_7'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['EBOOK_PLAYER_7',
+            'EBOOK_PLAYER_5M']
 
 class EEEREADER(USBMS):
 
@@ -252,6 +255,28 @@ class EEEREADER(USBMS):
     VENDOR_NAME = 'LINUX'
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET'
 
+class ADAM(USBMS):
+
+    name = 'Notion Ink Adam device interface'
+    gui_name = 'Adam'
+
+    description    = _('Communicate with the Adam tablet')
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['epub', 'pdf', 'doc']
+
+    VENDOR_ID   = [0x0955]
+    PRODUCT_ID  = [0x7100]
+    BCD         = [0x9999]
+
+    EBOOK_DIR_MAIN = 'eBooks'
+
+    VENDOR_NAME = 'NI'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['ADAM']
+    SUPPORTS_SUB_DIRS = True
+
 class NEXTBOOK(USBMS):
 
     name           = 'Nextbook device interface'

src/calibre/devices/nook/driver.py

@@ -107,6 +107,9 @@ class NOOK_COLOR(NOOK):
 
         return filepath
 
+    def upload_cover(self, path, filename, metadata, filepath):
+        pass
+
 class NOOK_TSR(NOOK):
     gui_name       = _('Nook Simple')
     description    = _('Communicate with the Nook TSR eBook reader.')
@@ -117,4 +120,6 @@ class NOOK_TSR(NOOK):
     EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'My Files/Books'
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_DISK'
 
+    def upload_cover(self, path, filename, metadata, filepath):
+        pass
 

src/calibre/devices/usbms/books.py

@@ -204,7 +204,8 @@ class CollectionsBookList(BookList):
                 elif fm['datatype'] == 'text' and fm['is_multiple']:
                     val = orig_val
                 elif fm['datatype'] == 'composite' and fm['is_multiple']:
-                    val = [v.strip() for v in val.split(fm['is_multiple'])]
+                    val = [v.strip() for v in
+                           val.split(fm['is_multiple']['ui_to_list'])]
                 else:
                     val = [val]
 

src/calibre/ebooks/metadata/book/base.py

@@ -621,10 +621,7 @@ class Metadata(object):
             orig_res = res
             datatype = cmeta['datatype']
             if datatype == 'text' and cmeta['is_multiple']:
-                if cmeta['display'].get('is_names', False):
+                res = cmeta['is_multiple']['list_to_ui'].join(res)
-                    res = u' & '.join(res)
-                else:
-                    res = u', '.join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 if self.get_extra(key) is not None:
                     res = res + \
@@ -668,7 +665,7 @@ class Metadata(object):
             elif datatype == 'text' and fmeta['is_multiple']:
                 if isinstance(res, dict):
                     res = [k + ':' + v for k,v in res.items()]
-                res = u', '.join(sorted(res, key=sort_key))
+                res = fmeta['is_multiple']['list_to_ui'].join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 res = res + ' [%s]'%self.format_series_index()
             elif datatype == 'datetime':

src/calibre/ebooks/metadata/book/json_codec.py

@@ -5,8 +5,7 @@ Created on 4 Jun 2010
 '''
 
 from base64 import b64encode, b64decode
-import json
+import json, traceback
-import traceback
 
 from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
 from calibre.constants import filesystem_encoding, preferred_encoding
@@ -69,6 +68,40 @@ def object_to_unicode(obj, enc=preferred_encoding):
         return ans
     return obj
 
+def encode_is_multiple(fm):
+    if fm.get('is_multiple', None):
+        # migrate is_multiple back to a character
+        fm['is_multiple2'] = fm.get('is_multiple', {})
+        dt = fm.get('datatype', None)
+        if dt == 'composite':
+            fm['is_multiple'] = ','
+        else:
+            fm['is_multiple'] =  '|'
+    else:
+        fm['is_multiple'] = None
+        fm['is_multiple2'] = {}
+
+def decode_is_multiple(fm):
+    im = fm.get('is_multiple2',  None)
+    if im:
+        fm['is_multiple'] = im
+        del fm['is_multiple2']
+    else:
+        # Must migrate the is_multiple from char to dict
+        im = fm.get('is_multiple',  {})
+        if im:
+            dt = fm.get('datatype', None)
+            if dt == 'composite':
+                im = {'cache_to_list': ',', 'ui_to_list': ',',
+                      'list_to_ui': ', '}
+            elif fm.get('display', {}).get('is_names', False):
+                im = {'cache_to_list': '|', 'ui_to_list': '&',
+                      'list_to_ui': ', '}
+            else:
+                im = {'cache_to_list': '|', 'ui_to_list': ',',
+                      'list_to_ui': ', '}
+        fm['is_multiple'] = im
+
 class JsonCodec(object):
 
     def __init__(self):
@@ -93,9 +126,10 @@ class JsonCodec(object):
     def encode_metadata_attr(self, book, key):
         if key == 'user_metadata':
             meta = book.get_all_user_metadata(make_copy=True)
-            for k in meta:
+            for fm in meta.itervalues():
-                if meta[k]['datatype'] == 'datetime':
+                if fm['datatype'] == 'datetime':
-                    meta[k]['#value#'] = datetime_to_string(meta[k]['#value#'])
+                    fm['#value#'] = datetime_to_string(fm['#value#'])
+                encode_is_multiple(fm)
             return meta
         if key in self.field_metadata:
             datatype = self.field_metadata[key]['datatype']
@@ -135,9 +169,10 @@ class JsonCodec(object):
         if key == 'classifiers':
             key = 'identifiers'
         if key == 'user_metadata':
-            for k in value:
+            for fm in value.itervalues():
-                if value[k]['datatype'] == 'datetime':
+                if fm['datatype'] == 'datetime':
-                    value[k]['#value#'] = string_to_datetime(value[k]['#value#'])
+                    fm['#value#'] = string_to_datetime(fm['#value#'])
+                decode_is_multiple(fm)
             return value
         elif key in self.field_metadata:
             if self.field_metadata[key]['datatype'] == 'datetime':

src/calibre/ebooks/metadata/opf2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 lxml based OPF parser.
 '''
 
-import re, sys, unittest, functools, os, uuid, glob, cStringIO, json
+import re, sys, unittest, functools, os, uuid, glob, cStringIO, json, copy
 from urllib import unquote
 from urlparse import urlparse
 
@@ -453,10 +453,13 @@ class TitleSortField(MetadataField):
 
 def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)):
     from calibre.utils.config import to_json
-    from calibre.ebooks.metadata.book.json_codec import object_to_unicode
+    from calibre.ebooks.metadata.book.json_codec import (object_to_unicode,
+                                                         encode_is_multiple)
 
     for name, fm in all_user_metadata.items():
         try:
+            fm = copy.copy(fm)
+            encode_is_multiple(fm)
             fm = object_to_unicode(fm)
             fm = json.dumps(fm, default=to_json, ensure_ascii=False)
         except:
@@ -575,6 +578,7 @@ class OPF(object): # {{{
         self._user_metadata_ = {}
         temp = Metadata('x', ['x'])
         from calibre.utils.config import from_json
+        from calibre.ebooks.metadata.book.json_codec import decode_is_multiple
         elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,'
                 '"calibre:user_metadata:") and @content]')
         for elem in elems:
@@ -585,6 +589,7 @@ class OPF(object): # {{{
             fm = elem.get('content')
             try:
                 fm = json.loads(fm, object_hook=from_json)
+                decode_is_multiple(fm)
                 temp.set_user_metadata(name, fm)
             except:
                 prints('Failed to read user metadata:', name)

src/calibre/ebooks/metadata/sources/amazon.py

@@ -42,6 +42,7 @@ class Worker(Thread): # Get details {{{
         months = {
                 'de': {
             1 : ['jän'],
+            2 : ['februar'],
             3 : ['märz'],
             5 : ['mai'],
             6 : ['juni'],

src/calibre/ebooks/oeb/stylizer.py

@@ -13,7 +13,13 @@ from weakref import WeakKeyDictionary
 from xml.dom import SyntaxErr as CSSSyntaxError
 import cssutils
 from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration,
-    CSSValueList, CSSFontFaceRule, cssproperties)
+    CSSFontFaceRule, cssproperties)
+try:
+    from cssutils.css import CSSValueList
+    CSSValueList
+except ImportError:
+    # cssutils >= 0.9.8
+    from cssutils.css import PropertyValue as CSSValueList
 from cssutils import profile as cssprofiles
 from lxml import etree
 from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError

src/calibre/ebooks/rtf/rtfml.py

@@ -68,8 +68,13 @@ TODO:
 '''
 
 def txt2rtf(text):
+    # Escape { and } in the text.
+    text = text.replace('{', r'\'7b')
+    text = text.replace('}', r'\'7d')
+
     if not isinstance(text, unicode):
         return text
+
     buf = cStringIO.StringIO()
     for x in text:
         val = ord(x)

src/calibre/gui2/actions/delete.py

@@ -94,6 +94,9 @@ class DeleteAction(InterfaceAction):
         self.delete_menu.addAction(
                 _('Remove all formats from selected books, except...'),
                 self.delete_all_but_selected_formats)
+        self.delete_menu.addAction(
+                _('Remove all formats from selected books'),
+                self.delete_all_formats)
         self.delete_menu.addAction(
                 _('Remove covers from selected books'), self.delete_covers)
         self.delete_menu.addSeparator()
@@ -174,6 +177,28 @@ class DeleteAction(InterfaceAction):
         if ids:
             self.gui.tags_view.recount()
 
+    def delete_all_formats(self, *args):
+        ids = self._get_selected_ids()
+        if not ids:
+            return
+        if not confirm('<p>'+_('<b>All formats</b> for the selected books will '
+                               'be <b>deleted</b> from your library.<br>'
+                               'The book metadata will be kept. Are you sure?')
+                            +'</p>', 'delete_all_formats', self.gui):
+            return
+        db = self.gui.library_view.model().db
+        for id in ids:
+            fmts = db.formats(id, index_is_id=True, verify_formats=False)
+            if fmts:
+                for fmt in fmts.split(','):
+                    self.gui.library_view.model().db.remove_format(id, fmt,
+                            index_is_id=True, notify=False)
+        self.gui.library_view.model().refresh_ids(ids)
+        self.gui.library_view.model().current_changed(self.gui.library_view.currentIndex(),
+                self.gui.library_view.currentIndex())
+        if ids:
+            self.gui.tags_view.recount()
+
     def remove_matching_books_from_device(self, *args):
         if not self.gui.device_manager.is_device_connected:
             d = error_dialog(self.gui, _('Cannot delete books'),

src/calibre/gui2/complete.py

@@ -12,16 +12,23 @@ from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
 from calibre.gui2.widgets import EnComboBox, LineEditECM
+from calibre.utils.config_base import tweaks
 
 class CompleteModel(QAbstractListModel):
 
     def __init__(self, parent=None):
         QAbstractListModel.__init__(self, parent)
         self.items = []
+        self.sorting = QCompleter.UnsortedModel
 
     def set_items(self, items):
         items = [unicode(x.strip()) for x in items]
-        self.items = list(sorted(items, key=lambda x: sort_key(x)))
+        if len(items) < tweaks['completion_change_to_ascii_sorting']:
+            self.items = sorted(items, key=lambda x: sort_key(x))
+            self.sorting = QCompleter.UnsortedModel
+        else:
+            self.items = sorted(items, key=lambda x:x.lower())
+            self.sorting = QCompleter.CaseInsensitivelySortedModel
         self.lowered_items = [lower(x) for x in self.items]
         self.reset()
 
@@ -62,7 +69,7 @@ class MultiCompleteLineEdit(QLineEdit, LineEditECM):
         c.setWidget(self)
         c.setCompletionMode(QCompleter.PopupCompletion)
         c.setCaseSensitivity(Qt.CaseInsensitive)
-        c.setModelSorting(QCompleter.UnsortedModel)
+        c.setModelSorting(self._model.sorting)
         c.setCompletionRole(Qt.DisplayRole)
         p = c.popup()
         p.setMouseTracking(True)
@@ -146,6 +153,7 @@ class MultiCompleteLineEdit(QLineEdit, LineEditECM):
             return self._model.items
         def fset(self, items):
             self._model.set_items(items)
+            self._completer.setModelSorting(self._model.sorting)
         return property(fget=fget, fset=fset)
 
 class MultiCompleteComboBox(EnComboBox):

src/calibre/gui2/custom_column_widgets.py

@@ -226,16 +226,14 @@ class Comments(Base):
 class Text(Base):
 
     def setup_ui(self, parent):
-        if self.col_metadata['display'].get('is_names', False):
+        self.sep = self.col_metadata['multiple_seps']
-            self.sep = u' & '
-        else:
-            self.sep = u', '
         values = self.all_values = list(self.db.all_custom(num=self.col_id))
         values.sort(key=sort_key)
+
         if self.col_metadata['is_multiple']:
             w = MultiCompleteLineEdit(parent)
-            w.set_separator(self.sep.strip())
+            w.set_separator(self.sep['ui_to_list'])
-            if self.sep == u' & ':
+            if self.sep['ui_to_list'] == '&':
                 w.set_space_before_sep(True)
                 w.set_add_separator(tweaks['authors_completer_append_separator'])
             w.update_items_cache(values)
@@ -269,12 +267,12 @@ class Text(Base):
         if self.col_metadata['is_multiple']:
             if not val:
                 val = []
-            self.widgets[1].setText(self.sep.join(val))
+            self.widgets[1].setText(self.sep['list_to_ui'].join(val))
 
     def getter(self):
         if self.col_metadata['is_multiple']:
             val = unicode(self.widgets[1].text()).strip()
-            ans = [x.strip() for x in val.split(self.sep.strip()) if x.strip()]
+            ans = [x.strip() for x in val.split(self.sep['ui_to_list']) if x.strip()]
             if not ans:
                 ans = None
             return ans
@@ -899,9 +897,10 @@ class BulkText(BulkBase):
         if not self.a_c_checkbox.isChecked():
             return
         if self.col_metadata['is_multiple']:
+            ism = self.col_metadata['multiple_seps']
             if self.col_metadata['display'].get('is_names', False):
                 val = self.gui_val
-                add = [v.strip() for v in val.split('&') if v.strip()]
+                add = [v.strip() for v in val.split(ism['ui_to_list']) if v.strip()]
                 self.db.set_custom_bulk(book_ids, add, num=self.col_id)
             else:
                 remove_all, adding, rtext = self.gui_val
@@ -911,10 +910,10 @@ class BulkText(BulkBase):
                 else:
                     txt = rtext
                     if txt:
-                        remove = set([v.strip() for v in txt.split(',')])
+                        remove = set([v.strip() for v in txt.split(ism['ui_to_list'])])
                 txt = adding
                 if txt:
-                    add = set([v.strip() for v in txt.split(',')])
+                    add = set([v.strip() for v in txt.split(ism['ui_to_list'])])
                 else:
                     add = set()
                 self.db.set_custom_bulk_multiple(book_ids, add=add,

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -520,7 +520,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             elif not fm['is_multiple']:
                 val = [val]
             elif fm['datatype'] == 'composite':
-                val = [v.strip() for v in val.split(fm['is_multiple'])]
+                val = [v.strip() for v in val.split(fm['is_multiple']['ui_to_list'])]
             elif field == 'authors':
                 val = [v.replace('|', ',') for v in val]
         else:
@@ -655,19 +655,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
 
         if self.destination_field_fm['is_multiple']:
             if self.comma_separated.isChecked():
-                if dest == 'authors' or \
+                splitter = self.destination_field_fm['is_multiple']['ui_to_list']
-                        (self.destination_field_fm['is_custom'] and
-                         self.destination_field_fm['datatype'] == 'text' and
-                         self.destination_field_fm['display'].get('is_names', False)):
-                    splitter = ' & '
-                else:
-                    splitter = ','
-
                 res = []
                 for v in val:
-                    for x in v.split(splitter):
+                    res.extend([x.strip() for x in v.split(splitter) if x.strip()])
-                        if x.strip():
-                            res.append(x.strip())
                 val = res
             else:
                 val = [v.replace(',', '') for v in val]

src/calibre/gui2/dialogs/template_dialog.py

@@ -254,6 +254,15 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
         self.textbox_changed()
         self.rule = (None, '')
 
+        tt = _('Template language tutorial')
+        self.template_tutorial.setText(
+                '<a href="http://manual.calibre-ebook.com/template_lang.html">'
+                '%s</a>'%tt)
+        tt = _('Template function reference')
+        self.template_func_reference.setText(
+                '<a href="http://manual.calibre-ebook.com/template_ref.html">'
+                '%s</a>'%tt)
+
     def textbox_changed(self):
         cur_text = unicode(self.textbox.toPlainText())
         if self.last_text != cur_text:
@@ -299,4 +308,4 @@ class TemplateDialog(QDialog, Ui_TemplateDialog):
                 return
 
         self.rule = (unicode(self.colored_field.currentText()), txt)
-        QDialog.accept(self)
+        QDialog.accept(self)

src/calibre/gui2/dialogs/template_dialog.ui

@@ -125,6 +125,20 @@
      <item row="9" column="1">
       <widget class="QPlainTextEdit" name="source_code"/>
      </item>
+     <item row="10" column="1">
+      <widget class="QLabel" name="template_tutorial">
+       <property name="openExternalLinks">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="11" column="1">
+      <widget class="QLabel" name="template_func_reference">
+       <property name="openExternalLinks">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
     </layout>
    </item>
   </layout>

src/calibre/gui2/dialogs/tweak_epub.py

@@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, shutil
-from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED
+from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED
 
 from PyQt4.Qt import QDialog
 

src/calibre/gui2/library/delegates.py

@@ -288,6 +288,8 @@ class CcNumberDelegate(QStyledItemDelegate): # {{{
     def setEditorData(self, editor, index):
         m = index.model()
         val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']]
+        if val is None:
+            val = 0
         editor.setValue(val)
 
 # }}}

src/calibre/gui2/library/models.py

@@ -608,10 +608,11 @@ class BooksModel(QAbstractTableModel): # {{{
 
         def text_type(r, mult=None, idx=-1):
             text = self.db.data[r][idx]
-            if text and mult is not None:
+            if text and mult:
-                if mult:
+                jv = mult['list_to_ui']
-                    return QVariant(u' & '.join(text.split('|')))
+                sv = mult['cache_to_list']
-                return QVariant(u', '.join(sorted(text.split('|'),key=sort_key)))
+                return QVariant(jv.join(
+                    sorted([t.strip() for t in text.split(sv)], key=sort_key)))
             return QVariant(text)
 
         def decorated_text_type(r, idx=-1):
@@ -665,8 +666,6 @@ class BooksModel(QAbstractTableModel): # {{{
             datatype = self.custom_columns[col]['datatype']
             if datatype in ('text', 'comments', 'composite', 'enumeration'):
                 mult=self.custom_columns[col]['is_multiple']
-                if mult is not None:
-                    mult = self.custom_columns[col]['display'].get('is_names', False)
                 self.dc[col] = functools.partial(text_type, idx=idx, mult=mult)
                 if datatype in ['text', 'composite', 'enumeration'] and not mult:
                     if self.custom_columns[col]['display'].get('use_decorations', False):
@@ -722,9 +721,9 @@ class BooksModel(QAbstractTableModel): # {{{
                 if id_ in self.color_cache:
                     if key in self.color_cache[id_]:
                         return self.color_cache[id_][key]
-                if mi is None:
-                    mi = self.db.get_metadata(id_, index_is_id=True)
                 try:
+                    if mi is None:
+                        mi = self.db.get_metadata(id_, index_is_id=True)
                     color = composite_formatter.safe_format(fmt, mi, '', mi)
                     if color in self.colors:
                         color = QColor(color)

src/calibre/gui2/preferences/coloring.py

@@ -12,6 +12,7 @@ from PyQt4.Qt import (QWidget, QDialog, QLabel, QGridLayout, QComboBox, QSize,
         QScrollArea, QPushButton, QVBoxLayout, QDialogButtonBox, QToolButton,
         QListView, QAbstractListModel, pyqtSignal, QSizePolicy, QSpacerItem)
 
+from calibre import prepare_string_for_xml
 from calibre.utils.icu import sort_key
 from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.template_dialog import TemplateDialog
@@ -158,21 +159,22 @@ class ConditionEditor(QWidget): # {{{
         self.action_box.clear()
         self.action_box.addItem('', '')
         col = self.current_col
-        m = self.fm[col]
+        if col:
-        dt = m['datatype']
+            m = self.fm[col]
-        if dt in self.action_map:
+            dt = m['datatype']
-            actions = self.action_map[dt]
+            if dt in self.action_map:
-        else:
+                actions = self.action_map[dt]
-            if col == 'ondevice':
-                k = 'ondevice'
-            elif col == 'identifiers':
-                k = 'identifiers'
             else:
-                k = 'multiple' if m['is_multiple'] else 'single'
+                if col == 'ondevice':
-            actions = self.action_map[k]
+                    k = 'ondevice'
+                elif col == 'identifiers':
+                    k = 'identifiers'
+                else:
+                    k = 'multiple' if m['is_multiple'] else 'single'
+                actions = self.action_map[k]
 
-        for text, key in actions:
+            for text, key in actions:
-            self.action_box.addItem(text, key)
+                self.action_box.addItem(text, key)
         self.action_box.setCurrentIndex(0)
         self.action_box.blockSignals(False)
         self.init_value_box()
@@ -183,11 +185,15 @@ class ConditionEditor(QWidget): # {{{
         self.value_box.setInputMask('')
         self.value_box.setValidator(None)
         col = self.current_col
+        if not col:
+            return
         m = self.fm[col]
         dt = m['datatype']
         action = self.current_action
-        if not col or not action:
+        if not action:
             return
+        m = self.fm[col]
+        dt = m['datatype']
         tt = ''
         if col == 'identifiers':
             tt = _('Enter either an identifier type or an '
@@ -205,7 +211,7 @@ class ConditionEditor(QWidget): # {{{
                 tt = _('Enter a regular expression')
             elif m.get('is_multiple', False):
                 tt += '\n' + _('You can match multiple values by separating'
-                        ' them with %s')%m['is_multiple']
+                        ' them with %s')%m['is_multiple']['ui_to_list']
         self.value_box.setToolTip(tt)
         if action in ('is set', 'is not set', 'is true', 'is false',
                 'is undefined'):
@@ -430,7 +436,7 @@ class RulesModel(QAbstractListModel): # {{{
             return _('''
             <p>Advanced Rule for column <b>%s</b>:
             <pre>%s</pre>
-            ''')%(col, rule)
+            ''')%(col, prepare_string_for_xml(rule))
         conditions = [self.condition_to_html(c) for c in rule.conditions]
         return _('''\
             <p>Set the color of <b>%s</b> to <b>%s</b> if the following
@@ -439,9 +445,10 @@ class RulesModel(QAbstractListModel): # {{{
             ''') % (col, rule.color, ''.join(conditions))
 
     def condition_to_html(self, condition):
+        c, a, v = condition
         return (
             _('<li>If the <b>%s</b> column <b>%s</b> value: <b>%s</b>') %
-                tuple(condition))
+                (c, a, prepare_string_for_xml(v)))
 
 # }}}
 

src/calibre/gui2/preferences/create_custom_column.py

@@ -13,6 +13,9 @@ from calibre.gui2 import error_dialog
 
 class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
 
+    # Note: in this class, we are treating is_multiple as the boolean that
+    # custom_columns expects to find in its structure. It does not use the dict
+
     column_types = {
                     0:{'datatype':'text',
                         'text':_('Text, column shown in the tag browser'),

src/calibre/gui2/preferences/toolbar.ui

@@ -16,13 +16,28 @@
   <layout class="QGridLayout" name="gridLayout">
    <item row="0" column="0">
     <widget class="QLabel" name="label">
+     <property name="font">
+      <font>
+       <weight>75</weight>
+       <bold>true</bold>
+      </font>
+     </property>
      <property name="text">
-      <string>Customize the actions in:</string>
+      <string>Choose the &amp;toolbar to customize:</string>
+     </property>
+     <property name="buddy">
+      <cstring>what</cstring>
      </property>
     </widget>
    </item>
    <item row="0" column="1" colspan="3">
     <widget class="QComboBox" name="what">
+     <property name="font">
+      <font>
+       <weight>75</weight>
+       <bold>true</bold>
+      </font>
+     </property>
      <property name="sizeAdjustPolicy">
       <enum>QComboBox::AdjustToMinimumContentsLengthWithIcon</enum>
      </property>

src/calibre/gui2/shortcuts.py

@@ -44,9 +44,9 @@ class Customize(QFrame, Ui_Frame):
             clear.clicked.connect(partial(self.clear_clicked, which=x))
 
     def clear_clicked(self, which=0):
-         button = getattr(self, 'button%d'%which)
+        button = getattr(self, 'button%d'%which)
-         button.setText(_('None'))
+        button.setText(_('None'))
-         setattr(self, 'shortcut%d'%which, None)
+        setattr(self, 'shortcut%d'%which, None)
 
     def custom_toggled(self, checked):
         for w in ('1', '2'):

src/calibre/gui2/store/amazon_de_plugin.py

@@ -6,21 +6,23 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import re, urllib
+from contextlib import closing
+
+from lxml import html
+
 from PyQt4.Qt import QUrl
 
+from calibre import browser
 from calibre.gui2 import open_url
-from calibre.gui2.store.amazon_plugin import AmazonKindleStore
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.search_result import SearchResult
 
-class AmazonDEKindleStore(AmazonKindleStore):
+class AmazonDEKindleStore(StorePlugin):
     '''
     For comments on the implementation, please see amazon_plugin.py
     '''
 
-    search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords='
-    details_url = 'http://amazon.de/dp/'
-    drm_search_text = u'Gleichzeitige Verwendung von Geräten'
-    drm_free_text = u'Keine Einschränkung'
-
     def open(self, parent=None, detail_item=None, external=False):
         aff_id = {'tag': 'charhale0a-21'}
         store_link = ('http://www.amazon.de/gp/redirect.html?ie=UTF8&site-redirect=de'
@@ -32,3 +34,94 @@ class AmazonDEKindleStore(AmazonKindleStore):
                           '&location=http://www.amazon.de/dp/%(asin)s&site-redirect=de'
                           '&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742') % aff_id
         open_url(QUrl(store_link))
+
+    def search(self, query, max_results=10, timeout=60):
+        search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords='
+        url =  search_url + urllib.quote_plus(query)
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+
+            # Amazon has two results pages.
+            is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])')
+            # Horizontal grid of books.
+            if is_shot:
+                data_xpath = '//div[contains(@class, "result")]'
+                format_xpath = './/div[@class="productTitle"]/text()'
+                cover_xpath = './/div[@class="productTitle"]//img/@src'
+            # Vertical list of books.
+            else:
+                data_xpath = '//div[@class="productData"]'
+                format_xpath = './/span[@class="format"]/text()'
+                cover_xpath = '../div[@class="productImage"]/a/img/@src'
+
+            for data in doc.xpath(data_xpath):
+                if counter <= 0:
+                    break
+
+                # Even though we are searching digital-text only Amazon will still
+                # put in results for non Kindle books (author pages). Se we need
+                # to explicitly check if the item is a Kindle book and ignore it
+                # if it isn't.
+                format = ''.join(data.xpath(format_xpath))
+                if 'kindle' not in format.lower():
+                    continue
+
+                # We must have an asin otherwise we can't easily reference the
+                # book later.
+                asin_href = None
+                asin_a = data.xpath('.//div[@class="productTitle"]/a[1]')
+                if asin_a:
+                    asin_href = asin_a[0].get('href', '')
+                    m = re.search(r'/dp/(?P<asin>.+?)(/|$)', asin_href)
+                    if m:
+                        asin = m.group('asin')
+                    else:
+                        continue
+                else:
+                    continue
+
+                cover_url = ''.join(data.xpath(cover_xpath))
+
+                title = ''.join(data.xpath('.//div[@class="productTitle"]/a/text()'))
+                price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()'))
+
+                if is_shot:
+                    author = format.split(' von ')[-1]
+                else:
+                    author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
+                    author = author.split(' von ')[-1]
+
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url.strip()
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = asin.strip()
+                s.formats = 'Kindle'
+
+                yield s
+
+    def get_details(self, search_result, timeout):
+        drm_search_text = u'Gleichzeitige Verwendung von Geräten'
+        drm_free_text = u'Keine Einschränkung'
+        url = 'http://amazon.de/dp/'
+
+        br = browser()
+        with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' +
+                           drm_search_text + '")])'):
+                if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' +
+                               drm_free_text + '") and contains(b, "' +
+                               drm_search_text + '")])'):
+                    search_result.drm = SearchResult.DRM_UNLOCKED
+                else:
+                    search_result.drm = SearchResult.DRM_UNKNOWN
+            else:
+                search_result.drm = SearchResult.DRM_LOCKED
+        return True

src/calibre/gui2/store/amazon_plugin.py

@@ -131,16 +131,22 @@ class AmazonKindleStore(StorePlugin):
             
             # Amazon has two results pages.
             is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])')
-            # Horizontal grid of books.
+            # Horizontal grid of books. Search "Paolo Bacigalupi"
             if is_shot:
                 data_xpath = '//div[contains(@class, "result")]'
-                format_xpath = './/div[@class="productTitle"]/text()'
+                format_xpath = './/div[@class="productTitle"]//text()'
+                asin_xpath = './/div[@class="productTitle"]//a'
                 cover_xpath = './/div[@class="productTitle"]//img/@src'
-            # Vertical list of books.
+                title_xpath = './/div[@class="productTitle"]/a//text()'
+                price_xpath = './/div[@class="newPrice"]/span/text()'
+            # Vertical list of books. Search "martin"
             else:
-                data_xpath = '//div[@class="productData"]'
+                data_xpath = '//div[contains(@class, "results")]//div[contains(@class, "result")]'
-                format_xpath = './/span[@class="format"]/text()'
+                format_xpath = './/span[@class="binding"]//text()'
-                cover_xpath = '../div[@class="productImage"]/a/img/@src'
+                asin_xpath = './/div[@class="image"]/a[1]'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/a[@class="title"]/text()'
+                price_xpath = './/span[@class="price"]/text()'
             
             for data in doc.xpath(data_xpath):
                 if counter <= 0:
@@ -157,7 +163,7 @@ class AmazonKindleStore(StorePlugin):
                 # We must have an asin otherwise we can't easily reference the
                 # book later.
                 asin_href = None
-                asin_a = data.xpath('.//div[@class="productTitle"]/a[1]')
+                asin_a = data.xpath(asin_xpath)
                 if asin_a:
                     asin_href = asin_a[0].get('href', '')
                     m = re.search(r'/dp/(?P<asin>.+?)(/|$)', asin_href)
@@ -170,14 +176,14 @@ class AmazonKindleStore(StorePlugin):
                 
                 cover_url = ''.join(data.xpath(cover_xpath))
 
-                title = ''.join(data.xpath('.//div[@class="productTitle"]/a/text()'))
+                title = ''.join(data.xpath(title_xpath))
-                price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()'))
+                price = ''.join(data.xpath(price_xpath))
                 
                 if is_shot:
                     author = format.split(' by ')[-1]
                 else:
-                    author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
+                    author = ''.join(data.xpath('.//span[@class="ptBrand"]/text()'))
-                    author = author.split(' by ')[-1]
+                    author = author.split('by ')[-1]
                 
                 counter -= 1
     

src/calibre/gui2/store/amazon_uk_plugin.py

@@ -15,17 +15,14 @@ from PyQt4.Qt import QUrl
 
 from calibre import browser
 from calibre.gui2 import open_url
-from calibre.gui2.store.amazon_plugin import AmazonKindleStore
+from calibre.gui2.store import StorePlugin
 from calibre.gui2.store.search_result import SearchResult
 
-class AmazonUKKindleStore(AmazonKindleStore):
+class AmazonUKKindleStore(StorePlugin):
     '''
     For comments on the implementation, please see amazon_plugin.py
     '''
 
-    search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords='
-    details_url = 'http://amazon.co.uk/dp/'
-
     def open(self, parent=None, detail_item=None, external=False):
         aff_id = {'tag': 'calcharles-21'}
         store_link = 'http://www.amazon.co.uk/gp/redirect.html?ie=UTF8&location=http://www.amazon.co.uk/Kindle-eBooks/b?ie=UTF8&node=341689031&ref_=sa_menu_kbo2&tag=%(tag)s&linkCode=ur2&camp=1634&creative=19450' % aff_id
@@ -36,7 +33,8 @@ class AmazonUKKindleStore(AmazonKindleStore):
         open_url(QUrl(store_link))
 
     def search(self, query, max_results=10, timeout=60):
-        url =  self.search_url + urllib.quote_plus(query)
+        search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords='
+        url =  search_url + urllib.quote_plus(query)
         br = browser()
 
         counter = max_results
@@ -75,15 +73,18 @@ class AmazonUKKindleStore(AmazonKindleStore):
                 s.title = title.strip()
                 s.price = price.strip()
                 s.detail_item = asin.strip()
-                s.formats = 'Kindle'
+                s.formats = ''
 
                 if is_shot:
                     # Amazon UK does not include the author on the grid layout
                     s.author = ''
                     self.get_details(s, timeout)
+                    if s.formats != 'Kindle':
+                        continue
                 else:
                     author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
                     s.author = author.split(' by ')[-1].strip()
+                    s.formats = 'Kindle'
 
                 yield s
 
@@ -92,18 +93,23 @@ class AmazonUKKindleStore(AmazonKindleStore):
         if search_result.drm:
             return
 
-        url = self.details_url
+        url = 'http://amazon.co.uk/dp/'
+        drm_search_text = u'Simultaneous Device Usage'
+        drm_free_text = u'Unlimited'
 
         br = browser()
         with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
             idata = html.fromstring(nf.read())
             if not search_result.author:
                 search_result.author = ''.join(idata.xpath('//div[@class="buying" and contains(., "Author")]/a/text()'))
+                is_kindle = idata.xpath('boolean(//div[@class="buying"]/h1/span/span[contains(text(), "Kindle Edition")])')
+                if is_kindle:
+                    search_result.formats = 'Kindle'
             if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' +
-                           self.drm_search_text + '")])'):
+                           drm_search_text + '")])'):
                 if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' +
-                               self.drm_free_text + '") and contains(b, "' +
+                               drm_free_text + '") and contains(b, "' +
-                               self.drm_search_text + '")])'):
+                               drm_search_text + '")])'):
                     search_result.drm = SearchResult.DRM_UNLOCKED
                 else:
                     search_result.drm = SearchResult.DRM_UNKNOWN

src/calibre/gui2/store/bn_plugin.py

@@ -8,7 +8,6 @@ __docformat__ = 'restructuredtext en'
 
 import random
 import re
-import urllib
 from contextlib import closing
 
 from lxml import html
@@ -47,26 +46,26 @@ class BNStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://productsearch.barnesandnoble.com/search/results.aspx?STORE=EBOOK&SZE=%s&WRD=' % max_results
+        query = query.replace(' ', '-')
-        url += urllib.quote_plus(query)
+        url = 'http://www.barnesandnoble.com/s/%s?store=ebook&sze=%s' % (query, max_results)
         
         br = browser()
         
         counter = max_results
         with closing(br.open(url, timeout=timeout)) as f:
             doc = html.fromstring(f.read())
-            for data in doc.xpath('//ul[contains(@class, "wgt-search-results-display")]/li[contains(@class, "search-result-item") and contains(@class, "nook-result-item")]'):
+            for data in doc.xpath('//ul[contains(@class, "result-set")]/li[contains(@class, "result")]'):
                 if counter <= 0:
                     break
                 
-                id = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/@href'))
+                id = ''.join(data.xpath('.//div[contains(@class, "image")]/a/@href'))
                 if not id:
                     continue
-                cover_url = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/img/@src'))
+                cover_url = ''.join(data.xpath('.//div[contains(@class, "image")]//img/@src'))
                 
-                title = ''.join(data.xpath('.//span[@class="product-title"]/a/text()'))
+                title = ''.join(data.xpath('.//p[@class="title"]//span[@class="name"]/text()'))
-                author = ', '.join(data.xpath('.//span[@class="contributers-line"]/a/text()'))
+                author = ', '.join(data.xpath('.//ul[@class="contributors"]//li[position()>1]//a/text()'))
-                price = ''.join(data.xpath('.//span[contains(@class, "onlinePriceValue2")]/text()'))
+                price = ''.join(data.xpath('.//table[@class="displayed-formats"]//a[@class="subtle"]/text()'))
                 
                 counter -= 1
                 
@@ -74,9 +73,9 @@ class BNStore(BasicStoreConfig, StorePlugin):
                 s.cover_url = cover_url
                 s.title = title.strip()
                 s.author = author.strip()
-                s.price = price
+                s.price = price.strip()
                 s.detail_item = id.strip()
                 s.drm = SearchResult.DRM_UNKNOWN
                 s.formats = 'Nook'
-                
+
                 yield s

src/calibre/gui2/store/declined.txt

@@ -2,7 +2,8 @@ This is a list of stores that objected, declined
 or asked not to be included in the store integration.
 
 * Borders (http://www.borders.com/)
-* WH Smith (http://www.whsmith.co.uk/)
+* Indigo (http://www.chapters.indigo.ca/)
-  Refused to permit signing up for the affiliate program
 * Libraria Rizzoli (http://libreriarizzoli.corriere.it/).
-  No reply with two attempts over 2 weeks
+  No reply with two attempts over 2 weeks
+* WH Smith (http://www.whsmith.co.uk/)
+  Refused to permit signing up for the affiliate program

src/calibre/gui2/store/search/models.py

@@ -33,7 +33,7 @@ class Matches(QAbstractItemModel):
 
     total_changed = pyqtSignal(int)
 
-    HEADERS = [_('Cover'), _('Title'), _('Price'), _('DRM'), _('Store'), _('')]
+    HEADERS = [_('Cover'), _('Title'), _('Price'), _('DRM'), _('Store'), '']
     HTML_COLS = (1, 4)
 
     def __init__(self, cover_thread_count=2, detail_thread_count=4):
@@ -58,7 +58,7 @@ class Matches(QAbstractItemModel):
         self.search_filter = SearchFilter()
         self.cover_pool = CoverThreadPool(cover_thread_count)
         self.details_pool = DetailsThreadPool(detail_thread_count)
-        
+
         self.filter_results_dispatcher = FunctionDispatcher(self.filter_results)
         self.got_result_details_dispatcher = FunctionDispatcher(self.got_result_details)
 

src/calibre/library/caches.py

@@ -509,7 +509,8 @@ class ResultCache(SearchQueryParser): # {{{
             valq_mkind, valq = self._matchkind(query)
 
         loc = self.field_metadata[location]['rec_index']
-        split_char = self.field_metadata[location]['is_multiple']
+        split_char = self.field_metadata[location]['is_multiple'].get(
+                'cache_to_list', ',')
         for id_ in candidates:
             item = self._data[id_]
             if item is None:
@@ -665,7 +666,8 @@ class ResultCache(SearchQueryParser): # {{{
                 if fm['is_multiple'] and \
                         len(query) > 1 and query.startswith('#') and \
                         query[1:1] in '=<>!':
-                    vf = lambda item, loc=fm['rec_index'], ms=fm['is_multiple']:\
+                    vf = lambda item, loc=fm['rec_index'], \
+                                ms=fm['is_multiple']['cache_to_list']:\
                             len(item[loc].split(ms)) if item[loc] is not None else 0
                     return self.get_numeric_matches(location, query[1:],
                                                     candidates, val_func=vf)
@@ -703,7 +705,8 @@ class ResultCache(SearchQueryParser): # {{{
                             ['composite', 'text', 'comments', 'series', 'enumeration']:
                         exclude_fields.append(db_col[x])
                     col_datatype[db_col[x]] = self.field_metadata[x]['datatype']
-                    is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple']
+                    is_multiple_cols[db_col[x]] = \
+                        self.field_metadata[x]['is_multiple'].get('cache_to_list', None)
 
             try:
                 rating_query = int(query) * 2
@@ -1045,13 +1048,14 @@ class SortKeyGenerator(object):
 
             elif dt in ('text', 'comments', 'composite', 'enumeration'):
                 if val:
-                    sep = fm['is_multiple']
+                    if fm['is_multiple']:
-                    if sep:
+                        jv = fm['is_multiple']['list_to_ui']
-                        if fm['display'].get('is_names', False):
+                        sv = fm['is_multiple']['cache_to_list']
-                            val = sep.join(
+                        if '&' in jv:
-                                [author_to_author_sort(v) for v in val.split(sep)])
+                            val = jv.join(
+                                [author_to_author_sort(v) for v in val.split(sv)])
                         else:
-                            val = sep.join(sorted(val.split(sep),
+                            val = jv.join(sorted(val.split(sv),
                                               key=self.string_sort_key))
                 val = self.string_sort_key(val)
 

src/calibre/library/coloring.py

@@ -79,16 +79,19 @@ class Rule(object): # {{{
         if dt == 'bool':
             return self.bool_condition(col, action, val)
 
-        if dt in ('int', 'float', 'rating'):
+        if dt in ('int', 'float'):
             return self.number_condition(col, action, val)
 
+        if dt == 'rating':
+            return self.rating_condition(col, action, val)
+
         if dt == 'datetime':
             return self.date_condition(col, action, val)
 
-        if dt in ('comments', 'series', 'text', 'enumeration'):
+        if dt in ('comments', 'series', 'text', 'enumeration', 'composite'):
             ism = m.get('is_multiple', False)
             if ism:
-                return self.multiple_condition(col, action, val, ism)
+                return self.multiple_condition(col, action, val, ism['ui_to_list'])
             return self.text_condition(col, action, val)
 
     def identifiers_condition(self, col, action, val):
@@ -114,9 +117,16 @@ class Rule(object): # {{{
                 'lt': ('1', '', ''),
                 'gt': ('', '', '1')
         }[action]
-        lt, eq, gt = '', '1', ''
         return "cmp(raw_field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt)
 
+    def rating_condition(self, col, action, val):
+        lt, eq, gt = {
+                'eq': ('', '1', ''),
+                'lt': ('1', '', ''),
+                'gt': ('', '', '1')
+        }[action]
+        return "cmp(field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt)
+
     def date_condition(self, col, action, val):
         lt, eq, gt = {
                 'eq': ('', '1', ''),
@@ -127,10 +137,12 @@ class Rule(object): # {{{
                 val, lt, eq, gt)
 
     def multiple_condition(self, col, action, val, sep):
+        if not sep or sep == '|':
+            sep = ','
         if action == 'is set':
-            return "test('%s', '1', '')"%col
+            return "test(field('%s'), '1', '')"%col
         if action == 'is not set':
-            return "test('%s', '', '1')"%col
+            return "test(field('%s'), '', '1')"%col
         if action == 'has':
             return "str_in_list(field('%s'), '%s', \"%s\", '1', '')"%(col, sep, val)
         if action == 'does not have':
@@ -142,9 +154,9 @@ class Rule(object): # {{{
 
     def text_condition(self, col, action, val):
         if action == 'is set':
-            return "test('%s', '1', '')"%col
+            return "test(field('%s'), '1', '')"%col
         if action == 'is not set':
-            return "test('%s', '', '1')"%col
+            return "test(field('%s'), '', '1')"%col
         if action == 'is':
             return "strcmp(field('%s'), \"%s\", '', '1', '')"%(col, val)
         if action == 'is not':
@@ -183,7 +195,7 @@ def conditionable_columns(fm):
         m = fm[key]
         dt = m['datatype']
         if m.get('name', False) and dt in ('bool', 'int', 'float', 'rating', 'series',
-                'comments', 'text', 'enumeration', 'datetime'):
+                'comments', 'text', 'enumeration', 'datetime', 'composite'):
             if key == 'sort':
                 yield 'title_sort'
             else:

src/calibre/library/custom_columns.py

@@ -78,6 +78,18 @@ class CustomColumns(object):
                     }
             if data['display'] is None:
                 data['display'] = {}
+            # set up the is_multiple separator dict
+            if data['is_multiple']:
+                if data['display'].get('is_names', False):
+                    seps = {'cache_to_list': '|', 'ui_to_list': '&', 'list_to_ui': ' & '}
+                elif data['datatype'] == 'composite':
+                    seps = {'cache_to_list': ',', 'ui_to_list': ',', 'list_to_ui': ', '}
+                else:
+                    seps = {'cache_to_list': '|', 'ui_to_list': ',', 'list_to_ui': ', '}
+            else:
+                seps = {}
+            data['multiple_seps'] = seps
+
             table, lt = self.custom_table_names(data['num'])
             if table not in custom_tables or (data['normalized'] and lt not in
                     custom_tables):
@@ -119,7 +131,7 @@ class CustomColumns(object):
                 if x is None:
                     return []
                 if isinstance(x, (str, unicode, bytes)):
-                    x = x.split('&' if d['display'].get('is_names', False) else',')
+                    x = x.split(d['multiple_seps']['ui_to_list'])
                 x = [y.strip() for y in x if y.strip()]
                 x = [y.decode(preferred_encoding, 'replace') if not isinstance(y,
                     unicode) else y for y in x]
@@ -181,10 +193,7 @@ class CustomColumns(object):
                 is_category = True
             else:
                 is_category = False
-            if v['is_multiple']:
+            is_m = v['multiple_seps']
-                is_m = ',' if v['datatype'] == 'composite' else '|'
-            else:
-                is_m = None
             tn = 'custom_column_{0}'.format(v['num'])
             self.field_metadata.add_custom_field(label=v['label'],
                     table=tn, column='value', datatype=v['datatype'],
@@ -200,7 +209,7 @@ class CustomColumns(object):
         row = self.data._data[idx] if index_is_id else self.data[idx]
         ans = row[self.FIELD_MAP[data['num']]]
         if data['is_multiple'] and data['datatype'] == 'text':
-            ans = ans.split('|') if ans else []
+            ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else []
             if data['display'].get('sort_alpha', False):
                 ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
         return ans
@@ -566,14 +575,21 @@ class CustomColumns(object):
     def custom_columns_in_meta(self):
         lines = {}
         for data in self.custom_column_label_map.values():
-            display = data['display']
             table, lt = self.custom_table_names(data['num'])
             if data['normalized']:
                 query = '%s.value'
                 if data['is_multiple']:
-                    query = 'group_concat(%s.value, "|")'
+#                    query = 'group_concat(%s.value, "{0}")'.format(
-                    if not display.get('sort_alpha', False):
+#                                        data['multiple_seps']['cache_to_list'])
-                        query = 'sort_concat(link.id, %s.value)'
+#                    if not display.get('sort_alpha', False):
+                    if data['multiple_seps']['cache_to_list'] == '|':
+                        query = 'sortconcat_bar(link.id, %s.value)'
+                    elif data['multiple_seps']['cache_to_list'] == '&':
+                        query = 'sortconcat_amper(link.id, %s.value)'
+                    else:
+                        prints('WARNING: unknown value in multiple_seps',
+                               data['multiple_seps']['cache_to_list'])
+                        query = 'sortconcat_bar(link.id, %s.value)'
                 line = '''(SELECT {query} FROM {lt} AS link INNER JOIN
                     {table} ON(link.value={table}.id) WHERE link.book=books.id)
                     custom_{num}

src/calibre/library/database2.py

@@ -595,7 +595,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 if f is None:
                     continue
                 with tempfile.SpooledTemporaryFile(max_size=100*(1024**2)) as stream:
-                    shutil.copyfileobj(f, stream)
+                    with f:
+                        shutil.copyfileobj(f, stream)
                     stream.seek(0)
                     self.add_format(id, format, stream, index_is_id=True,
                         path=tpath, notify=False)
@@ -1249,7 +1250,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             dex = field['rec_index']
             for book in self.data.iterall():
                 if field['is_multiple']:
-                    vals = [v.strip() for v in book[dex].split(field['is_multiple'])
+                    vals = [v.strip() for v in
+                            book[dex].split(field['is_multiple']['cache_to_list'])
                             if v.strip()]
                     if id_ in vals:
                         ans.add(book[0])
@@ -1377,7 +1379,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             tcategories[category] = {}
             # create a list of category/field_index for the books scan to use.
             # This saves iterating through field_metadata for each book
-            md.append((category, cat['rec_index'], cat['is_multiple'], False))
+            md.append((category, cat['rec_index'],
+                       cat['is_multiple'].get('cache_to_list', None), False))
 
         for category in tb_cats.iterkeys():
             cat = tb_cats[category]
@@ -1385,7 +1388,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                 cat['display'].get('make_category', False):
                 tids[category] = {}
                 tcategories[category] = {}
-                md.append((category, cat['rec_index'], cat['is_multiple'],
+                md.append((category, cat['rec_index'],
+                           cat['is_multiple'].get('cache_to_list', None),
                            cat['datatype'] == 'composite'))
         #print 'end phase "collection":', time.clock() - last, 'seconds'
         #last = time.clock()

src/calibre/library/field_metadata.py

@@ -50,9 +50,16 @@ class FieldMetadata(dict):
 
     datatype: the type of information in the field. Valid values are listed in
     VALID_DATA_TYPES below.
-    is_multiple: valid for the text datatype. If None, the field is to be
+    is_multiple: valid for the text datatype. If {}, the field is to be
-    treated as a single term. If not None, it contains a string, and the field
+    treated as a single term. If not None, it contains a dict of the form
-    is assumed to contain a list of terms separated by that string
+            {'cache_to_list': ',',
+             'ui_to_list': ',',
+             'list_to_ui': ', '}
+    where the cache_to_list contains the character used to split the value in
+    the meta2 table, ui_to_list contains the character used to create a list
+    from a value shown in the ui (each resulting value must be strip()ed and
+    empty values removed), and list_to_ui contains the string used in join()
+    to create a displayable string from the list.
 
     kind == field: is a db field.
     kind == category: standard tag category that isn't a field. see news.
@@ -97,7 +104,9 @@ class FieldMetadata(dict):
                            'link_column':'author',
                            'category_sort':'sort',
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': '&',
+                                          'list_to_ui': ' & '},
                            'kind':'field',
                            'name':_('Authors'),
                            'search_terms':['authors', 'author'],
@@ -109,7 +118,7 @@ class FieldMetadata(dict):
                            'link_column':'series',
                            'category_sort':'(title_sort(name))',
                            'datatype':'series',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Series'),
                            'search_terms':['series'],
@@ -119,7 +128,9 @@ class FieldMetadata(dict):
             ('formats',   {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': ',',
+                                          'list_to_ui': ', '},
                            'kind':'field',
                            'name':_('Formats'),
                            'search_terms':['formats', 'format'],
@@ -131,7 +142,7 @@ class FieldMetadata(dict):
                            'link_column':'publisher',
                            'category_sort':'name',
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Publishers'),
                            'search_terms':['publisher'],
@@ -143,7 +154,7 @@ class FieldMetadata(dict):
                            'link_column':'rating',
                            'category_sort':'rating',
                            'datatype':'rating',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Ratings'),
                            'search_terms':['rating'],
@@ -154,7 +165,7 @@ class FieldMetadata(dict):
                            'column':'name',
                            'category_sort':'name',
                            'datatype':None,
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'category',
                            'name':_('News'),
                            'search_terms':[],
@@ -166,7 +177,9 @@ class FieldMetadata(dict):
                            'link_column': 'tag',
                            'category_sort':'name',
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': ',',
+                                          'list_to_ui': ', '},
                            'kind':'field',
                            'name':_('Tags'),
                            'search_terms':['tags', 'tag'],
@@ -176,7 +189,9 @@ class FieldMetadata(dict):
             ('identifiers',   {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': ',',
+                                          'list_to_ui': ', '},
                            'kind':'field',
                            'name':_('Identifiers'),
                            'search_terms':['identifiers', 'identifier', 'isbn'],
@@ -186,7 +201,7 @@ class FieldMetadata(dict):
             ('author_sort',{'table':None,
                             'column':None,
                             'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Author Sort'),
                            'search_terms':['author_sort'],
@@ -196,7 +211,9 @@ class FieldMetadata(dict):
             ('au_map',    {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':',',
+                           'is_multiple':{'cache_to_list': ',',
+                                          'ui_to_list': None,
+                                          'list_to_ui': None},
                            'kind':'field',
                            'name':None,
                            'search_terms':[],
@@ -206,7 +223,7 @@ class FieldMetadata(dict):
             ('comments',  {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Comments'),
                            'search_terms':['comments', 'comment'],
@@ -216,7 +233,7 @@ class FieldMetadata(dict):
             ('cover',     {'table':None,
                            'column':None,
                            'datatype':'int',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':None,
                            'search_terms':['cover'],
@@ -226,7 +243,7 @@ class FieldMetadata(dict):
             ('id',        {'table':None,
                            'column':None,
                            'datatype':'int',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':None,
                            'search_terms':[],
@@ -236,7 +253,7 @@ class FieldMetadata(dict):
             ('last_modified', {'table':None,
                            'column':None,
                            'datatype':'datetime',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Modified'),
                            'search_terms':['last_modified'],
@@ -246,7 +263,7 @@ class FieldMetadata(dict):
             ('ondevice',  {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('On Device'),
                            'search_terms':['ondevice'],
@@ -256,7 +273,7 @@ class FieldMetadata(dict):
             ('path',      {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Path'),
                            'search_terms':[],
@@ -266,7 +283,7 @@ class FieldMetadata(dict):
             ('pubdate',   {'table':None,
                            'column':None,
                            'datatype':'datetime',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Published'),
                            'search_terms':['pubdate'],
@@ -276,7 +293,7 @@ class FieldMetadata(dict):
             ('marked',    {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name': None,
                            'search_terms':['marked'],
@@ -286,7 +303,7 @@ class FieldMetadata(dict):
             ('series_index',{'table':None,
                              'column':None,
                              'datatype':'float',
-                             'is_multiple':None,
+                             'is_multiple':{},
                              'kind':'field',
                              'name':None,
                              'search_terms':['series_index'],
@@ -296,7 +313,7 @@ class FieldMetadata(dict):
             ('sort',      {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Title Sort'),
                            'search_terms':['title_sort'],
@@ -306,7 +323,7 @@ class FieldMetadata(dict):
             ('size',      {'table':None,
                            'column':None,
                            'datatype':'float',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Size'),
                            'search_terms':['size'],
@@ -316,7 +333,7 @@ class FieldMetadata(dict):
             ('timestamp', {'table':None,
                            'column':None,
                            'datatype':'datetime',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Date'),
                            'search_terms':['date'],
@@ -326,7 +343,7 @@ class FieldMetadata(dict):
             ('title',     {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':_('Title'),
                            'search_terms':['title'],
@@ -336,7 +353,7 @@ class FieldMetadata(dict):
             ('uuid',      {'table':None,
                            'column':None,
                            'datatype':'text',
-                           'is_multiple':None,
+                           'is_multiple':{},
                            'kind':'field',
                            'name':None,
                            'search_terms':[],
@@ -508,7 +525,7 @@ class FieldMetadata(dict):
         if datatype == 'series':
             key += '_index'
             self._tb_cats[key] = {'table':None,        'column':None,
-                                 'datatype':'float',   'is_multiple':None,
+                                 'datatype':'float',   'is_multiple':{},
                                  'kind':'field',       'name':'',
                                  'search_terms':[key], 'label':label+'_index',
                                  'colnum':None,        'display':{},
@@ -560,7 +577,7 @@ class FieldMetadata(dict):
         if icu_lower(label) != label:
             st.append(icu_lower(label))
         self._tb_cats[label] = {'table':None,          'column':None,
-                                'datatype':None,       'is_multiple':None,
+                                'datatype':None,       'is_multiple':{},
                                 'kind':'user',         'name':name,
                                 'search_terms':st,     'is_custom':False,
                                 'is_category':True,    'is_csp': False}
@@ -570,7 +587,7 @@ class FieldMetadata(dict):
         if label in self._tb_cats:
             raise ValueError('Duplicate user field [%s]'%(label))
         self._tb_cats[label] = {'table':None,        'column':None,
-                                'datatype':None,     'is_multiple':None,
+                                'datatype':None,     'is_multiple':{},
                                 'kind':'search',     'name':name,
                                 'search_terms':[],   'is_custom':False,
                                 'is_category':True,  'is_csp': False}

src/calibre/library/restore.py

@@ -171,7 +171,7 @@ class Restore(Thread):
                 for x in fields:
                     if x in cfm:
                         if x == 'is_multiple':
-                            args.append(cfm[x] is not None)
+                            args.append(bool(cfm[x]))
                         else:
                             args.append(cfm[x])
                 if len(args) == len(fields):

src/calibre/library/server/mobile.py

@@ -231,7 +231,8 @@ class MobileServer(object):
             book['size'] = human_readable(book['size'])
 
             aus = record[FM['authors']] if record[FM['authors']] else __builtin__._('Unknown')
-            authors = '|'.join([i.replace('|', ',') for i in aus.split(',')])
+            aut_is = CFM['authors']['is_multiple']
+            authors = aut_is['list_to_ui'].join([i.replace('|', ',') for i in aus.split(',')])
             book['authors'] = authors
             book['series_index'] = fmt_sidx(float(record[FM['series_index']]))
             book['series'] = record[FM['series']]
@@ -254,8 +255,10 @@ class MobileServer(object):
                     continue
                 if datatype == 'text' and CFM[key]['is_multiple']:
                     book[key] = concat(name,
-                                       format_tag_string(val, ',',
+                                       format_tag_string(val,
-                                                         no_tag_count=True))
+                                           CFM[key]['is_multiple']['ui_to_list'],
+                                           no_tag_count=True,
+                                           joinval=CFM[key]['is_multiple']['list_to_ui']))
                 else:
                     book[key] = concat(name, val)
 

src/calibre/library/server/opds.py

@@ -180,9 +180,12 @@ def ACQUISITION_ENTRY(item, version, db, updated, CFM, CKEYS, prefix):
         if val:
             datatype = CFM[key]['datatype']
             if datatype == 'text' and CFM[key]['is_multiple']:
-                extra.append('%s: %s<br />'%(xml(name), xml(format_tag_string(val, ',',
+                extra.append('%s: %s<br />'%
-                                                           ignore_max=True,
+                             (xml(name),
-                                                           no_tag_count=True))))
+                              xml(format_tag_string(val,
+                                    CFM[key]['is_multiple']['ui_to_list'],
+                                    ignore_max=True, no_tag_count=True,
+                                    joinval=CFM[key]['is_multiple']['list_to_ui']))))
             elif datatype == 'comments':
                 extra.append('%s: %s<br />'%(xml(name), comments_to_html(unicode(val))))
             else:

src/calibre/library/server/utils.py

@@ -68,7 +68,7 @@ def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
     except:
         return _strftime(fmt, nowf().timetuple())
 
-def format_tag_string(tags, sep, ignore_max=False, no_tag_count=False):
+def format_tag_string(tags, sep, ignore_max=False, no_tag_count=False, joinval=', '):
     MAX = sys.maxint if ignore_max else tweaks['max_content_server_tags_shown']
     if tags:
         tlist = [t.strip() for t in tags.split(sep)]
@@ -78,10 +78,10 @@ def format_tag_string(tags, sep, ignore_max=False, no_tag_count=False):
     if len(tlist) > MAX:
         tlist = tlist[:MAX]+['...']
     if no_tag_count:
-        return ', '.join(tlist) if tlist else ''
+        return joinval.join(tlist) if tlist else ''
     else:
         return u'%s:&:%s'%(tweaks['max_content_server_tags_shown'],
-                     ', '.join(tlist)) if tlist else ''
+                     joinval.join(tlist)) if tlist else ''
 
 def quote(s):
     if isinstance(s, unicode):

src/calibre/library/server/xml.py

@@ -121,8 +121,12 @@ class XMLServer(object):
                 name = CFM[key]['name']
                 custcols.append(k)
                 if datatype == 'text' and CFM[key]['is_multiple']:
-                    kwargs[k] = concat('#T#'+name, format_tag_string(val,',',
+                    kwargs[k] = \
-                                                            ignore_max=True))
+                        concat('#T#'+name,
+                               format_tag_string(val,
+                                   CFM[key]['is_multiple']['ui_to_list'],
+                                   ignore_max=True,
+                                   joinval=CFM[key]['is_multiple']['list_to_ui']))
                 else:
                     kwargs[k] = concat(name, val)
             kwargs['custcols'] = ','.join(custcols)

src/calibre/library/sqlite.py

@@ -121,9 +121,12 @@ class SortedConcatenate(object):
             return None
         return self.sep.join(map(self.ans.get, sorted(self.ans.keys())))
 
-class SafeSortedConcatenate(SortedConcatenate):
+class SortedConcatenateBar(SortedConcatenate):
     sep = '|'
 
+class SortedConcatenateAmper(SortedConcatenate):
+    sep = '&'
+
 class IdentifiersConcat(object):
     '''String concatenation aggregator for the identifiers map'''
     def __init__(self):
@@ -220,7 +223,8 @@ class DBThread(Thread):
         self.conn.execute('pragma cache_size=5000')
         encoding = self.conn.execute('pragma encoding').fetchone()[0]
         self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
-        self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
+        self.conn.create_aggregate('sortconcat_bar', 2, SortedConcatenateBar)
+        self.conn.create_aggregate('sortconcat_amper', 2, SortedConcatenateAmper)
         self.conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat)
         load_c_extensions(self.conn)
         self.conn.row_factory = sqlite.Row if self.row_factory else  lambda cursor, row : list(row)

src/calibre/library/sqlite_custom.c

@@ -141,6 +141,22 @@ static void sort_concat_finalize2(sqlite3_context *context) {
 
 }
 
+static void sort_concat_finalize3(sqlite3_context *context) {
+    SortConcatList *list;
+    unsigned char *ans;
+
+    list = (SortConcatList*) sqlite3_aggregate_context(context, sizeof(*list));
+
+    if (list != NULL && list->vals != NULL && list->count > 0) {
+        qsort(list->vals, list->count, sizeof(list->vals[0]), sort_concat_cmp);
+        ans = sort_concat_do_finalize(list, '&');
+        if (ans != NULL) sqlite3_result_text(context, (char*)ans, -1, SQLITE_TRANSIENT);
+        free(ans);
+        sort_concat_free(list);
+    }
+
+}
+
 // }}}
 
 // identifiers_concat {{{
@@ -237,7 +253,8 @@ MYEXPORT int sqlite3_extension_init(
     sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi){
   SQLITE_EXTENSION_INIT2(pApi);
   sqlite3_create_function(db, "sortconcat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize);
-  sqlite3_create_function(db, "sort_concat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize2);
+  sqlite3_create_function(db, "sortconcat_bar", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize2);
+  sqlite3_create_function(db, "sortconcat_amper", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize3);
   sqlite3_create_function(db, "identifiers_concat", 2, SQLITE_UTF8, NULL, NULL, identifiers_concat_step, identifiers_concat_finalize);
   return 0;
 }

src/calibre/linux.py

@@ -23,7 +23,6 @@ entry_points = {
              'calibre-server     = calibre.library.server.main:main',
              'lrf2lrs            = calibre.ebooks.lrf.lrfparser:main',
              'lrs2lrf            = calibre.ebooks.lrf.lrs.convert_from:main',
-             'librarything       = calibre.ebooks.metadata.library_thing:main',
              'calibre-debug      = calibre.debug:main',
              'calibredb          = calibre.library.cli:main',
              'calibre-parallel   = calibre.utils.ipc.worker:main',

src/calibre/manual/faq.rst

@@ -562,6 +562,16 @@ You have two choices:
  1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_.
  2. `Open a ticket <http://calibre-ebook.com/bugs>`_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it.
 
+Why doesn't |app| have an automatic update?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For many reasons:
+
+  * *There is no need to update every week*. If you are happy with how |app| works turn off the update notification and be on your merry way. Check back to see if you want to update once a year or so.
+  * Pre downloading the updates for all users in the background would mean require about 80TB of bandwidth *every week*. That costs thousands of dollars a month. And |app| is currently growing at 300,000 new users every month.
+  * If I implement a dialog that downloads the update and launches it, instead of going to the website as it does now, that would save the most ardent |app| updater, *at most five clicks a week*. There are far higher priority things to do in |app| development.
+  * If you really, really hate downloading |app| every week but still want to be upto the latest, I encourage you to run from source, which makes updating trivial. Instructions are :ref:`here <develop>`.
+
 How is |app| licensed?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 |app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode <http://code.google.com/p/calibre-ebook/downloads/list>`_. You are free to use the results of conversions from |app| however you want. You cannot use code, libraries from |app| in your software without making your software open source. For details, see `The GNU GPL v3 <http://www.gnu.org/licenses/gpl.html>`_.

src/calibre/manual/gui.rst

@@ -74,10 +74,7 @@ Edit metadata
     1. **Edit metadata individually**: This allows you to edit the metadata of books one-by-one, with the option of fetching metadata, including covers from the internet. It also allows you to add/remove particular ebook formats from a book. 
     2. **Edit metadata in bulk**: This allows you to edit common metadata fields for large numbers of books simulataneously. It operates on all the books you have selected in the :ref:`Library view <search_sort>`.
     3. **Download metadata and covers**: Downloads metadata and covers (if available), for the books that are selected in the book list.
-    4. **Download only metadata**: Downloads only metadata (if available), for the books that are selected in the book list.
+    4. **Merge Book Records**: Gives you the capability of merging the metadata and formats of two or more book records together. You can choose to either delete or keep the records that were not clicked first.
-    5. **Download only covers**: Downloads only covers (if available), for the books that are selected in the book list.
-    6. **Download only social metadata**: Downloads only social metadata such as tags and reviews (if available), for the books that are selected in the book list.
-    7. **Merge Book Records**: Gives you the capability of merging the metadata and formats of two or more book records together. You can choose to either delete or keep the records that were not clicked first.
 
 For more details see :ref:`metadata`.
 

src/calibre/manual/portable.rst

@@ -11,7 +11,7 @@ You can "install" calibre onto a USB stick that you can take with you and use on
     * Run a Mobile Calibre installation with both the Calibre binaries and your ebook library resident on a USB disk or other portable media. In particular it is not necessary to have Calibre installed on the Windows PC that is to run Calibre. This batch file also does not care what drive letter is assigned when you plug in the USB device. It also will not affect any settings on the host machine being a completely self-contained Calibre installation.
     * Run a networked Calibre installation optimised for performance when the ebook files are located on a networked share.
 
-If you find setting up the bat file too challenging, there is a third party portable calibre build available at `portableapps.com http://portableapps.com`_.
+If you find setting up the bat file too challenging, there is a third party portable calibre build available at `portableapps.com <http://portableapps.com>`_.
 
 This calibre-portable.bat file is intended for use on Windows based systems, but the principles are easily adapted for use on Linux or OS X based systems. Note that calibre requires the Microsoft Visual C++ 2008 runtimes to run. Most windows computers have them installed already, but it may be a good idea to have the installer for installing them on your USB stick. The installer is available from `Microsoft <http://www.microsoft.com/downloads/details.aspx?FamilyID=9b2da534-3e03-4391-8a4d-074b9f2bc1bf&displaylang=en>`_.
 

src/calibre/manual/templates/search.html

@@ -0,0 +1,49 @@
+{#
+    basic/search.html
+    ~~~~~~~~~~~~~~~~~
+
+    Template for the search page, using google adsense search
+
+    :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+#}
+{% extends "layout.html" %}
+{% set title = _('Search') %}
+{% block extrahead %}
+    <link rel="stylesheet" href="http://www.google.com/cse/style/look/default.css" type="text/css" />
+  {{ super() }}
+{% endblock %}
+{% block body %}
+  <h1 id="search-documentation">{{ _('Search') }}</h1>
+  <div id="fallback" class="admonition warning">
+  <script type="text/javascript">$('#fallback').hide();</script>
+  <p>
+    {% trans %}Please activate JavaScript to enable the search
+    functionality.{% endtrans %}
+  </p>
+  </div>
+  <p>
+    {% trans %}From here you can search these documents. Enter your search
+    words into the box below and click "search". Note that the search
+    function will automatically search for all of the words. Pages
+    containing fewer words won't appear in the result list.{% endtrans %}
+  </p>
+  <div id="cse" style="width: 100%;">Loading</div>
+<script src="http://www.google.com/jsapi" type="text/javascript"></script>
+<script type="text/javascript"> 
+  google.load('search', '1');
+  google.setOnLoadCallback(function() {
+    var customSearchControl = new google.search.CustomSearchControl('partner-pub-2595272032872519:0657791363');
+    customSearchControl.setResultSetSize(google.search.Search.FILTERED_CSE_RESULTSET);
+    var options = new google.search.DrawOptions();
+    options.setAutoComplete(true);
+    customSearchControl.draw('cse', options);
+    var params = $.getQueryParameters();
+    if (params.q) {
+        var query = params.q[0];
+        $('input[name="search"]')[0].value = query;
+        $('input.gsc-search-button')[0].click();
+    }
+  }, true);
+</script>
+{% endblock %}

src/calibre/test_build.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+from future_builtins import map
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+'''
+Test a binary calibre build to ensure that all needed binary images/libraries have loaded.
+'''
+
+import cStringIO
+from calibre.constants import plugins, iswindows
+
+def test_plugins():
+    for name in plugins:
+        mod, err = plugins[name]
+        if err or not mod:
+            raise RuntimeError('Plugin %s failed to load with error: %s' %
+                    (name, err))
+        print (mod, 'loaded')
+
+def test_lxml():
+    from lxml import etree
+    raw = '<a/>'
+    root = etree.fromstring(raw)
+    if etree.tostring(root) == raw:
+        print ('lxml OK!')
+    else:
+        raise RuntimeError('lxml failed')
+
+def test_fontconfig():
+    from calibre.utils.fonts import fontconfig
+    families = fontconfig.find_font_families()
+    num = len(families)
+    if num < 10:
+        raise RuntimeError('Fontconfig found only %d font families'%num)
+    print ('Fontconfig OK! (%d families)'%num)
+
+def test_winutil():
+    from calibre.devices.scanner import win_pnp_drives
+    matches = win_pnp_drives.scanner()
+    if len(matches) < 1:
+        raise RuntimeError('win_pnp_drives returned no drives')
+    print ('win_pnp_drives OK!')
+
+def test_win32():
+    from calibre.utils.winshell import desktop
+    d = desktop()
+    if not d:
+        raise RuntimeError('winshell failed')
+    print ('winshell OK! (%s is the desktop)'%d)
+
+def test_sqlite():
+    import sqlite3
+    conn = sqlite3.connect(':memory:')
+    from calibre.library.sqlite import load_c_extensions
+    if not load_c_extensions(conn, True):
+        raise RuntimeError('Failed to load sqlite extension')
+    print ('sqlite OK!')
+
+def test_qt():
+    from PyQt4.Qt import (QWebView, QDialog, QImageReader, QNetworkAccessManager)
+    fmts = set(map(unicode, QImageReader.supportedImageFormats()))
+    if 'jpg' not in fmts or 'png' not in fmts:
+        raise RuntimeError(
+            "Qt doesn't seem to be able to load its image plugins")
+    QWebView, QDialog
+    na = QNetworkAccessManager()
+    if not hasattr(na, 'sslErrors'):
+        raise RuntimeError('Qt not compiled with openssl')
+    print ('Qt OK!')
+
+def test_imaging():
+    from calibre.utils.magick.draw import create_canvas, Image
+    im = create_canvas(20, 20, '#ffffff')
+    jpg = im.export('jpg')
+    Image().load(jpg)
+    im.export('png')
+    print ('ImageMagick OK!')
+    from PIL import Image
+    i = Image.open(cStringIO.StringIO(jpg))
+    if i.size != (20, 20):
+        raise RuntimeError('PIL choked!')
+    print ('PIL OK!')
+
+def test():
+    test_plugins()
+    test_lxml()
+    test_fontconfig()
+    test_sqlite()
+    if iswindows:
+        test_winutil()
+        test_win32()
+    test_qt()
+    test_imaging()
+
+if __name__ == '__main__':
+    test()
+

src/calibre/translations/dynamic.py

@@ -5,10 +5,9 @@ Dynamic language lookup of translations for user-visible strings.
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-import os
+import cStringIO
-
 from gettext import GNUTranslations
-from calibre.utils.localization import get_lc_messages_path
+from calibre.utils.localization import get_lc_messages_path, ZipFile
 
 __all__ = ['translate']
 
@@ -21,10 +20,15 @@ def translate(lang, text):
     else:
         mpath = get_lc_messages_path(lang)
         if mpath is not None:
-            p = os.path.join(mpath, 'messages.mo')
+            with ZipFile(P('localization/locales.zip',
-            if os.path.exists(p):
+                allow_user_override=False), 'r') as zf:
-                trans = GNUTranslations(open(p, 'rb'))
+                try:
-                _CACHE[lang] = trans
+                    buf = cStringIO.StringIO(zf.read(mpath + '/messages.mo'))
+                except:
+                    pass
+                else:
+                    trans = GNUTranslations(buf)
+                    _CACHE[lang] = trans
     if trans is None:
         return getattr(__builtins__, '_', lambda x: x)(text)
     return trans.ugettext(text)