sync with Kovid's branch

Changelog.yaml

@@ -19,6 +19,139 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.9.8
+  date: 2012-11-30
+
+  new features:
+    - title: "Add an option to show the cover size in the book details panel on the right. Option is in Preferences->Look & Feel->Book Details"
+
+    - title: "Kobo driver: Add support for firmware 2.2. Also add an option to send series information to the device."
+      description: "The newest Kobo firmware can display series information. Unfortunately, the Kobo does not read this information from the ebook file itself. It has to be sent separately after the Kobo has finished processing the new files. So you might have to connect - send books - disconnect and then re-connect for the series infor to show up. Fixes #1084388 (Add support for series on Kobo devices)"
+
+    - title: "Catalogs: Allow using custom columns as the source for Genres when generating catalogs"
+
+    - title: "When the user asks calibre to convert a book, show a small animation to highlight that the convert job has been queued to run in the background"
+
+    - title: "Add support for the notification center in OS X 10.8"
+
+    - title: "calibredb: Add an option to specify the cover to use when adding books with calibredb add."
+      tickets: [1083932]
+
+    - title: "EPUB Input: Add support for EPUB files with broken central directory records *and* data descriptors"
+
+    - title: "Comic metadata: Support reading metadata from cbr files. Also read the comments and published date info from the metadata."
+      tickets: [1082340]
+
+    - title: "Speed up processing of RAR and CBR files by avoiding an extra file copy"
+
+    - title: "Add driver for Nexus 10 on linux."
+      tickets: [1082563]
+
+  bug fixes:
+    - title: "KF8 Input: Handle invalid KF8 files with links pointing to non-existent locations and incorrect values in the div table."
+      tickets: [1082669]
+
+    - title: "Viewer: Fix handling of empty self closing tags."
+      tickets: [1083278]
+
+    - title: "Fix use of {formats} in save to disk templates. Fix some formatter functions causing plugboards to not validate."
+
+    - title: "Fix calibre quitting when minimized to system tray and an update available message is shown and then closed."
+      tickets: [1082630]
+
+    - title: "Viewer: Fix vertical margin at the top of the first page of a chapter incorrect in a certain rare circumstance (first child of body being an empty paragraph)."
+      tickets: [1082640]
+
+    - title: "E-book viewer: Fix bug that caused the default language for hyphenation to be ignored for books that do not specify a language"
+
+  improved recipes:
+    - Pro Physik
+    - Aachener Nachrichten
+    - Science News
+
+- version: 0.9.7
+  date: 2012-11-23
+
+  new features:
+    - title: "Edit metadata dialog: Show the size of the current book cover in the edit metadata dialog."
+      tickets: [1079781]
+
+    - title: "Get Books: Allow easy searching by title and author in addition to any keyword, to prevent large numbers of spurious matches."
+
+    - title: "An option to automatically convert any added book to the current output format, found under Preferences->Adding books"
+
+    - title: "E-book viewer: Allow viewing tables in a separate popup window by right clicking on the table and selecting 'View table'. Useful for reference books that have lots of large tables."
+      tickets: [1080710]
+
+    - title: "Catalogs: Add the current library name as an available field when generating catalogs in csv/xml format."
+      tickets: [1078422]
+
+    - title: "Enable colored text in the output from the command line tools on windows"
+
+    - title: "E-book viewer: Add an option to hide the help message when entering full screen mode"
+
+    - title: "E-book viewer: Add an option to always start the viewer in full screen mode"
+
+    - title: "E-book viewer: Add many more controls to the context menu, particularly useful in full screen mode"
+
+    - title: "E-book viewer: Allow easy searching of the selected word or phrase in google via the context menu"
+
+    - title: "Add a new type of FileType plugin, postimport, that runs after a book has been added to the database."
+
+    - title: "Get Books: Remove Gandalf store, add Publio store. Update the Legimi store plugin for website changes"
+ 
+  bug fixes:
+    - title: "Conversion: Correctly handle values of left and right for the deprecated align attribute of images, mapping them to the CSS float property instead of to text-align."
+      tickets: [1081094]
+
+    - title: "MOBI Output: When generating joint MOBI6/KF8 files do not set incorrect display CSS values for tables in the KF8 part"
+
+    - title: "Connect to iTunes: Ignore AAC audio files."
+      tickets: [1081096]
+
+    - title: "E-book viewer: Fix restoring from fullscreen not respecting maximized window state"
+
+    - title: "Fix rows in the device books view sometimes being too high"
+
+    - title: "Catalogs: Fixed a problem occurring when merging comments with a custom field whose type is a list."
+
+    - title: "Linux binary: Use exec in the wrapper shell scripts that are used to set env vars and launch calibre utilities."
+      tickets: [1077884]
+
+    - title: "E-book viewer: Fix blank pages after every page when viewing some comic files in paged mode"
+
+    - title: "E-book viewer: When printing, respect the specified page range."
+      tickets: [1074220]
+
+    - title: "Font subsetting: Parse the GSUB table for glyph substitution rules and do not remove any glyphs that could act as substitutes. Keep zero length glyphs like the glyphs for non printable characters when subsetting TrueType outlines."
+
+    - title: "Smarten punctuation: Fix self closing script tags causing smarten punctuation to fail"
+
+
+  improved recipes:
+    - Arguments and facts
+    - Business Standard
+    - The New Yorker
+
+  new recipes:
+    - title: Various Czech and Hungarian news sources 
+      author: bubak
+
+    - title: Various Polish recipes 
+      author: Artur Stachecki
+
+    - title: Buchreport 
+      author: a.peter
+
+    - title: Red Voltaire 
+      author: atordo
+
+    - title: Autosport 
+      author: Mr Stefan 
+
+    - title: House News
+      author: Eddie Lau
+
 - version: 0.9.6
   date: 2012-11-10
 

recipes/aachener_nachrichten.recipe

@@ -2,41 +2,70 @@ from calibre.web.feeds.recipes import BasicNewsRecipe
 class AdvancedUserRecipe(BasicNewsRecipe):
 
     title          = u'Aachener Nachrichten'
-    __author__  = 'schuster'
+    __author__  = 'schuster' #AGE update 2012-11-28
-    oldest_article = 1
+    oldest_article        = 1
     max_articles_per_feed = 100
-    use_embedded_content   = False
+    no_stylesheets        = True
-    language               = 'de'
+    remove_javascript     = True
-    remove_javascript      = True
+    remove_empty_feeds    = True
-    cover_url = 'http://www.an-online.de/einwaage/images/an_logo.png'
+    language              = 'de'
-    masthead_url = 'http://www.an-online.de/einwaage/images/an_logo.png'
-    extra_css = '''
-	.fliesstext_detail:{margin-bottom:10%;}
-	.headline_1:{margin-bottom:25%;}
-                    b{font-family:Arial,Helvetica,sans-serif; font-weight:200;font-size:large;}
-                    a{font-family:Arial,Helvetica,sans-serif; font-weight:400;font-size:large;}
-                    ll{font-family:Arial,Helvetica,sans-serif; font-weight:100;font-size:large;}
-                    h4{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
-                    img {min-width:300px; max-width:600px; min-height:300px; max-height:800px}
-                    dd{font-family:Arial,Helvetica,sans-serif;font-size:large;}
-                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
-	'''
-
 
+#    cover_url     = 'http://www.aachener-nachrichten.de/img/logos/an_website_retina.png'
+    masthead_url  = 'http://www.aachener-nachrichten.de/img/logos/an_website_retina.png'
 
     keep_only_tags    = [
-                       dict(name='span', attrs={'class':['fliesstext_detail', 'headline_1', 'autor_detail']}),
+                          dict(name='article', attrs={'class':['single']})
-                       dict(id=['header-logo'])
+                        ]
-  ]
 
-    feeds          = [(u'Euregio', u'http://www.an-online.de/an/rss/Euregio.xml'),
+    remove_tags = [
-                          (u'Aachen', u'http://www.an-online.de/an/rss/Aachen.xml'),
+                    dict(name='div', attrs={'class':["clearfix navi-wrapper"]}),
-                          (u'Nordkreis', u'http://www.an-online.de/an/rss/Nordkreis.xml'),
+                    dict(name='div', attrs={'id':["article_actions"]}),
-                          (u'Düren', u'http://www.an-online.de/an/rss/Dueren.xml'),
+                    dict(name='style', attrs={'type':["text/css"]}),
-                          (u'Eiffel', u'http://www.an-online.de/an/rss/Eifel.xml'),
+                    dict(name='aside'),
-                          (u'Eschweiler', u'http://www.an-online.de/an/rss/Eschweiler.xml'),
+                    dict(name='a', attrs={'class':["btn btn-action"]})
-                          (u'Geilenkirchen', u'http://www.an-online.de/an/rss/Geilenkirchen.xml'),
+                  ]
-                          (u'Heinsberg', u'http://www.an-online.de/an/rss/Heinsberg.xml'),
+
-                          (u'Jülich', u'http://www.an-online.de/an/rss/Juelich.xml'),
+    feeds = [
-                          (u'Stolberg', u'http://www.an-online.de/an/rss/Stolberg.xml'),
+              (u'Lokales - Euregio', u'http://www.aachener-nachrichten.de/cmlink/euregio-rss-1.357285'),
-                          (u'Ratgebenr', u'http://www.an-online.de/an/rss/Ratgeber.xml')]
+              (u'Lokales - Aachen', u'http://www.aachener-nachrichten.de/cmlink/aachen-rss-1.357286'),
+              (u'Lokales - Nordkreis', u'http://www.aachener-nachrichten.de/cmlink/nordkreis-rss-1.358150'),
+              (u'Lokales - Düren', u'http://www.aachener-nachrichten.de/cmlink/dueren-rss-1.358626'),
+              (u'Lokales - Eiffel', u'http://www.aachener-nachrichten.de/cmlink/eifel-rss-1.358978'),
+              (u'Lokales - Eschweiler', u'http://www.aachener-nachrichten.de/cmlink/eschweiler-rss-1.359332'),
+              (u'Lokales - Geilenkirchen', u'http://www.aachener-nachrichten.de/cmlink/geilenkirchen-rss-1.359643'),
+              (u'Lokales - Heinsberg', u'http://www.aachener-nachrichten.de/cmlink/heinsberg-rss-1.359724'),
+              (u'Lokales - Jülich', u'http://www.aachener-nachrichten.de/cmlink/juelich-rss-1.359725'),
+              (u'Lokales - Stolberg', u'http://www.aachener-nachrichten.de/cmlink/stolberg-rss-1.359726'),
+              (u'News - Politik', u'http://www.aachener-nachrichten.de/cmlink/politik-rss-1.359727'),
+              (u'News - Aus aller Welt', u'http://www.aachener-nachrichten.de/cmlink/ausallerwelt-rss-1.453282'),
+              (u'News - Wirtschaft', u'http://www.aachener-nachrichten.de/cmlink/wirtschaft-rss-1.359872'),
+              (u'News - Kultur', u'http://www.aachener-nachrichten.de/cmlink/kultur-rss-1.365018'),
+              (u'News - Kino', u'http://www.aachener-nachrichten.de/cmlink/kino-rss-1.365019'),
+              (u'News - Digital', u'http://www.aachener-nachrichten.de/cmlink/digital-rss-1.365020'),
+              (u'News - Wissenschaft', u'http://www.aachener-nachrichten.de/cmlink/wissenschaft-rss-1.365021'),
+              (u'News - Hochschule', u'http://www.aachener-nachrichten.de/cmlink/hochschule-rss-1.365022'),
+              (u'News - Auto', u'http://www.aachener-nachrichten.de/cmlink/auto-rss-1.365023'),
+              (u'News - Kurioses', u'http://www.aachener-nachrichten.de/cmlink/kurioses-rss-1.365067'),
+              (u'News - Musik', u'http://www.aachener-nachrichten.de/cmlink/musik-rss-1.365305'),
+              (u'News - Tagesthema', u'http://www.aachener-nachrichten.de/cmlink/tagesthema-rss-1.365519'),
+              (u'News - Newsticker', u'http://www.aachener-nachrichten.de/cmlink/newsticker-rss-1.451948'),
+              (u'Sport - Aktuell', u'http://www.aachener-nachrichten.de/cmlink/aktuell-rss-1.366716'),
+              (u'Sport - Fußball', u'http://www.aachener-nachrichten.de/cmlink/fussball-rss-1.367060'),
+              (u'Sport - Bundesliga', u'http://www.aachener-nachrichten.de/cmlink/bundesliga-rss-1.453367'),
+              (u'Sport - Alemannia Aachen', u'http://www.aachener-nachrichten.de/cmlink/alemanniaaachen-rss-1.366057'),
+              (u'Sport - Volleyball', u'http://www.aachener-nachrichten.de/cmlink/volleyball-rss-1.453370'),
+              (u'Sport - Chio', u'http://www.aachener-nachrichten.de/cmlink/chio-rss-1.453371'),
+              (u'Dossier - Kinderuni', u'http://www.aachener-nachrichten.de/cmlink/kinderuni-rss-1.453375'),
+              (u'Dossier - Karlspreis', u'http://www.aachener-nachrichten.de/cmlink/karlspreis-rss-1.453376'),
+              (u'Dossier - Ritterorden', u'http://www.aachener-nachrichten.de/cmlink/ritterorden-rss-1.453377'),
+              (u'Dossier - ZAB-Aachen', u'http://www.aachener-nachrichten.de/cmlink/zabaachen-rss-1.453380'),
+              (u'Dossier - Karneval', u'http://www.aachener-nachrichten.de/cmlink/karneval-rss-1.453384'),
+              (u'Ratgeber - Geld', u'http://www.aachener-nachrichten.de/cmlink/geld-rss-1.453385'),
+              (u'Ratgeber - Recht', u'http://www.aachener-nachrichten.de/cmlink/recht-rss-1.453386'),
+              (u'Ratgeber - Gesundheit', u'http://www.aachener-nachrichten.de/cmlink/gesundheit-rss-1.453387'),
+              (u'Ratgeber - Familie', u'http://www.aachener-nachrichten.de/cmlink/familie-rss-1.453388'),
+              (u'Ratgeber - Livestyle', u'http://www.aachener-nachrichten.de/cmlink/lifestyle-rss-1.453389'),
+              (u'Ratgeber - Reisen', u'http://www.aachener-nachrichten.de/cmlink/reisen-rss-1.453390'),
+              (u'Ratgeber - Bauen und Wohnen', u'http://www.aachener-nachrichten.de/cmlink/bauen-rss-1.453398'),
+              (u'Ratgeber - Bildung und Beruf', u'http://www.aachener-nachrichten.de/cmlink/bildung-rss-1.453400'),
+            ]

recipes/app_funds.recipe

@@ -0,0 +1,27 @@
+#!/usr/bin/env  python
+
+__license__ = 'GPL v3'
+__author__ = 'teepel <teepel44@gmail.com>'
+
+'''
+appfunds.blogspot.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class app_funds(BasicNewsRecipe):
+    title          = u'APP Funds'
+    __author__ = 'teepel <teepel44@gmail.com>'
+    language       = 'pl'
+    description ='Blog inwestora dla inwestorów i oszczędzających'
+    INDEX='http://appfunds.blogspot.com'
+    remove_empty_feeds= True
+    oldest_article = 7
+    max_articles_per_feed = 100
+    simultaneous_downloads = 5
+    remove_javascript=True
+    no_stylesheets=True
+    auto_cleanup = True
+
+    feeds          = [(u'blog', u'http://feeds.feedburner.com/blogspot/etVI')]
+    

recipes/endgadget.recipe

@@ -21,10 +21,11 @@ class Engadget(BasicNewsRecipe):
     use_embedded_content  = False
     remove_javascript     = True
     remove_empty_feeds    = True
+    auto_cleanup = True
 
-    keep_only_tags = [dict(name='div', attrs={'class':['post_content permalink ','post_content permalink alt-post-full']})]
+    #keep_only_tags = [dict(name='div', attrs={'class':['post_content permalink ','post_content permalink alt-post-full']})]
-    remove_tags = [dict(name='div', attrs={'class':['filed_under','post_footer']})]
+    #remove_tags = [dict(name='div', attrs={'class':['filed_under','post_footer']})]
-    remove_tags_after =  [dict(name='div', attrs={'class':['post_footer']})]
+    #remove_tags_after =  [dict(name='div', attrs={'class':['post_footer']})]
 
     feeds = [(u'Posts', u'http://www.engadget.com/rss.xml')]
 
@@ -33,5 +34,5 @@ class Engadget(BasicNewsRecipe):
                     h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
                     p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
                     body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
-		'''
+        '''
 

recipes/fronda.recipe

@@ -1,39 +1,88 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = u'2010, Tomasz Dlugosz <tomek3d@gmail.com>'
+__copyright__ = u'2010-2012, Tomasz Dlugosz <tomek3d@gmail.com>'
 '''
 fronda.pl
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
-import re
+from datetime import timedelta, date
 
 class Fronda(BasicNewsRecipe):
     title          = u'Fronda.pl'
     publisher      = u'Fronda.pl'
-    description    = u'Portal po\u015bwi\u0119cony - Infformacje'
+    description    = u'Portal po\u015bwi\u0119cony - Informacje'
     language = 'pl'
     __author__ = u'Tomasz D\u0142ugosz'
     oldest_article = 7
     max_articles_per_feed = 100
     use_embedded_content = False
+    no_stylesheets = True
 
-    feeds          = [(u'Infformacje', u'http://fronda.pl/news/feed')]
+    extra_css = '''
+        h1 {font-size:150%}
+        .body {text-align:left;}
+        div.headline {font-weight:bold}
+    '''
 
-    keep_only_tags = [dict(name='h2', attrs={'class':'news_title'}),
+    earliest_date = date.today() - timedelta(days=oldest_article)
-                      dict(name='div', attrs={'class':'naglowek_tresc'}),
-                      dict(name='div', attrs={'id':'czytaj'}) ]
 
-    remove_tags = [dict(name='a', attrs={'class':'print'})]
+    def date_cut(self,datestr):
+        # eg. 5.11.2012, 12:07
+        timestamp = datestr.split(',')[0]
+        parts = timestamp.split('.')
+        art_date = date(int(parts[2]),int(parts[1]),int(parts[0]))
+        return True if art_date < self.earliest_date else False
 
-    preprocess_regexps = [
+    def parse_index(self):
-        (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in 
+        genres = [
-            [ (r'<p><a href="http://fronda.pl/sklepy">.*</a></p>', lambda match: ''),
+            ('ekonomia,4.html', 'Ekonomia'),
-              (r'<p><a href="http://fronda.pl/pasaz">.*</a></p>', lambda match: ''),
+            ('filozofia,15.html', 'Filozofia'),
-              (r'<h3><strong>W.* lektury.*</a></p></div>', lambda match: '</div>'),
+            ('historia,6.html', 'Historia'),
-              (r'<h3>Zobacz t.*?</div>', lambda match: '</div>'),
+            ('kosciol,8.html', 'Kościół'),
-              (r'<p[^>]*>&nbsp;</p>', lambda match: ''),
+            ('kultura,5.html', 'Kultura'),
-              (r'<p><span style=".*?"><br /></span></p> ', lambda match: ''),
+            ('media,10.html', 'Media'),
-              (r'<a style=\'float:right;margin-top:3px;\' href="http://www.facebook.com/share.php?.*?</a>', lambda match: '')]
+            ('nauka,9.html', 'Nauka'),
-    ]
+            ('polityka,11.html', 'Polityka'),
+            ('polska,12.html', 'Polska'),
+            ('prolife,3.html', 'Prolife'),
+            ('religia,7.html', 'Religia'),
+            ('rodzina,13.html', 'Rodzina'),
+            ('swiat,14.html', 'Świat'),
+            ('wydarzenie,16.html', 'Wydarzenie')
+        ]
+        feeds = []
+        articles = {}
+
+        for url, genName in genres:
+            soup = self.index_to_soup('http://www.fronda.pl/c/'+ url)
+            articles[genName] = []
+            for item in soup.findAll('li'):
+                article_h = item.find('h2')
+                if not article_h:
+                    continue
+                article_date = self.tag_to_string(item.find('b'))
+                if self.date_cut(article_date):
+                    continue
+                article_a = article_h.find('a')
+                article_url = 'http://www.fronda.pl' + article_a['href']
+                article_title = self.tag_to_string(article_a)
+                articles[genName].append( { 'title' : article_title, 'url' : article_url, 'date' : article_date })
+            feeds.append((genName, articles[genName]))
+        return feeds
+
+    keep_only_tags = [ 
+        dict(name='div', attrs={'class':'yui-g'})
+        ]
+
+    remove_tags = [
+        dict(name='div', attrs={'class':['related-articles',
+                'button right',
+                'pagination']}),
+        dict(name='h3', attrs={'class':'block-header article comments'}),
+        dict(name='ul', attrs={'class':'comment-list'}),
+        dict(name='ul', attrs={'class':'category'}),
+        dict(name='p', attrs={'id':'comments-disclaimer'}),
+        dict(name='div', attrs={'id':'comment-form'})
+        ]

recipes/house_news.recipe

@@ -0,0 +1,30 @@
+__license__   = 'GPL v3'
+__copyright__ = '2012, Eddie Lau'
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipeHouseNews(BasicNewsRecipe):
+    title          = u'House News \u4e3b\u5834\u65b0\u805e'
+    __author__     = 'Eddie Lau'
+    publisher      = 'House News'
+    oldest_article = 1
+    max_articles_per_feed = 100
+    auto_cleanup = False
+    language = 'zh'
+    encoding = 'utf-8'
+    description = 'http://thehousenews.com'
+    category    = 'Chinese, Blogs, Opinion, News, Hong Kong'
+    masthead_url = 'http://thehousenews.com/static/images/housebeta.jpg'
+    extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px; max-height:90%;} p[class=date] {font-size:50%;} div[class=author] {font-size:75%;} p[class=caption] {font-size:50%;}'
+    feeds          = [(u'Latest', u'http://thehousenews.com/rss/')]
+    keep_only_tags = [dict(name='h1'),
+                      dict(name='div', attrs={'class':['photo']}),
+                      dict(name='p', attrs={'class':'caption'}),
+                      dict(name='div', attrs={'class':'articleTextWrap'}),
+                      dict(name='div', attrs={'class':['author']}),
+                      dict(name='p', attrs={'class':'date'})]
+
+    def populate_article_metadata(self, article, soup, first):
+        if first and hasattr(self, 'add_toc_thumbnail'):
+            picdiv = soup.find('img')
+            if picdiv is not None:
+                self.add_toc_thumbnail(article,picdiv['src'])

recipes/kp.recipe

@@ -0,0 +1,52 @@
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class KrytykaPolitycznaRecipe(BasicNewsRecipe):
+    __license__ = 'GPL v3'
+    __author__ = u'intromatyk <intromatyk@gmail.com>'
+    language = 'pl'
+    version = 1
+
+    title = u'Krytyka Polityczna'
+    category = u'News'
+    description = u' Lewicowe pismo zaangażowane w bieg spraw publicznych w Polsce.'
+    cover_url=''
+    remove_empty_feeds= True
+    no_stylesheets=True
+    oldest_article = 7
+    max_articles_per_feed = 100000
+    recursions = 0
+
+    no_stylesheets = True
+    remove_javascript = True
+    simultaneous_downloads = 3
+
+    keep_only_tags =[]
+    keep_only_tags.append(dict(name = 'h1', attrs = {'class' : 'print-title'}))
+    keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'print-content'}))
+
+    remove_tags =[]
+    remove_tags.append(dict(attrs = {'class' : ['field field-type-text field-field-story-switch', 'field field-type-filefield field-field-story-temp' , 'field field-type-text field-field-story-author', 'field field-type-text field-field-story-lead-switch']}))
+
+    extra_css = '''
+                    body {font-family: verdana, arial, helvetica, geneva, sans-serif ;}
+                    td.contentheading{font-size: large; font-weight: bold;}
+                    '''
+
+    feeds          = [
+                            ('Wszystkie', 'http://www.krytykapolityczna.pl/rss.xml')
+                          ]
+
+    def print_version(self, url):
+        soup = self.index_to_soup(url)
+        print_ico = soup.find(attrs = {'class' : 'print-page'})
+        print_uri = print_ico['href']
+        self.log('PRINT', print_uri)
+        return 'http://www.krytykapolityczna.pl/' + print_uri
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup

recipes/newsweek_polska.recipe

@@ -13,7 +13,7 @@ import datetime
 class Newsweek(BasicNewsRecipe):
 
 	# how many issues to go back, 0 means get the most current one
-	BACK_ISSUES = 1
+	BACK_ISSUES = 2
 
 	EDITION = '0'
 	DATE = None

recipes/prawica_net.recipe

@@ -0,0 +1,40 @@
+#!/usr/bin/env  python
+
+__license__ = 'GPL v3'
+__author__ = 'teepel <teepel44@gmail.com>'
+
+'''
+http://prawica.net
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class prawica_recipe(BasicNewsRecipe):
+    title          = u'prawica.net'
+    __author__ = 'teepel <teepel44@gmail.com>'
+    language       = 'pl'
+    description ='Wiadomości ze strony prawica.net'
+    INDEX='http://prawica.net/'
+    remove_empty_feeds= True
+    oldest_article = 1
+    max_articles_per_feed = 100
+    remove_javascript=True
+    no_stylesheets=True
+
+    feeds          = [(u'all', u'http://prawica.net/all/feed')]
+
+
+    keep_only_tags =[]
+    #this line should show title of the article, but it doesnt work
+    keep_only_tags.append(dict(name = 'h1', attrs = {'class' : 'print-title'}))
+    keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'content'}))
+
+
+    remove_tags =[]
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'field field-type-viewfield field-field-autor2'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'field field-type-viewfield field-field-publikacje-autora'}))
+    remove_tags.append(dict(name = 'div', attrs = {'id' : 'rate-widget-2 rate-widget clear-block rate-average rate-widget-fivestar rate-daa7512627f21dcf15e0af47e5279f0e rate-processed'}))
+    remove_tags_after =[(dict(name = 'div', attrs = {'class' : 'field-label-inline-first'}))]
+
+    def print_version(self, url):
+        return url.replace('http://prawica.net/', 'http://prawica.net/print/')

recipes/pro_physik.recipe

@@ -2,21 +2,46 @@ from calibre.web.feeds.recipes import BasicNewsRecipe
 class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
     title          = u'Pro Physik'
-    __author__  = 'schuster'
+    __author__  = 'schuster, Armin Geller' # AGE Upd. 2012-11-28
-    oldest_article = 4
+    oldest_article        = 4
     max_articles_per_feed = 100
-    no_stylesheets         = True
+
-    use_embedded_content   = False
+    no_stylesheets        = True
-    language               = 'de'
+    remove_javascript     = True
-    remove_javascript      = True
+    remove_empty_feeds    = True
-    cover_url = 'http://www.pro-physik.de/Phy/images/site/prophysik_logo1.jpg'
+    language              = 'de'
+
+    cover_url = 'http://www.pro-physik.de/prophy/images/bg_logo_prophy.gif'
 
 
-    def print_version(self, url):
+    keep_only_tags    = [
-        return url.replace('leadArticle.do', 'print.do')
+                          dict(name='div', attrs={'class':['leftColRight']})
+                        ]
 
+    remove_tags = [
+                    dict(name='div', attrs={'class':["withMargin socialWrapper addthis_toolbox addthis_default_style"]}),
+                  # AGe: If you don't like to see further informations for the article
+                  #      and additional links please remove # in belows next line
+#                    dict(name='div', attrs={'class':["insideBox"]}),
+                  ]
 
-    feeds          = [(u'Hightech', u'http://www.pro-physik.de/Phy/hightechfeed.xml'),
+    feeds = [
-                          (u'Forschung', u'http://www.pro-physik.de/Phy/forschungfeed.xml'),
+              (u'Nachrichten', u'http://www.pro-physik.de/graphicalrss/prophy/newsFeed.xml'),
-                          (u'Magazin', u'http://www.pro-physik.de/Phy/magazinfeed.xml')]
+              (u'Forschung', u'http://www.pro-physik.de/graphicalrss/prophy/newsforschungFeed.xml'),
+              (u'Techologie', u'http://www.pro-physik.de/graphicalrss/prophy/newstechnologieFeed.xml'),
+              (u'Industrie', u'http://www.pro-physik.de/graphicalrss/prophy/newsindustrieFeed.xml'),
+              (u'Hochschule', u'http://www.pro-physik.de/graphicalrss/prophy/newshochschuleFeed.xml'),
+              (u'Panorama', u'http://www.pro-physik.de/graphicalrss/prophy/newspanoramaFeed.xml'),
+              (u'DPG', u'http://www.pro-physik.de/graphicalrss/prophy/newsdpgFeed.xml'),
+              (u'Physik Jornal', u'http://www.pro-physik.de/graphicalrss/prophy/pjnewsFeed.xml'),
+              (u'Veranstaltungen', u'http://www.pro-physik.de/rss/prophy/eventsFeed.xml'),
 
+              # AGe if you like to see job offers please remove # on next lines below
+
+#              (u'Stellenmarkt', u'http://www.pro-physik.de/rss/prophy/jobsFeed.xml'),
+#              (u'Industrie Stellenanzeigen', u'http://www.pro-physik.de/rss/prophy/jobsindustrieFeed.xml'),
+#              (u'PhD Stellenanzeigen', u'http://www.pro-physik.de/rss/prophy/jobsphdFeed.xml'),
+#              (u'PostDoc Stellenanzeigen', u'http://www.pro-physik.de/rss/prophy/jobspostdocFeed.xml'),
+#              (u'Öffentlicher Dienst Stellenanzeigen', u'http://www.pro-physik.de/rss/prophy/jobsdienstFeed.xml'),
+#              (u'Hochschule Stellenanzeigen', u'http://www.pro-physik.de/rss/prophy/jobshochschuleFeed.xml'),
+            ]

recipes/rybinski.recipe

@@ -0,0 +1,29 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2012, Tomasz Dlugosz <tomek3d@gmail.com>'
+'''
+rybinski.eu
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Rybinski(BasicNewsRecipe):
+    title          = u'Rybinski.eu - economy of the XXI century'
+    description    = u'Blog ekonomiczny dra hab. Krzysztofa Rybi\u0144skiego'
+    language = 'pl'
+    __author__ = u'Tomasz D\u0142ugosz'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+
+    feeds          = [(u'wpisy', u'http://www.rybinski.eu/?feed=rss2&lang=pl')]
+
+    keep_only_tags = [dict(name='div', attrs={'class':'post'})]
+
+    remove_tags = [
+        dict(name = 'div', attrs = {'class' : 'post-meta-1'}),
+        dict(name = 'div', attrs = {'class' : 'post-meta-2'}),
+        dict(name = 'div', attrs = {'class' : 'post-comments'})
+    ]
+

recipes/samcik_blox.recipe

@@ -0,0 +1,26 @@
+#!/usr/bin/env  python
+
+__license__ = 'GPL v3'
+__author__ = 'teepel <teepel44@gmail.com>'
+
+'''
+samcik.blox.pl
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class samcik(BasicNewsRecipe):
+    title          = u'Maciej Samcik Blog'
+    __author__ = 'teepel <teepel44@gmail.com>'
+    language       = 'pl'
+    description =u'Blog Macieja Samcika, długoletniego dziennikarza ekonomicznego Gazety Wyborczej . O finansach małych i dużych. Mnóstwo ciekawostek na temat pieniędzy.'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    remove_javascript=True
+    no_stylesheets=True
+    simultaneous_downloads = 3
+
+    remove_tags =[]
+    remove_tags.append(dict(name = 'table', attrs = {'border' : '0'}))
+      
+    feeds          = [(u'Wpisy', u'http://samcik.blox.pl/rss2')]

recipes/science_news.recipe

@@ -17,6 +17,7 @@ class Sciencenews(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
+    auto_cleanup = True
     timefmt               = ' [%A, %d %B, %Y]'
 
     extra_css = '''
@@ -31,14 +32,14 @@ class Sciencenews(BasicNewsRecipe):
                 .credit{color:#A6A6A6;font-family:helvetica,arial ;font-size: xx-small ;}
                 '''
 
-    keep_only_tags = [ dict(name='div', attrs={'id':'column_action'}) ]
+    #keep_only_tags = [ dict(name='div', attrs={'id':'column_action'}) ]
-    remove_tags_after = dict(name='ul', attrs={'id':'content_functions_bottom'})
+    #remove_tags_after = dict(name='ul', attrs={'id':'content_functions_bottom'})
-    remove_tags = [
+    #remove_tags = [
-                     dict(name='ul', attrs={'id':'content_functions_bottom'})
+                     #dict(name='ul', attrs={'id':'content_functions_bottom'})
-                    ,dict(name='div', attrs={'id':['content_functions_top','breadcrumb_content']})
+                    #,dict(name='div', attrs={'id':['content_functions_top','breadcrumb_content']})
-                    ,dict(name='img', attrs={'class':'icon'})
+                    #,dict(name='img', attrs={'class':'icon'})
-                    ,dict(name='div', attrs={'class': 'embiggen'})
+                    #,dict(name='div', attrs={'class': 'embiggen'})
-                  ]
+                  #]
 
     feeds       = [(u"Science News / News Items", u'http://sciencenews.org/index.php/feed/type/news/name/news.rss/view/feed/name/all.rss')]
 
@@ -53,9 +54,9 @@ class Sciencenews(BasicNewsRecipe):
 
         return cover_url
 
-    def preprocess_html(self, soup):
+    #def preprocess_html(self, soup):
 
-            for tag in soup.findAll(name=['span']):
+            #for tag in soup.findAll(name=['span']):
-                tag.name = 'div'
+                #tag.name = 'div'
 
-            return soup
+            #return soup

session.vim

@@ -11,6 +11,7 @@ let g:syntastic_cpp_include_dirs = [
             \'/usr/include/freetype2',
             \'/usr/include/fontconfig',
             \'src/qtcurve/common', 'src/qtcurve',
+            \'src/unrar',
             \'/usr/include/ImageMagick',
             \]
 let g:syntastic_c_include_dirs = g:syntastic_cpp_include_dirs

setup/__init__.py

@@ -215,6 +215,8 @@ class Command(object):
         sys.stdout.flush()
 
 def installer_name(ext, is64bit=False):
+    if is64bit and ext == 'msi':
+        return 'dist/%s-64bit-%s.msi'%(__appname__, __version__)
     if ext in ('exe', 'msi'):
         return 'dist/%s-%s.%s'%(__appname__, __version__, ext)
     if ext == 'dmg':

setup/build_environment.py

@@ -6,17 +6,16 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, socket, struct, subprocess, sys
+import os, socket, struct, subprocess, sys, glob
 from distutils.spawn import find_executable
 
 from PyQt4 import pyqtconfig
 
-from setup import isosx, iswindows, islinux
+from setup import isosx, iswindows, islinux, is64bit
 
 OSX_SDK = '/Developer/SDKs/MacOSX10.5.sdk'
 
 os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.5'
-is64bit = sys.maxsize > 2**32
 
 NMAKE = RC = msvc = MT = win_inc = win_lib = win_ddk = win_ddk_lib_dirs = None
 if iswindows:
@@ -36,7 +35,7 @@ if iswindows:
             MT = os.path.join(os.path.dirname(p), 'bin', 'mt.exe')
     MT = os.path.join(SDK, 'bin', 'mt.exe')
     os.environ['QMAKESPEC'] = 'win32-msvc'
-    ICU = r'Q:\icu'
+    ICU = os.environ.get('ICU_DIR', r'Q:\icu')
 
 QMAKE = '/Volumes/sw/qt/bin/qmake' if isosx else 'qmake'
 if find_executable('qmake-qt4'):
@@ -122,7 +121,8 @@ if iswindows:
     zlib_lib_dirs = [sw_lib_dir]
     zlib_libs = ['zlib']
 
-    magick_inc_dirs = [os.path.join(prefix, 'build', 'ImageMagick-6.7.6')]
+    md = glob.glob(os.path.join(prefix, 'build', 'ImageMagick-*'))[-1]
+    magick_inc_dirs = [md]
     magick_lib_dirs = [os.path.join(magick_inc_dirs[0], 'VisualMagick', 'lib')]
     magick_libs = ['CORE_RL_wand_', 'CORE_RL_magick_']
     podofo_inc = os.path.join(sw_inc_dir, 'podofo')

setup/commands.py

@@ -20,7 +20,7 @@ __all__ = [
         'upload_user_manual', 'upload_demo', 'reupload',
         'linux32', 'linux64', 'linux', 'linux_freeze',
         'osx32_freeze', 'osx', 'rsync', 'push',
-        'win32_freeze', 'win32', 'win',
+        'win32_freeze', 'win32', 'win64', 'win',
         'stage1', 'stage2', 'stage3', 'stage4', 'stage5', 'publish'
         ]
 
@@ -91,9 +91,10 @@ osx = OSX()
 from setup.installer.osx.app.main import OSX32_Freeze
 osx32_freeze = OSX32_Freeze()
 
-from setup.installer.windows import Win, Win32
+from setup.installer.windows import Win, Win32, Win64
 win = Win()
 win32 = Win32()
+win64 = Win64()
 from setup.installer.windows.freeze import Win32Freeze
 win32_freeze = Win32Freeze()
 

setup/extensions.py

@@ -47,6 +47,13 @@ class Extension(object):
         self.ldflags = kwargs.get('ldflags', [])
         self.optional = kwargs.get('optional', False)
         self.needs_ddk = kwargs.get('needs_ddk', False)
+        of = kwargs.get('optimize_level', None)
+        if of is None:
+            of = '/Ox' if iswindows else '-O3'
+        else:
+            flag = '/O%d' if iswindows else '-O%d'
+            of = flag % of
+        self.cflags.insert(0, of)
 
     def preflight(self, obj_dir, compiler, linker, builder, cflags, ldflags):
         pass
@@ -176,6 +183,24 @@ extensions = [
                 sip_files = ['calibre/gui2/progress_indicator/QProgressIndicator.sip']
                 ),
 
+    Extension('unrar',
+              ['unrar/%s.cpp'%(x.partition('.')[0]) for x in '''
+               rar.o strlist.o strfn.o pathfn.o savepos.o smallfn.o global.o file.o
+               filefn.o filcreat.o archive.o arcread.o unicode.o system.o
+               isnt.o crypt.o crc.o rawread.o encname.o resource.o match.o
+               timefn.o rdwrfn.o consio.o options.o ulinks.o errhnd.o rarvm.o
+               secpassword.o rijndael.o getbits.o sha1.o extinfo.o extract.o
+               volume.o list.o find.o unpack.o cmddata.o filestr.o scantree.o
+               '''.split()] + ['calibre/utils/unrar.cpp'],
+              inc_dirs=['unrar'],
+              cflags = [('/' if iswindows else '-') + x for x in (
+                  'DSILENT', 'DRARDLL', 'DUNRAR')] + (
+                  [] if iswindows else ['-D_FILE_OFFSET_BITS=64',
+                                        '-D_LARGEFILE_SOURCE']),
+              optimize_level=2,
+              libraries=['User32', 'Advapi32', 'kernel32', 'Shell32'] if iswindows else []
+              ),
+
     ]
 
 
@@ -239,7 +264,7 @@ if isunix:
     cxx = os.environ.get('CXX', 'g++')
     cflags = os.environ.get('OVERRIDE_CFLAGS',
         # '-Wall -DNDEBUG -ggdb -fno-strict-aliasing -pipe')
-        '-O3 -Wall -DNDEBUG -fno-strict-aliasing -pipe')
+        '-Wall -DNDEBUG -fno-strict-aliasing -pipe')
     cflags = shlex.split(cflags) + ['-fPIC']
     ldflags = os.environ.get('OVERRIDE_LDFLAGS', '-Wall')
     ldflags = shlex.split(ldflags)
@@ -274,7 +299,7 @@ if isosx:
 
 if iswindows:
     cc = cxx = msvc.cc
-    cflags = '/c /nologo /Ox /MD /W3 /EHsc /DNDEBUG'.split()
+    cflags = '/c /nologo /MD /W3 /EHsc /DNDEBUG'.split()
     ldflags = '/DLL /nologo /INCREMENTAL:NO /NODEFAULTLIB:libcmt.lib'.split()
     #cflags = '/c /nologo /Ox /MD /W3 /EHsc /Zi'.split()
     #ldflags = '/DLL /nologo /INCREMENTAL:NO /DEBUG'.split()

setup/installer/linux/freeze.py

@@ -43,7 +43,6 @@ class LinuxFreeze(Command):
                         '/usr/lib/liblcms2.so.2',
                         '/usr/lib/libstlport.so.5.1',
                         '/tmp/calibre-mount-helper',
-                        '/usr/lib/libunrar.so',
                         '/usr/lib/libchm.so.0',
                         '/usr/lib/libsqlite3.so.0',
                         '/usr/lib/libmng.so.1',

setup/installer/linux/freeze2.py

@@ -32,7 +32,6 @@ binary_includes = [
                 '/usr/lib/liblcms.so.1',
                 '/usr/lib/liblzma.so.0',
                 '/usr/lib/libexpat.so.1',
-                '/usr/lib/libunrar.so',
                 '/usr/lib/libsqlite3.so.0',
                 '/usr/lib/libmng.so.1',
                 '/usr/lib/libpodofo.so.0.9.1',

setup/installer/osx/app/main.py

@@ -437,8 +437,8 @@ class Py2App(object):
 
     @flush
     def add_misc_libraries(self):
-        for x in ('usb-1.0.0', 'mtp.9', 'unrar', 'readline.6.1',
+        for x in ('usb-1.0.0', 'mtp.9', 'readline.6.1', 'wmflite-0.2.7',
-                    'wmflite-0.2.7', 'chm.0', 'sqlite3.0'):
+                  'chm.0', 'sqlite3.0'):
             info('\nAdding', x)
             x = 'lib%s.dylib'%x
             shutil.copy2(join(SW, 'lib', x), self.frameworks_dir)
@@ -597,6 +597,8 @@ class Py2App(object):
             else:
                 os.symlink(join('../..', x),
                            join(cc_dir, x))
+        shutil.copytree(join(SW, 'build/notifier.app'), join(
+            self.contents_dir, 'calibre-notifier.app'))
 
     @flush
     def copy_site(self):

setup/installer/osx/freeze.py

@@ -388,7 +388,7 @@ def main():
                          'dist_dir' : 'build/py2app',
                          'argv_emulation' : True,
                          'iconfile' : icon,
-                         'frameworks': ['libusb.dylib', 'libunrar.dylib'],
+                         'frameworks': ['libusb.dylib'],
                          'includes' : ['sip', 'pkg_resources', 'PyQt4.QtXml',
                                        'PyQt4.QtSvg', 'PyQt4.QtWebKit', 'commands',
                                        'mechanize', 'ClientForm', 'usbobserver',

setup/installer/windows/MemoryModule.h

@@ -1,58 +1,48 @@
-/*
+/*
- * Memory DLL loading code
+ * Memory DLL loading code
- * Version 0.0.2
+ * Version 0.0.3
- *
+ *
- * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
+ * Copyright (c) 2004-2012 by Joachim Bauch / mail@joachim-bauch.de
- * http://www.joachim-bauch.de
+ * http://www.joachim-bauch.de
- *
+ *
- * The contents of this file are subject to the Mozilla Public License Version
+ * The contents of this file are subject to the Mozilla Public License Version
- * 1.1 (the "License"); you may not use this file except in compliance with
+ * 2.0 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * the License. You may obtain a copy of the License at
- * http://www.mozilla.org/MPL/
+ * http://www.mozilla.org/MPL/
- *
+ *
- * Software distributed under the License is distributed on an "AS IS" basis,
+ * Software distributed under the License is distributed on an "AS IS" basis,
- * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
- * for the specific language governing rights and limitations under the
+ * for the specific language governing rights and limitations under the
- * License.
+ * License.
- *
+ *
- * The Original Code is MemoryModule.h
+ * The Original Code is MemoryModule.h
- *
+ *
- * The Initial Developer of the Original Code is Joachim Bauch.
+ * The Initial Developer of the Original Code is Joachim Bauch.
- *
+ *
- * Portions created by Joachim Bauch are Copyright (C) 2004-2005
+ * Portions created by Joachim Bauch are Copyright (C) 2004-2012
- * Joachim Bauch. All Rights Reserved.
+ * Joachim Bauch. All Rights Reserved.
- *
+ *
- */
+ */
-
+
-#ifndef __MEMORY_MODULE_HEADER
+#ifndef __MEMORY_MODULE_HEADER
-#define __MEMORY_MODULE_HEADER
+#define __MEMORY_MODULE_HEADER
-
+
-#include <Windows.h>
+#include <Windows.h>
-
+
-typedef void *HMEMORYMODULE;
+typedef void *HMEMORYMODULE;
-
+
-#ifdef __cplusplus
+#ifdef __cplusplus
-extern "C" {
+extern "C" {
-#endif
+#endif
-
+
-typedef void *(*FINDPROC)();
+HMEMORYMODULE MemoryLoadLibrary(const void *);
-
+
-extern FINDPROC findproc;
+FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *);
-extern void *findproc_data;
+
-
+void MemoryFreeLibrary(HMEMORYMODULE);
-HMEMORYMODULE MemoryLoadLibrary(char *, const void *);
+
-
+#ifdef __cplusplus
-FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *);
+}
-
+#endif
-void MemoryFreeLibrary(HMEMORYMODULE);
+
-
+#endif  // __MEMORY_MODULE_HEADER
-BOOL MyFreeLibrary(HMODULE hModule);
-HMODULE MyLoadLibrary(char *lpFileName);
-FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName);
-HMODULE MyGetModuleHandle(LPCTSTR lpModuleName);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // __MEMORY_MODULE_HEADER

setup/installer/windows/__init__.py

@@ -8,53 +8,84 @@ __docformat__ = 'restructuredtext en'
 
 import os, shutil, subprocess
 
-from setup import Command, __appname__, __version__
+from setup import Command, __appname__, __version__, installer_name
 from setup.installer import VMInstaller
 
 class Win(Command):
 
     description = 'Build windows binary installers'
 
-    sub_commands = ['win32']
+    sub_commands = ['win64', 'win32']
 
     def run(self, opts):
         pass
 
-
+class WinBase(VMInstaller):
-class Win32(VMInstaller):
-
-    description = 'Build 32bit windows binary installer'
-
-    INSTALLER_EXT = 'exe'
-    VM_NAME = 'xp_build'
-    VM = '/vmware/bin/%s'%VM_NAME
-    VM_CHECK = 'calibre_windows_xp_home'
     FREEZE_COMMAND = 'win32_freeze'
     FREEZE_TEMPLATE = 'python -OO setup.py {freeze_command} --no-ice'
     INSTALLER_EXT = 'msi'
     SHUTDOWN_CMD = ['shutdown.exe', '-s', '-f', '-t', '0']
 
+
+class Win32(WinBase):
+
+    description = 'Build 32bit windows binary installer'
+
+    VM_NAME = 'xp_build'
+    VM = '/vmware/bin/%s'%VM_NAME
+    VM_CHECK = 'calibre_windows_xp_home'
+
+    @property
+    def msi64(self):
+        return installer_name('msi', is64bit=True)
+
     def sign_msi(self):
+        import xattr
         print ('Signing installers ...')
+        sign64 = False
+        msi64 = self.msi64
+        if os.path.exists(msi64) and 'user.signed' not in xattr.list(msi64):
+            subprocess.check_call(['scp', msi64, self.VM_NAME +
+                    ':build/%s/%s'%(__appname__, msi64)])
+            sign64 = True
         subprocess.check_call(['ssh', self.VM_NAME, '~/sign.sh'], shell=False)
+        return sign64
+
+    def do_dl(self, installer, errmsg):
+        subprocess.check_call(('scp',
+            '%s:build/%s/%s'%(self.VM_NAME, __appname__, installer), 'dist'))
+        if not os.path.exists(installer):
+            self.warn(errmsg)
+            raise SystemExit(1)
 
     def download_installer(self):
         installer = self.installer()
         if os.path.exists('build/winfrozen'):
             shutil.rmtree('build/winfrozen')
-        self.sign_msi()
+        sign64 = self.sign_msi()
+        if sign64:
+            self.do_dl(self.msi64, 'Failed to d/l signed 64 bit installer')
+            import xattr
+            xattr.set(self.msi64, 'user.signed', 'true')
 
-        subprocess.check_call(('scp',
+        self.do_dl(installer, 'Failed to freeze')
-            'xp_build:build/%s/%s'%(__appname__, installer), 'dist'))
-        if not os.path.exists(installer):
-            self.warn('Failed to freeze')
-            raise SystemExit(1)
 
         installer = 'dist/%s-portable-installer-%s.exe'%(__appname__, __version__)
-        subprocess.check_call(('scp',
+        self.do_dl(installer, 'Failed to get portable installer')
-            'xp_build:build/%s/%s'%(__appname__, installer), 'dist'))
+
-        if not os.path.exists(installer):
+class Win64(WinBase):
-            self.warn('Failed to get portable installer')
+
-            raise SystemExit(1)
+    description = 'Build 64bit windows binary installer'
+
+    VM_NAME = 'win64'
+    VM = '/vmware/bin/%s'%VM_NAME
+    VM_CHECK = 'win64'
+    IS_64_BIT = True
+    BUILD_PREFIX = WinBase.BUILD_PREFIX + [
+        'if [ -f "$HOME/.bash_profile" ] ; then',
+        '    source "$HOME/.bash_profile"',
+        'fi',
+    ]
+
 
 

setup/installer/windows/freeze.py

@@ -10,14 +10,13 @@ import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time, textwra
 
 from setup import (Command, modules, functions, basenames, __version__,
     __appname__)
-from setup.build_environment import msvc, MT, RC
+from setup.build_environment import msvc, MT, RC, is64bit
 from setup.installer.windows.wix import WixMixIn
 
 ICU_DIR = os.environ.get('ICU_DIR', r'Q:\icu')
 OPENSSL_DIR = os.environ.get('OPENSSL_DIR', r'Q:\openssl')
 QT_DIR = os.environ.get('QT_DIR', 'Q:\\Qt\\4.8.2')
 QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
-LIBUNRAR         = os.environ.get('UNRARDLL', 'C:\\Program Files\\UnrarDLL\\unrar.dll')
 SW               = r'C:\cygwin\home\kovid\sw'
 IMAGEMAGICK = os.path.join(SW, 'build',
                             'ImageMagick-*\\VisualMagick\\bin')
@@ -26,6 +25,7 @@ LZMA = r'Q:\easylzma\build\easylzma-0.0.8'
 
 VERSION = re.sub('[a-z]\d+', '', __version__)
 WINVER = VERSION+'.0'
+machine = 'X64' if is64bit else 'X86'
 
 DESCRIPTIONS = {
         'calibre' : 'The main calibre program',
@@ -88,8 +88,9 @@ class Win32Freeze(Command, WixMixIn):
         self.archive_lib_dir()
         self.remove_CRT_from_manifests()
         self.create_installer()
-        self.build_portable()
+        if not is64bit:
-        self.build_portable_installer()
+            self.build_portable()
+            self.build_portable_installer()
 
     def remove_CRT_from_manifests(self):
         '''
@@ -110,7 +111,7 @@ class Win32Freeze(Command, WixMixIn):
             self.info('Removing CRT dependency from manifest of: %s'%bn)
             # Blank out the bytes corresponding to the dependency specification
             nraw = repl_pat.sub(lambda m: b' '*len(m.group()), raw)
-            if len(nraw) != len(raw):
+            if len(nraw) != len(raw) or nraw == raw:
                 raise Exception('Something went wrong with %s'%bn)
             with open(dll, 'wb') as f:
                 f.write(nraw)
@@ -132,6 +133,23 @@ class Win32Freeze(Command, WixMixIn):
             # used instead
             shutil.copy2(f, tgt)
 
+    def fix_pyd_bootstraps_in(self, folder):
+        for dirpath, dirnames, filenames in os.walk(folder):
+            for f in filenames:
+                name, ext = os.path.splitext(f)
+                bpy = self.j(dirpath, name + '.py')
+                if ext == '.pyd' and os.path.exists(bpy):
+                    with open(bpy, 'rb') as f:
+                        raw = f.read().strip()
+                    if (not raw.startswith('def __bootstrap__') or not
+                            raw.endswith('__bootstrap__()')):
+                        raise Exception('The file %r has non'
+                                ' bootstrap code'%self.j(dirpath, f))
+                    for ext in ('.py', '.pyc', '.pyo'):
+                        x = self.j(dirpath, name+ext)
+                        if os.path.exists(x):
+                            os.remove(x)
+
     def freeze(self):
         shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
 
@@ -184,23 +202,12 @@ class Win32Freeze(Command, WixMixIn):
         shutil.copytree(self.j(comext, 'shell'), self.j(sp_dir, 'win32com', 'shell'))
         shutil.rmtree(comext)
 
-        # Fix PyCrypto, removing the bootstrap .py modules that load the .pyd
+        # Fix PyCrypto and Pillow, removing the bootstrap .py modules that load
-        # modules, since they do not work when in a zip file
+        # the .pyd modules, since they do not work when in a zip file
-        for crypto_dir in glob.glob(self.j(sp_dir, 'pycrypto-*', 'Crypto')):
+        for folder in os.listdir(sp_dir):
-            for dirpath, dirnames, filenames in os.walk(crypto_dir):
+            folder = self.j(sp_dir, folder)
-                for f in filenames:
+            if os.path.isdir(folder):
-                    name, ext = os.path.splitext(f)
+                self.fix_pyd_bootstraps_in(folder)
-                    if ext == '.pyd':
-                        with open(self.j(dirpath, name+'.py')) as f:
-                            raw = f.read().strip()
-                        if (not raw.startswith('def __bootstrap__') or not
-                                raw.endswith('__bootstrap__()')):
-                            raise Exception('The PyCrypto file %r has non'
-                                    ' bootstrap code'%self.j(dirpath, f))
-                        for ext in ('.py', '.pyc', '.pyo'):
-                            x = self.j(dirpath, name+ext)
-                            if os.path.exists(x):
-                                os.remove(x)
 
         for pat in (r'PyQt4\uic\port_v3', ):
             x = glob.glob(self.j(self.lib_dir, 'site-packages', pat))[0]
@@ -260,9 +267,6 @@ class Win32Freeze(Command, WixMixIn):
 
         print
         print 'Adding third party dependencies'
-        print '\tAdding unrar'
-        shutil.copyfile(LIBUNRAR, os.path.join(self.dll_dir,
-                        os.path.basename(LIBUNRAR).replace('64', '')))
 
         print '\tAdding misc binary deps'
         bindir = os.path.join(SW, 'bin')
@@ -370,7 +374,7 @@ class Win32Freeze(Command, WixMixIn):
         if not self.opts.keep_site:
             os.remove(y)
 
-    def run_builder(self, cmd):
+    def run_builder(self, cmd, show_output=False):
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE)
         if p.wait() != 0:
@@ -379,6 +383,9 @@ class Win32Freeze(Command, WixMixIn):
             self.info(p.stdout.read())
             self.info(p.stderr.read())
             sys.exit(1)
+        if show_output:
+            self.info(p.stdout.read())
+            self.info(p.stderr.read())
 
     def build_portable_installer(self):
         zf = self.a(self.j('dist', 'calibre-portable-%s.zip.lz'%VERSION))
@@ -404,7 +411,7 @@ class Win32Freeze(Command, WixMixIn):
         exe = self.j('dist', 'calibre-portable-installer-%s.exe'%VERSION)
         if self.newer(exe, [obj, xobj]):
             self.info('Linking', exe)
-            cmd = [msvc.linker] + ['/INCREMENTAL:NO', '/MACHINE:X86',
+            cmd = [msvc.linker] + ['/INCREMENTAL:NO', '/MACHINE:'+machine,
                     '/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:WINDOWS',
                     '/LIBPATH:'+(LZMA+r'\lib\Release'),
                     '/RELEASE', '/MANIFEST', '/MANIFESTUAC:level="asInvoker" uiAccess="false"',
@@ -461,7 +468,7 @@ class Win32Freeze(Command, WixMixIn):
         exe = self.j(base, 'calibre-portable.exe')
         if self.newer(exe, [obj]):
             self.info('Linking', exe)
-            cmd = [msvc.linker] + ['/INCREMENTAL:NO', '/MACHINE:X86',
+            cmd = [msvc.linker] + ['/INCREMENTAL:NO', '/MACHINE:'+machine,
                     '/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:WINDOWS',
                     '/RELEASE',
                     '/ENTRY:wWinMainCRTStartup',
@@ -502,9 +509,11 @@ class Win32Freeze(Command, WixMixIn):
         finally:
             os.chdir(cwd)
 
-    def build_launchers(self):
+    def build_launchers(self, debug=False):
         if not os.path.exists(self.obj_dir):
             os.makedirs(self.obj_dir)
+        dflags = (['/Zi'] if debug else [])
+        dlflags = (['/DEBUG'] if debug else ['/INCREMENTAL:NO'])
         base = self.j(self.src_root, 'setup', 'installer', 'windows')
         sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']]
         headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']]
@@ -513,20 +522,20 @@ class Win32Freeze(Command, WixMixIn):
         cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver]
         for src, obj in zip(sources, objects):
             if not self.newer(obj, headers+[src]): continue
-            cmd = [msvc.cc] + cflags + ['/Fo'+obj, '/Tc'+src]
+            cmd = [msvc.cc] + cflags + dflags + ['/Fo'+obj, '/Tc'+src]
-            self.run_builder(cmd)
+            self.run_builder(cmd, show_output=True)
 
         dll = self.j(self.obj_dir, 'calibre-launcher.dll')
         ver = '.'.join(__version__.split('.')[:2])
         if self.newer(dll, objects):
-            cmd = [msvc.linker, '/DLL', '/INCREMENTAL:NO', '/VERSION:'+ver,
+            cmd = [msvc.linker, '/DLL', '/VERSION:'+ver, '/OUT:'+dll,
-                    '/OUT:'+dll, '/nologo', '/MACHINE:X86'] + objects + \
+                   '/nologo', '/MACHINE:'+machine] + dlflags + objects + \
                 [self.embed_resources(dll),
                 '/LIBPATH:C:/Python%s/libs'%self.py_ver,
                 'python%s.lib'%self.py_ver,
                 '/delayload:python%s.dll'%self.py_ver]
             self.info('Linking calibre-launcher.dll')
-            self.run_builder(cmd)
+            self.run_builder(cmd, show_output=True)
 
         src = self.j(base, 'main.c')
         shutil.copy2(dll, self.base)
@@ -544,16 +553,16 @@ class Win32Freeze(Command, WixMixIn):
                 dest = self.j(self.obj_dir, bname+'.obj')
                 if self.newer(dest, [src]+headers):
                     self.info('Compiling', bname)
-                    cmd = [msvc.cc] + xflags + ['/Tc'+src, '/Fo'+dest]
+                    cmd = [msvc.cc] + xflags + dflags + ['/Tc'+src, '/Fo'+dest]
                     self.run_builder(cmd)
                 exe = self.j(self.base, bname+'.exe')
                 lib = dll.replace('.dll', '.lib')
                 if self.newer(exe, [dest, lib, self.rc_template, __file__]):
                     self.info('Linking', bname)
-                    cmd = [msvc.linker] + ['/INCREMENTAL:NO', '/MACHINE:X86',
+                    cmd = [msvc.linker] + ['/MACHINE:'+machine,
                             '/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:'+subsys,
                             '/LIBPATH:C:/Python%s/libs'%self.py_ver, '/RELEASE',
-                            '/OUT:'+exe, self.embed_resources(exe),
+                            '/OUT:'+exe] + dlflags + [self.embed_resources(exe),
                             dest, lib]
                     self.run_builder(cmd)
 
@@ -566,9 +575,18 @@ class Win32Freeze(Command, WixMixIn):
             for x in (self.plugins_dir, self.dll_dir):
                 for pyd in os.listdir(x):
                     if pyd.endswith('.pyd') and pyd not in {
-                            'sqlite_custom.pyd', 'calibre_style.pyd'}:
                         # sqlite_custom has to be a file for
                         # sqlite_load_extension to work
+                        'sqlite_custom.pyd',
+                        # calibre_style has to be loaded by Qt therefore it
+                        # must be a file
+                        'calibre_style.pyd',
+                        # Because of https://github.com/fancycode/MemoryModule/issues/4
+                        # any extensions that use C++ exceptions must be loaded
+                        # from files
+                        'unrar.pyd', 'wpd.pyd', 'podofo.pyd',
+                        'progress_indicator.pyd',
+                        }:
                         self.add_to_zipfile(zf, pyd, x)
                         os.remove(self.j(x, pyd))
 
@@ -581,7 +599,8 @@ class Win32Freeze(Command, WixMixIn):
             sp = self.j(self.lib_dir, 'site-packages')
             # Special handling for PIL and pywin32
             handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32'])
-            self.add_to_zipfile(zf, 'PIL', sp)
+            if not is64bit:
+                self.add_to_zipfile(zf, 'PIL', sp)
             base = self.j(sp, 'win32', 'lib')
             for x in os.listdir(base):
                 if os.path.splitext(x)[1] not in ('.exe',):
@@ -593,16 +612,17 @@ class Win32Freeze(Command, WixMixIn):
                         self.add_to_zipfile(zf, x, base)
 
             handled.add('easy-install.pth')
+            # We dont want the site.py from site-packages
+            handled.add('site.pyo')
+
             for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')):
                 handled.add(self.b(d))
                 for x in os.listdir(d):
-                    if x == 'EGG-INFO':
+                    if x in {'EGG-INFO', 'site.py', 'site.pyc', 'site.pyo'}:
                         continue
                     self.add_to_zipfile(zf, x, d)
 
             # The rest of site-packages
-            # We dont want the site.py from site-packages
-            handled.add('site.pyo')
             for x in os.listdir(sp):
                 if x in handled or x.endswith('.egg-info'):
                     continue
@@ -622,8 +642,10 @@ class Win32Freeze(Command, WixMixIn):
             line = line.strip()
             if not line or line.startswith('#') or line.startswith('import'):
                 continue
-            candidate = self.j(base, line)
+            candidate = os.path.abspath(self.j(base, line))
             if os.path.exists(candidate):
+                if not os.path.isdir(candidate):
+                    raise ValueError('%s is not a directory'%candidate)
                 yield candidate
 
     def add_to_zipfile(self, zf, name, base, exclude=frozenset()):

setup/installer/windows/notes.rst

@@ -109,10 +109,8 @@ of mimetypes from the windows registry
 Python packages
 ------------------
 
-Install setuptools from http://pypi.python.org/pypi/setuptools If there are no
+Install setuptools from http://pypi.python.org/pypi/setuptools. Use the source
-windows binaries already compiled for the version of python you are using then
+tarball. Edit setup.py and set zip_safe=False. Then run::
-download the source and run the following command in the folder where the
-source has been unpacked::
 
      python setup.py install
 
@@ -240,23 +238,6 @@ Run make (note that you must have GNU make installed in cygwin)
 
 Optionally run make check
 
-Libunrar
-----------
-
-Get the source from http://www.rarlab.com/rar_add.htm
-
-Open UnrarDll.vcproj, change build type to release.
-If building 64 bit change Win32 to x64.
-
-Build the Solution, find the dll in the build subdir. As best as I can tell,
-the vcproj already defines the SILENT preprocessor directive, but you should
-test this.
-
-.. http://www.rarlab.com/rar/UnRARDLL.exe install and add C:\Program Files\UnrarDLL to PATH
-
-TODO: 64-bit check that SILENT is defined and that the ctypes bindings actuall
-work
-
 zlib
 ------
 
@@ -549,6 +530,16 @@ Get it from http://lloyd.github.com/easylzma/ (use the trunk version)
 Run cmake and build the Visual Studio solution (generates CLI tools and dll and
 static lib automatically)
 
+chmlib
+-------
+
+Download the zip source code from: http://www.jedrea.com/chmlib/
+Run::
+    cd src && unzip ./ChmLib-ds6.zip
+Then open ChmLib.dsw in Visual Studio, change the configuration to Release
+(Win32|x64) and build solution, this will generate a static library in
+Release/ChmLib.lib
+
 calibre
 ---------
 

setup/installer/windows/portable-installer.cpp

@@ -418,8 +418,11 @@ static BOOL move_program() {
     }
 
     if (MoveFileEx(L"Calibre Portable\\Calibre", L"..\\Calibre", 0) == 0) {
-        show_last_error(L"Failed to move calibre program folder");
+        Sleep(4000); // Sleep and try again
-        return false;
+        if (MoveFileEx(L"Calibre Portable\\Calibre", L"..\\Calibre", 0) == 0) {
+            show_last_error(L"Failed to move calibre program folder. This is usually caused by an antivirus program or a file sync program like DropBox. Turn them off temporarily and try again. Underlying error: ");
+            return false;
+        }
     }
 
     if (!directory_exists(L"..\\Calibre Library")) {

setup/installer/windows/util.c

@@ -16,6 +16,7 @@ static char python_dll[] = PYDLL;
 void set_gui_app(char yes) { GUI_APP = yes; }
 char is_gui_app() { return GUI_APP; }
 
+int calibre_show_python_error(const wchar_t *preamble, int code);
 
 // memimporter {{{
 
@@ -63,17 +64,6 @@ static void* FindLibrary(char *name, PyObject *callback)
 	return p;
 }
 
-static PyObject *set_find_proc(PyObject *self, PyObject *args)
-{
-	PyObject *callback = NULL;
-	if (!PyArg_ParseTuple(args, "|O:set_find_proc", &callback))
-		return NULL;
-	Py_DECREF((PyObject *)findproc_data);
-	Py_INCREF(callback);
-	findproc_data = (void *)callback;
-    return Py_BuildValue("i", 1);
-}
-
 static PyObject *
 import_module(PyObject *self, PyObject *args)
 {
@@ -92,7 +82,7 @@ import_module(PyObject *self, PyObject *args)
 			      &data, &size,
 			      &initfuncname, &modname, &pathname))
 		return NULL;
-	hmem = MemoryLoadLibrary(pathname, data);
+	hmem = MemoryLoadLibrary(data);
 	if (!hmem) {
 		PyErr_Format(*DLL_ImportError,
 			     "MemoryLoadLibrary() failed loading %s", pathname);
@@ -119,14 +109,14 @@ import_module(PyObject *self, PyObject *args)
 static PyMethodDef methods[] = {
 	{ "import_module", import_module, METH_VARARGS,
 	  "import_module(code, initfunc, dllname[, finder]) -> module" },
-	{ "set_find_proc", set_find_proc, METH_VARARGS },
 	{ NULL, NULL },		/* Sentinel */
 };
 
 // }}}
 
 static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
-    wchar_t *buf, *cbuf;
+    wchar_t *buf;
+    char *cbuf;
     buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)*
             (wcslen(msg) + wcslen(preamble) + 80));
 
@@ -142,7 +132,7 @@ static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int co
     else {
         cbuf = (char*) calloc(10+(wcslen(buf)*4), sizeof(char));
         if (cbuf) {
-            if (WideCharToMultiByte(CP_UTF8, 0, buf, -1, cbuf, 10+(wcslen(buf)*4), NULL, NULL) != 0) printf_s(cbuf);
+            if (WideCharToMultiByte(CP_UTF8, 0, buf, -1, cbuf, (int)(10+(wcslen(buf)*4)), NULL, NULL) != 0) printf_s(cbuf);
             free(cbuf);
         }
     }
@@ -165,6 +155,7 @@ int show_last_error_crt(wchar_t *preamble) {
 int show_last_error(wchar_t *preamble) {
     wchar_t *msg = NULL;
     DWORD dw = GetLastError(); 
+    int ret;
 
     FormatMessage(
         FORMAT_MESSAGE_ALLOCATE_BUFFER | 
@@ -173,10 +164,13 @@ int show_last_error(wchar_t *preamble) {
         NULL,
         dw,
         MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-        &msg,
+        (LPWSTR)&msg, 
-        0, NULL );
+        0,
+        NULL );
 
-    return _show_error(preamble, msg, (int)dw);
+    ret = _show_error(preamble, msg, (int)dw);
+    if (msg != NULL) LocalFree(msg);
+    return ret;
 }
 
 char* get_app_dir() {
@@ -254,10 +248,10 @@ void setup_stream(const char *name, const char *errors, UINT cp) {
     else if (cp == CP_UTF7) _snprintf_s(buf, 100, _TRUNCATE, "%s", "utf-7");
     else _snprintf_s(buf, 100, _TRUNCATE, "cp%d", cp);
 
-    stream = PySys_GetObject(name);
+    stream = PySys_GetObject((char*)name);
 
-    if (!PyFile_SetEncodingAndErrors(stream, buf, errors)) 
+    if (!PyFile_SetEncodingAndErrors(stream, buf, (char*)errors)) 
-        ExitProcess(calibre_show_python_error("Failed to set stream encoding", 1));
+        ExitProcess(calibre_show_python_error(L"Failed to set stream encoding", 1));
 
     free(buf);
     
@@ -372,7 +366,6 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
     }
     PySys_SetObject("argv", argv);
 
-    findproc = FindLibrary;
 	Py_InitModule3("_memimporter", methods, module_doc);
 
 }

setup/installer/windows/wix-template.xml

@@ -2,13 +2,13 @@
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi' xmlns:util="http://schemas.microsoft.com/wix/UtilExtension"
 >		
 
-		<Product Name='{app}' Id='*' UpgradeCode='{upgrade_code}'
+<Product Name='{app}{x64}' Id='*' UpgradeCode='{upgrade_code}'
 				Language='1033' Codepage='1252' Version='{version}' Manufacturer='Kovid Goyal'>
 				
 				<Package Id='*' Keywords='Installer' Description="{app} Installer"
 						Comments='{app} is a registered trademark of Kovid Goyal' Manufacturer='Kovid Goyal'
 						InstallerVersion='300' Languages='1033' Compressed='yes'
-						SummaryCodepage='1252' />
+                        SummaryCodepage='1252' />
 
                 <Media Id="1" Cabinet="{app}.cab" CompressionLevel="{compression}" EmbedCab="yes" />
                 <!-- The following line ensures that DLLs are replaced even if their version is the same as before. This
@@ -29,19 +29,24 @@
                        Language="1033"
                        Property="NEWPRODUCTFOUND"/>
                </Upgrade>
-               <CustomAction Id="PreventDowngrading" Error="Newer version already installed."/>
+               <CustomAction Id="PreventDowngrading" Error="Newer version of {app} already installed. If you want to downgrade you must uninstall {app} first."/>
 
                <Property Id="APPLICATIONFOLDER">
                   <RegistrySearch Id='calibreInstDir' Type='raw'
-                      Root='HKLM' Key="Software\{app}\Installer" Name="InstallPath" />
+                      Root='HKLM' Key="Software\{app}{x64}\Installer" Name="InstallPath" />
                </Property>
 
                 <Directory Id='TARGETDIR' Name='SourceDir'>
-						<Directory Id='ProgramFilesFolder' Name='PFiles'>
+                        <Directory Id='{ProgramFilesFolder}' Name='PFiles'>
-								<Directory Id='APPLICATIONFOLDER' Name='{app}' />
+                            <!-- The name must be calibre on 32 bit to ensure
+                                 that the component guids dont change compared
+                                 to previous msis. However, on 64 bit it must
+                                 be Calibre2 otherwise by default it will
+                                 install to C:\Program Files\calibre -->
+                            <Directory Id='APPLICATIONFOLDER' Name="{appfolder}" />
 						</Directory>
                         <Directory Id="ProgramMenuFolder">
-                            <Directory Id="ApplicationProgramsFolder" Name="{app} - E-book Management"/>
+                            <Directory Id="ApplicationProgramsFolder" Name="{app}{x64} - E-book Management"/>
                         </Directory>
                         <Directory Id="DesktopFolder" Name="Desktop"/>
 				</Directory>
@@ -50,24 +55,24 @@
 						{app_components}
                         <Component Id="AddToPath" Guid="*">
                             <Environment Id='UpdatePath' Name='PATH' Action='set' System='yes' Part='last' Value='[APPLICATIONFOLDER]' />
-                            <RegistryValue Root="HKCU" Key="Software\Microsoft\{app}" Name="system_path_updated" Type="integer" Value="1" KeyPath="yes"/>
+                            <RegistryValue Root="HKCU" Key="Software\Microsoft\{app}{x64}" Name="system_path_updated" Type="integer" Value="1" KeyPath="yes"/>
                         </Component>
                         <Component Id="RememberInstallDir" Guid="*">
-                            <RegistryValue Root="HKLM" Key="Software\{app}\Installer" Name="InstallPath" Type="string" Value="[APPLICATIONFOLDER]" KeyPath="yes"/>
+                            <RegistryValue Root="HKLM" Key="Software\{app}{x64}\Installer" Name="InstallPath" Type="string" Value="[APPLICATIONFOLDER]" KeyPath="yes"/>
                         </Component>
                 </DirectoryRef>
 
                 <DirectoryRef Id="ApplicationProgramsFolder">
                     <Component Id="StartMenuShortcuts" Guid="*">
-                        <Shortcut Id="s1" Name="{app} - E-book management"
+                        <Shortcut Id="s1" Name="{app}{x64} - E-book management"
                             Description="Manage your e-book collection and download news"
                             Target="[#{exe_map[calibre]}]"
                              WorkingDirectory="APPLICATIONROOTDIRECTORY" />
-                        <Shortcut Id="s2" Name="E-book viewer"
+                         <Shortcut Id="s2" Name="E-book viewer{x64}"
                             Description="Viewer for all the major e-book formats"
                             Target="[#{exe_map[ebook-viewer]}]"
                              WorkingDirectory="APPLICATIONROOTDIRECTORY" />
-                        <Shortcut Id="s3" Name="LRF viewer"
+                         <Shortcut Id="s3" Name="LRF viewer{x64}"
                             Description="Viewer for LRF format e-books"
                             Target="[#{exe_map[lrfviewer]}]"
                              WorkingDirectory="APPLICATIONROOTDIRECTORY" />
@@ -79,17 +84,17 @@
                             Target="http://calibre-ebook.com/get-involved"/>
 
                         <RemoveFolder Id="ApplicationProgramsFolder" On="uninstall"/>
-                        <RegistryValue Root="HKCU" Key="Software\Microsoft\{app}" Name="start_menu_shortcuts_installed" Type="integer" Value="1" KeyPath="yes"/>
+                        <RegistryValue Root="HKCU" Key="Software\Microsoft\{app}{x64}" Name="start_menu_shortcuts_installed" Type="integer" Value="1" KeyPath="yes"/>
                     </Component>
                 </DirectoryRef>
 
                 <DirectoryRef Id="DesktopFolder">
                     <Component Id="DesktopShortcut" Guid="*">
-                        <Shortcut Id="ds1" Name="{app} - E-book management"
+                        <Shortcut Id="ds1" Name="{app}{x64} - E-book management"
                             Description="Manage your e-book collection and download news"
                             Target="[#{exe_map[calibre]}]"
                             WorkingDirectory="APPLICATIONROOTDIRECTORY" />
-                        <RegistryValue Root="HKCU" Key="Software\Microsoft\{app}" Name="desktop_shortcut_installed" Type="integer" Value="1" KeyPath="yes"/>
+                        <RegistryValue Root="HKCU" Key="Software\Microsoft\{app}{x64}" Name="desktop_shortcut_installed" Type="integer" Value="1" KeyPath="yes"/>
                     </Component> 
                 </DirectoryRef>
          
@@ -122,17 +127,35 @@
 				<!-- Add icon to entry in Add/Remove programs -->
                 <Icon Id="main_icon" SourceFile="{main_icon}"/>
 				<Property Id="ARPPRODUCTICON" Value="main_icon" />
+                <Property Id="ARPURLINFOABOUT" Value="http://calibre-ebook.com" />
+                <Property Id='ARPHELPLINK' Value="http://calibre-ebook.com/help" />
+                <Property Id='ARPURLUPDATEINFO' Value="http://calibre-ebook.com/download_windows" />
+                <SetProperty Id="ARPINSTALLLOCATION" Value="[APPLICATIONFOLDER]" After="CostFinalize" />
 
                 <Condition
-                    Message="This application is only supported on Windows XP SP3, or higher.">
+                    Message="This application is only supported on {minverhuman}, or higher.">
-                    <![CDATA[Installed OR (VersionNT >= 501)]]>
+                    <![CDATA[Installed OR (VersionNT >= {minver})]]>
                 </Condition>
+                <!-- On 64 bit installers there is a bug in WiX that causes the
+                     WixSetDefaultPerMachineFolder action to incorrectly set
+                     APPLICATIONFOLDER to the x86 value, so we override it. See
+                     http://stackoverflow.com/questions/5479790/wix-how-to-override-c-program-files-x86-on-x64-machine-in-wixui-advanced-s
+                     -->
+                <CustomAction
+                    Id="OverwriteWixSetDefaultPerMachineFolder"
+                    Property="WixPerMachineFolder"
+                    Value="[APPLICATIONFOLDER]"
+                    Execute="immediate"
+                    />
+
                 <InstallExecuteSequence>
                     <Custom Action="PreventDowngrading" After="FindRelatedProducts">NEWPRODUCTFOUND</Custom>
+                    {fix_wix}
                     <RemoveExistingProducts After="InstallFinalize" />
                 </InstallExecuteSequence>
                 <InstallUISequence>
                     <Custom Action="PreventDowngrading" After="FindRelatedProducts">NEWPRODUCTFOUND</Custom>
+                    {fix_wix}
                 </InstallUISequence>
 
                 <UI>

setup/installer/windows/wix.py

@@ -6,11 +6,20 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, shutil, subprocess
+import os, shutil, subprocess, sys
 
 from setup import __appname__, __version__, basenames
+from setup.build_environment import is64bit
+
+if is64bit:
+    WIXP = r'C:\Program Files (x86)\WiX Toolset v3.6'
+    UPGRADE_CODE = '5DD881FF-756B-4097-9D82-8C0F11D521EA'
+    MINVERHUMAN = 'Windows Vista'
+else:
+    WIXP = r'C:\Program Files\WiX Toolset v3.6'
+    UPGRADE_CODE = 'BEB2A80D-E902-4DAD-ADF9-8BD2DA42CFE1'
+    MINVERHUMAN = 'Windows XP SP3'
 
-WIXP = r'C:\Program Files\Windows Installer XML v3.5'
 CANDLE = WIXP+r'\bin\candle.exe'
 LIGHT = WIXP+r'\bin\light.exe'
 
@@ -27,15 +36,21 @@ class WixMixIn:
 
         components = self.get_components_from_files()
         wxs = template.format(
-                app               = __appname__,
+            app                = __appname__,
-                version           = __version__,
+            appfolder          = 'Calibre2' if is64bit else __appname__,
-                upgrade_code      = 'BEB2A80D-E902-4DAD-ADF9-8BD2DA42CFE1',
+            version            = __version__,
-                compression       = self.opts.msi_compression,
+            upgrade_code       = UPGRADE_CODE,
-                app_components    = components,
+            ProgramFilesFolder = 'ProgramFiles64Folder' if is64bit else 'ProgramFilesFolder',
-                exe_map           = self.smap,
+            x64                = ' 64bit' if is64bit else '',
-                main_icon         = self.j(self.src_root, 'icons', 'library.ico'),
+            minverhuman        = MINVERHUMAN,
-                web_icon          = self.j(self.src_root, 'icons', 'web.ico'),
+            minver             = '600' if is64bit else '501',
-                )
+            fix_wix = '<Custom Action="OverwriteWixSetDefaultPerMachineFolder" After="WixSetDefaultPerMachineFolder" />' if is64bit else '',
+            compression        = self.opts.msi_compression,
+            app_components     = components,
+            exe_map            = self.smap,
+            main_icon          = self.j(self.src_root, 'icons', 'library.ico'),
+            web_icon           = self.j(self.src_root, 'icons', 'web.ico'),
+        )
         template = open(self.j(self.d(__file__), 'en-us.xml'),
                 'rb').read()
         enus = template.format(app=__appname__)
@@ -48,14 +63,15 @@ class WixMixIn:
         with open(enusf, 'wb') as f:
             f.write(enus)
         wixobj = self.j(self.installer_dir, __appname__+'.wixobj')
-        cmd = [CANDLE, '-nologo', '-ext', 'WiXUtilExtension', '-o', wixobj, wxsf]
+        arch = 'x64' if is64bit else 'x86'
+        cmd = [CANDLE, '-nologo', '-arch', arch, '-ext', 'WiXUtilExtension', '-o', wixobj, wxsf]
         self.info(*cmd)
-        subprocess.check_call(cmd)
+        self.run_wix(cmd)
         self.installer = self.j(self.src_root, 'dist')
         if not os.path.exists(self.installer):
             os.makedirs(self.installer)
-        self.installer = self.j(self.installer, '%s-%s.msi' % (__appname__,
+        self.installer = self.j(self.installer, '%s%s-%s.msi' % (__appname__,
-            __version__))
+            ('-64bit' if is64bit else ''), __version__))
         license = self.j(self.src_root, 'LICENSE.rtf')
         banner  = self.j(self.src_root, 'icons', 'wix-banner.bmp')
         dialog  = self.j(self.src_root, 'icons', 'wix-dialog.bmp')
@@ -66,13 +82,27 @@ class WixMixIn:
                 '-dWixUILicenseRtf='+license,
                 '-dWixUIBannerBmp='+banner,
                 '-dWixUIDialogBmp='+dialog]
-        cmd.append('-sice:ICE60') # No language in dlls warning
+        cmd.extend([
+            '-sice:ICE60',# No language in dlls warning
+            '-sice:ICE61',# Allow upgrading with same version number
+            '-sice:ICE40', # Re-install mode overriden
+            '-sice:ICE69', # Shortcut components are part of a different feature than the files they point to
+        ])
         if self.opts.no_ice:
             cmd.append('-sval')
         if self.opts.verbose:
             cmd.append('-v')
         self.info(*cmd)
-        subprocess.check_call(cmd)
+        self.run_wix(cmd)
+
+    def run_wix(self, cmd):
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+        ret = p.wait()
+        self.info(p.stdout.read())
+        self.info(p.stderr.read())
+        if ret != 0:
+            sys.exit(1)
 
     def get_components_from_files(self):
 
@@ -103,7 +133,20 @@ class WixMixIn:
                         (fid, f, x, checksum),
                     '</Component>'
                     ]
-                    components.append(''.join(c))
+                    if x.endswith('.exe') and not x.startswith('pdf'):
+                        # Add the executable to app paths so that users can
+                        # launch it from the run dialog even if it is not on
+                        # the path. See http://msdn.microsoft.com/en-us/library/windows/desktop/ee872121(v=vs.85).aspx
+                        c[-1:-1] = [
+                        ('<RegistryValue Root="HKLM" '
+                         r'Key="SOFTWARE\Microsoft\Windows\CurrentVersion\App '
+                         r'Paths\%s" Value="[#file_%d]" Type="string" />'%(x, fid)),
+                        ('<RegistryValue Root="HKLM" '
+                         r'Key="SOFTWARE\Microsoft\Windows\CurrentVersion\App '
+                         r'Paths\{0}" Name="Path" Value="[APPLICATIONFOLDER]" '
+                         'Type="string" />'.format(x)),
+                        ]
+                    components.append('\n'.join(c))
             return components
 
         components = process_dir(os.path.abspath(self.base))
@@ -114,4 +157,3 @@ class WixMixIn:
         return '\t\t\t\t'+'\n\t\t\t\t'.join(components)
 
 
-

setup/iso_639/de.po

@@ -18,14 +18,14 @@ msgstr ""
 "Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
 "devel@lists.alioth.debian.org>\n"
 "POT-Creation-Date: 2011-11-25 14:01+0000\n"
-"PO-Revision-Date: 2012-09-04 18:42+0000\n"
+"PO-Revision-Date: 2012-11-08 15:28+0000\n"
-"Last-Translator: SimonFS <simonschuette@arcor.de>\n"
+"Last-Translator: Elmux <bla.mail@gmx.net>\n"
 "Language-Team: German <debian-l10n-german@lists.debian.org>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2012-09-05 04:37+0000\n"
+"X-Launchpad-Export-Date: 2012-11-09 04:39+0000\n"
-"X-Generator: Launchpad (build 15901)\n"
+"X-Generator: Launchpad (build 16250)\n"
 "Language: de\n"
 
 #. name for aaa
@@ -58,7 +58,7 @@ msgstr "Ambrak"
 
 #. name for aah
 msgid "Arapesh; Abu'"
-msgstr ""
+msgstr "Arapesh;Abu' (Papua-Neuguinea)"
 
 #. name for aai
 msgid "Arifama-Miniafia"
@@ -102,7 +102,7 @@ msgstr "Aasáx"
 
 #. name for aat
 msgid "Albanian; Arvanitika"
-msgstr ""
+msgstr "Albanisch, Arvanitikanisch"
 
 #. name for aau
 msgid "Abau"

setup/upload.py

@@ -29,6 +29,7 @@ STAGING_DIR = '/root/staging'
 def installers():
     installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2')))
     installers.append(installer_name('tar.bz2', is64bit=True))
+    installers.append(installer_name('msi', is64bit=True))
     installers.insert(0, 'dist/%s-%s.tar.xz'%(__appname__, __version__))
     installers.append('dist/%s-portable-installer-%s.exe'%(__appname__, __version__))
     return installers
@@ -40,7 +41,7 @@ def installer_description(fname):
         bits = '32' if 'i686' in fname else '64'
         return bits + 'bit Linux binary'
     if fname.endswith('.msi'):
-        return 'Windows installer'
+        return 'Windows %sinstaller'%('64bit ' if '64bit' in fname else '')
     if fname.endswith('.dmg'):
         return 'OS X dmg'
     if fname.endswith('.exe'):

src/calibre/__init__.py

@@ -264,7 +264,7 @@ def extract(path, dir):
     with open(path, 'rb') as f:
         id_ = f.read(3)
     if id_ == b'Rar':
-        from calibre.libunrar import extract as rarextract
+        from calibre.utils.unrar import extract as rarextract
         extractor = rarextract
     elif id_.startswith(b'PK'):
         from calibre.libunzip import extract as zipextract
@@ -276,7 +276,7 @@ def extract(path, dir):
             from calibre.libunzip import extract as zipextract
             extractor = zipextract
         elif ext in ['cbr', 'rar']:
-            from calibre.libunrar import extract as rarextract
+            from calibre.utils.unrar import extract as rarextract
             extractor = rarextract
     if extractor is None:
         raise Exception('Unknown archive type')

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 9, 6)
+numeric_version = (0, 9, 8)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 
@@ -28,6 +28,7 @@ isunix = isosx or islinux
 isportable = os.environ.get('CALIBRE_PORTABLE_BUILD', None) is not None
 ispy3 = sys.version_info.major > 2
 isxp = iswindows and sys.getwindowsversion().major < 6
+is64bit = sys.maxsize > (1 << 32)
 isworker = os.environ.has_key('CALIBRE_WORKER') or os.environ.has_key('CALIBRE_SIMPLE_WORKER')
 if isworker:
     os.environ.pop('CALIBRE_FORCE_ANSI', None)
@@ -43,6 +44,19 @@ winerror   = importlib.import_module('winerror') if iswindows else None
 win32api   = importlib.import_module('win32api') if iswindows else None
 fcntl      = None if iswindows else importlib.import_module('fcntl')
 
+_osx_ver = None
+def get_osx_version():
+    global _osx_ver
+    if _osx_ver is None:
+        import platform
+        from collections import namedtuple
+        OSX = namedtuple('OSX', 'major minor tertiary')
+        try:
+            _osx_ver = OSX(*(map(int, platform.mac_ver()[0].split('.'))))
+        except:
+            _osx_ver = OSX(0, 0, 0)
+    return _osx_ver
+
 filesystem_encoding = sys.getfilesystemencoding()
 if filesystem_encoding is None: filesystem_encoding = 'utf-8'
 else:
@@ -85,6 +99,7 @@ class Plugins(collections.Mapping):
                 'speedup',
                 'freetype',
                 'woff',
+                'unrar',
             ]
         if iswindows:
             plugins.extend(['winutil', 'wpd', 'winfonts'])
@@ -171,6 +186,9 @@ def get_version():
     v = __version__
     if getattr(sys, 'frozen', False) and dv and os.path.abspath(dv) in sys.path:
         v += '*'
+    if iswindows and is64bit:
+        v += ' [64bit]'
+
     return v
 
 def get_portable_base():

src/calibre/customize/builtins.py

@@ -8,7 +8,7 @@ from calibre import guess_type
 from calibre.customize import (FileTypePlugin, MetadataReaderPlugin,
     MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase)
 from calibre.constants import numeric_version
-from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata
+from calibre.ebooks.metadata.archive import ArchiveExtract, get_comic_metadata
 from calibre.ebooks.html.to_zip import HTML2ZIP
 
 plugins = []
@@ -140,7 +140,7 @@ class ComicMetadataReader(MetadataReaderPlugin):
             elif id_.startswith(b'PK'):
                 ftype = 'cbz'
         if ftype == 'cbr':
-            from calibre.libunrar import extract_first_alphabetically as extract_first
+            from calibre.utils.unrar import extract_first_alphabetically as extract_first
             extract_first
         else:
             from calibre.libunzip import extract_member
@@ -150,9 +150,9 @@ class ComicMetadataReader(MetadataReaderPlugin):
         ret = extract_first(stream)
         mi = MetaInformation(None, None)
         stream.seek(0)
-        if ftype == 'cbz':
+        if ftype in {'cbr', 'cbz'}:
             try:
-                mi.smart_update(get_cbz_metadata(stream))
+                mi.smart_update(get_comic_metadata(stream, ftype))
             except:
                 pass
         if ret is not None:

src/calibre/debug.py

@@ -15,7 +15,13 @@ def option_parser():
     parser = OptionParser(usage='''\
 %prog [options]
 
-Run an embedded python interpreter.
+Various command line interfaces useful for debugging calibre. With no options,
+this command starts an embedded python interpreter. You can also run the main
+calibre GUI and the calibre viewer in debug mode.
+
+It also contains interfaces to various bits of calibre that do not have
+dedicated command line tools, such as font subsetting, tweaking ebooks and so
+on.
 ''')
     parser.add_option('-c', '--command', help='Run python code.', default=None)
     parser.add_option('-e', '--exec-file', default=None, help='Run the python code in file.')
@@ -141,8 +147,10 @@ def print_basic_debug_info(out=None):
     if out is None: out = sys.stdout
     out = functools.partial(prints, file=out)
     import platform
-    from calibre.constants import __appname__, get_version, isportable, isosx
+    from calibre.constants import (__appname__, get_version, isportable, isosx,
-    out(__appname__, get_version(), 'Portable' if isportable else '')
+                                   isfrozen)
+    out(__appname__, get_version(), 'Portable' if isportable else '',
+        'isfrozen:', isfrozen)
     out(platform.platform(), platform.system())
     out(platform.system_alias(platform.system(), platform.release(),
             platform.version()))

src/calibre/devices/__init__.py

@@ -182,7 +182,7 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None,
                     out(ioreg)
 
         if hasattr(buf, 'getvalue'):
-            return buf.getvalue().decode('utf-8')
+            return buf.getvalue().decode('utf-8', 'replace')
     finally:
         sys.stdout = oldo
         sys.stderr = olde

src/calibre/devices/android/driver.py

@@ -232,7 +232,7 @@ class ANDROID(USBMS):
             'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE',
             'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID',
             'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E',
-            'NOVO7', 'MB526']
+            'NOVO7', 'MB526', '_USB#WYK7MSF8KE']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',

src/calibre/devices/apple/driver.py

@@ -227,7 +227,7 @@ class ITUNES(DriverBase):
     #  0x1297   iPhone 4
     #  0x129a   iPad
     #  0x129f   iPad2 (WiFi)
-    #  0x12a0   iPhone 4S
+    #  0x12a0   iPhone 4S (GSM)
     #  0x12a2   iPad2 (GSM)
     #  0x12a3   iPad2 (CDMA)
     #  0x12a6   iPad3 (GSM)
@@ -1196,10 +1196,25 @@ class ITUNES(DriverBase):
                         logger().error("  Device|Books playlist not found")
 
                 # Add the passed book to the Device|Books playlist
-                added = pl.add(appscript.mactypes.File(fpath),to=pl)
+                attempts = 2
-                if False:
+                delay = 1.0
-                    logger().info("  '%s' added to Device|Books" % metadata.title)
+                while attempts:
-
+                    try:
+                        added = pl.add(appscript.mactypes.File(fpath),to=pl)
+                        if False:
+                            logger().info("  '%s' added to Device|Books" % metadata.title)
+                        break
+                    except:
+                        attempts -= 1
+                        if DEBUG:
+                            logger().warning("  failed to add book, waiting %.1f seconds to try again (attempt #%d)" %
+                                     (delay, (3 - attempts)))
+                        time.sleep(delay)
+                else:
+                    if DEBUG:
+                        logger().error(" failed to add '%s' to Device|Books" % metadata.title)
+                    raise UserFeedback("Unable to add '%s' in direct connect mode" % metadata.title,
+                                        details=None, level=UserFeedback.ERROR)
                 self._wait_for_writable_metadata(added)
                 return added
 

src/calibre/devices/kobo/books.py

@@ -60,6 +60,8 @@ class Book(Book_):
         self.contentID          = None
         self.current_shelves    = []
         self.kobo_collections   = []
+        self.kobo_series        = None
+        self.kobo_series_number = None
 
         if thumbnail_name is not None:
             self.thumbnail = ImageWrapper(thumbnail_name)

src/calibre/devices/kobo/driver.py

@@ -33,7 +33,7 @@ class KOBO(USBMS):
     gui_name = 'Kobo Reader'
     description = _('Communicate with the Kobo Reader')
     author = 'Timothy Legge and David Forrester'
-    version = (2, 0, 3)
+    version = (2, 0, 4)
 
     dbversion = 0
     fwversion = 0
@@ -59,7 +59,7 @@ class KOBO(USBMS):
     SUPPORTS_SUB_DIRS = True
     SUPPORTS_ANNOTATIONS = True
 
-    # "kepubs" do not have an extension. The name looks like a GUID. Using an empty string seems to work. 
+    # "kepubs" do not have an extension. The name looks like a GUID. Using an empty string seems to work.
     VIRTUAL_BOOK_EXTENSIONS = frozenset(['kobo', ''])
 
     EXTRA_CUSTOMIZATION_MESSAGE = [
@@ -1201,8 +1201,9 @@ class KOBOTOUCH(KOBO):
     author = 'David Forrester'
     description = 'Communicate with the Kobo Touch, Glo and Mini firmware. Based on the existing Kobo driver by %s.' % (KOBO.author)
 
-    supported_dbversion = 70
+    supported_dbversion     = 75
     min_supported_dbversion = 53
+    min_dbversion_series    = 65
 
     booklist_class = KTCollectionsBookList
     book_class = Book
@@ -1236,16 +1237,21 @@ class KOBOTOUCH(KOBO):
                 ' by default they are no longer displayed as there is no good reason to '
                 'see them.  Enable if you wish to see/delete them.'),
             _('Show Recommendations') +
-            ':::'+_('Kobo now shows recommendations on the device.  In some case these have '
+            ':::'+_('Kobo shows recommendations on the device.  In some cases these have '
                 'files but in other cases they are just pointers to the web site to buy. '
                 'Enable if you wish to see/delete them.'),
+            _('Set Series information') +
+            ':::'+_('The book lists on the Kobo devices can display series information. '
+                    'This is not read by the device from the sideloaded books. '
+                    'Series information can only be added to the device after the book has been processed by the device. '
+                    'Enable if you wish to set series information.'),
             _('Attempt to support newer firmware') +
             ':::'+_('Kobo routinely updates the firmware and the '
                 'database version.  With this option Calibre will attempt '
                 'to perform full read-write functionality - Here be Dragons!! '
                 'Enable only if you are comfortable with restoring your kobo '
                 'to factory defaults and testing software. '
-                'This driver supports firmware V2.0.x and DBVersion up to ' + unicode(supported_dbversion)),
+                'This driver supports firmware V2.x.x and DBVersion up to ' + unicode(supported_dbversion)),
             _('Title to test when debugging') +
             ':::'+_('Part of title of a book that can be used when doing some tests for debugging. '
                     'The test is to see if the string is contained in the title of a book. '
@@ -1263,6 +1269,7 @@ class KOBOTOUCH(KOBO):
             False,
             False,
             False,
+            False,
             u''
             ]
 
@@ -1275,8 +1282,9 @@ class KOBOTOUCH(KOBO):
     OPT_SHOW_EXPIRED_BOOK_RECORDS   = 6
     OPT_SHOW_PREVIEWS               = 7
     OPT_SHOW_RECOMMENDATIONS        = 8
-    OPT_SUPPORT_NEWER_FIRMWARE      = 9
+    OPT_UPDATE_SERIES_DETAILS       = 9
-    OPT_DEBUGGING_TITLE             = 10
+    OPT_SUPPORT_NEWER_FIRMWARE      = 10
+    OPT_DEBUGGING_TITLE             = 11
 
     opts = None
 
@@ -1291,7 +1299,7 @@ class KOBOTOUCH(KOBO):
                           ' - N3_LIBRARY_GRID.parsed':[(149,198),0, 99,],   # Used for library lists
                           ' - N3_LIBRARY_LIST.parsed':[(60,90),0, 53,],
 #                          ' - N3_LIBRARY_SHELF.parsed': [(40,60),0, 52,],
-                          ' - N3_FULL.parsed':[(600,800),0, 99,],           # Used for screensaver, home screen 
+                          ' - N3_FULL.parsed':[(600,800),0, 99,],           # Used for screensaver, home screen
                           }
     #Following are the sizes used with pre2.1.4 firmware
 #    COVER_FILE_ENDINGS = {
@@ -1300,7 +1308,7 @@ class KOBOTOUCH(KOBO):
 #                          ' - N3_LIBRARY_GRID.parsed':[(149,233),0, 99,],   # Used for library lists
 #                          ' - N3_LIBRARY_LIST.parsed':[(60,90),0, 53,],
 #                          ' - N3_LIBRARY_SHELF.parsed': [(40,60),0, 52,],
-#                          ' - N3_FULL.parsed':[(600,800),0, 99,],           # Used for screensaver if "Full screen" is checked. 
+#                          ' - N3_FULL.parsed':[(600,800),0, 99,],           # Used for screensaver if "Full screen" is checked.
 #                          }
 
     def initialize(self):
@@ -1364,7 +1372,7 @@ class KOBOTOUCH(KOBO):
         for idx,b in enumerate(bl):
             bl_cache[b.lpath] = idx
 
-        def update_booklist(prefix, path, title, authors, mime, date, ContentID, ContentType, ImageID, readstatus, MimeType, expired, favouritesindex, accessibility, isdownloaded, bookshelves):
+        def update_booklist(prefix, path, title, authors, mime, date, ContentID, ContentType, ImageID, readstatus, MimeType, expired, favouritesindex, accessibility, isdownloaded, series, seriesnumber, bookshelves):
             show_debug = self.is_debugging_title(title)
 #            show_debug = authors == 'L. Frank Baum'
             if show_debug:
@@ -1416,7 +1424,7 @@ class KOBOTOUCH(KOBO):
                         playlist_map[lpath].append('Preview')
                 elif accessibility == 4:
                     playlist_map[lpath].append('Recommendation')
-                
+
                 kobo_collections = playlist_map[lpath][:]
 
                 if len(bookshelves) > 0:
@@ -1442,9 +1450,7 @@ class KOBOTOUCH(KOBO):
                         debug_print('KoboTouch:update_booklist - the authors=', bl[idx].authors)
                         debug_print('KoboTouch:update_booklist - application_id=', bl[idx].application_id)
                     bl_cache[lpath] = None
-                  # removed to allow recognizing of ePub with an UUID inside
+
-                  # if bl[idx].title_sort is not None:
-                  #     bl[idx].title = bl[idx].title_sort
                     if ImageID is not None:
                         imagename = self.imagefilename_from_imageID(ImageID)
                         if imagename is not None:
@@ -1460,7 +1466,9 @@ class KOBOTOUCH(KOBO):
 
                     if show_debug:
                         debug_print("KoboTouch:update_booklist - ContentID='%s'"%ContentID)
-                    bl[idx].contentID = ContentID
+                    bl[idx].contentID           = ContentID
+                    bl[idx].kobo_series         = series
+                    bl[idx].kobo_series_number  = seriesnumber
 
                     if lpath in playlist_map:
                         bl[idx].device_collections  = playlist_map.get(lpath,[])
@@ -1505,9 +1513,11 @@ class KOBOTOUCH(KOBO):
 
                     # print 'Update booklist'
                     book.device_collections  = playlist_map.get(lpath,[])# if lpath in playlist_map else []
-                    book.current_shelves = bookshelves
+                    book.current_shelves     = bookshelves
-                    book.kobo_collections = kobo_collections
+                    book.kobo_collections    = kobo_collections
-                    book.contentID = ContentID
+                    book.contentID           = ContentID
+                    book.kobo_series         = series
+                    book.kobo_series_number  = seriesnumber
 #                    debug_print('KoboTouch:update_booklist - title=', title, 'book.device_collections', book.device_collections)
 
                     if bl.add_book(book, replace_metadata=False):
@@ -1558,10 +1568,23 @@ class KOBOTOUCH(KOBO):
             debug_print("KoboTouch:books - shelf list:", self.bookshelvelist)
 
             opts = self.settings()
-            if self.dbversion >= 33:
+            if self.supports_series():
                 query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
-                    'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, IsDownloaded from content where ' \
+                    'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, ' \
-                    'BookID is Null %(previews)s %(recomendations)s and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % \
+                    'IsDownloaded, Series, SeriesNumber ' \
+                    ' from content ' \
+                    ' where BookID is Null %(previews)s %(recomendations)s and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % \
+                        dict(\
+                             expiry=' and ContentType = 6)' if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')', \
+                             previews=' and Accessibility <> 6' if opts.extra_customization[self.OPT_SHOW_PREVIEWS] == False else '', \
+                             recomendations=' and IsDownloaded in (\'true\', 1)' if opts.extra_customization[self.OPT_SHOW_RECOMMENDATIONS] == False else ''\
+                             )
+            elif self.dbversion >= 33:
+                query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
+                    'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, ' \
+                    'IsDownloaded, null as Series, null as SeriesNumber' \
+                    ' from content ' \
+                    ' where BookID is Null %(previews)s %(recomendations)s and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % \
                         dict(\
                              expiry=' and ContentType = 6)' if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')', \
                              previews=' and Accessibility <> 6' if opts.extra_customization[self.OPT_SHOW_PREVIEWS] == False else '', \
@@ -1569,35 +1592,33 @@ class KOBOTOUCH(KOBO):
                              )
             elif self.dbversion >= 16 and self.dbversion < 33:
                 query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
-                    'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, "1" as IsDownloaded from content where ' \
+                    'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility, ' \
-                    'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \
+                    '"1" as IsDownloaded, null as Series, null as SeriesNumber' \
-                    if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')')
+                    ' from content where ' \
-            elif self.dbversion < 16 and self.dbversion >= 14:
-                query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
-                    'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded from content where ' \
-                    'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \
-                    if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')')
-            elif self.dbversion < 14 and self.dbversion >= 8:
-                query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
-                    'ImageID, ReadStatus, ___ExpirationStatus, "-1" as FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded from content where ' \
                     'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \
                     if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')')
             else:
                 query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
-                    'ImageID, ReadStatus, "-1" as ___ExpirationStatus, "-1" as FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded from content where BookID is Null'
+                    'ImageID, ReadStatus, "-1" as ___ExpirationStatus, "-1" as FavouritesIndex, "-1" as Accessibility, ' \
+                    '"1" as IsDownloaded, null as Series, null as SeriesNumber' \
+                    ' from content where BookID is Null'
 
             debug_print("KoboTouch:books - query=", query)
             try:
                 cursor.execute (query)
             except Exception as e:
                 err = str(e)
-                if not ('___ExpirationStatus' in err or 'FavouritesIndex' in err or
+                if not ('___ExpirationStatus' in err
-                        'Accessibility' in err or 'IsDownloaded' in err):
+                        or 'FavouritesIndex' in err
+                        or 'Accessibility' in err
+                        or 'IsDownloaded' in err
+                        or 'Series' in err
+                        ):
                     raise
                 query= ('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, '
                     'ImageID, ReadStatus, "-1" as ___ExpirationStatus, "-1" as '
-                    'FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded from content where '
+                    'FavouritesIndex, "-1" as Accessibility, "1" as IsDownloaded, null as Series, null as SeriesNumber' \
-                    'BookID is Null')
+                    ' from content where BookID is Null')
                 cursor.execute(query)
 
             changed = False
@@ -1620,10 +1641,10 @@ class KOBOTOUCH(KOBO):
                 bookshelves = get_bookshelvesforbook(connection, row[3])
 
                 if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"):
-                    changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[3], row[5], row[6], row[7], row[4], row[8], row[9], row[10], row[11], bookshelves)
+                    changed = update_booklist(self._main_prefix,   path, row[0], row[1], mime, row[2], row[3], row[5], row[6], row[7], row[4], row[8], row[9], row[10], row[11], row[12], row[13], bookshelves)
                     # print "shortbook: " + path
                 elif oncard == 'carda' and row[3].startswith("file:///mnt/sd/"):
-                    changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[3], row[5], row[6], row[7], row[4], row[8], row[9], row[10], row[11], bookshelves)
+                    changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[3], row[5], row[6], row[7], row[4], row[8], row[9], row[10], row[11], row[12], row[13], bookshelves)
 
                 if changed:
                     need_sync = True
@@ -1669,9 +1690,6 @@ class KOBOTOUCH(KOBO):
         for ending, cover_options in self.COVER_FILE_ENDINGS.items():
             fpath = self._main_prefix + '.kobo/images/' + ImageID + ending
             fpath = self.normalize_path(fpath.replace('/', os.sep))
-            if show_debug:
-                debug_print("KoboTouch:imagefilename_from_imageID - ending=%s, path length=%d" % (ending, len(fpath)))
-                debug_print("KoboTouch:imagefilename_from_imageID - fpath=%s" % (fpath))
             if os.path.exists(fpath):
                 if show_debug:
                     debug_print("KoboTouch:imagefilename_from_imageID - have cover image fpath=%s" % (fpath))
@@ -1857,13 +1875,16 @@ class KOBOTOUCH(KOBO):
 
         opts = self.settings()
         if opts.extra_customization:
-            create_bookshelves = opts.extra_customization[self.OPT_CREATE_BOOKSHELVES] and self.supports_bookshelves()
+            create_bookshelves      = opts.extra_customization[self.OPT_CREATE_BOOKSHELVES] and self.supports_bookshelves()
-            delete_empty_shelves = opts.extra_customization[self.OPT_DELETE_BOOKSHELVES] and self.supports_bookshelves()
+            delete_empty_shelves    = opts.extra_customization[self.OPT_DELETE_BOOKSHELVES] and self.supports_bookshelves()
-            debugging_title = opts.extra_customization[self.OPT_DEBUGGING_TITLE]
+            update_series_details   = opts.extra_customization[self.OPT_UPDATE_SERIES_DETAILS] and self.supports_series()
+            debugging_title         = opts.extra_customization[self.OPT_DEBUGGING_TITLE]
             debug_print("KoboTouch:update_device_database_collections - set_debugging_title to", debugging_title )
             booklists.set_debugging_title(debugging_title)
         else:
-            delete_empty_shelves = False
+            delete_empty_shelves    = False
+            create_bookshelves      = False
+            update_series_details   = False
 
         collections = booklists.get_collections(collections_attributes)
 #        debug_print('KoboTouch:update_device_database_collections - Collections:', collections)
@@ -1916,7 +1937,7 @@ class KOBOTOUCH(KOBO):
                             debug_print('    class=%s'%book.__class__)
                             debug_print('    book.contentID="%s"'%book.contentID)
                             debug_print('    book.application_id="%s"'%book.application_id)
-                        
+
                         if book.application_id is None:
                             continue
 
@@ -1972,21 +1993,32 @@ class KOBOTOUCH(KOBO):
                     debug_print("No Collections - resetting FavouritesIndex")
                     self.reset_favouritesindex(connection, oncard)
 
-            if self.supports_bookshelves():
+            if self.supports_bookshelves() or self.supports_series():
-                debug_print("KoboTouch:update_device_database_collections - managing bookshelves.")
+                debug_print("KoboTouch:update_device_database_collections - managing bookshelves and series.")
-                if bookshelf_attribute:
+
-                    debug_print("KoboTouch:update_device_database_collections - bookshelf_attribute=", bookshelf_attribute)
+                self.series_set  = 0
-                    for book in booklists:
+                books_in_library = 0
-                        if book.application_id is not None:
+                for book in booklists:
-#                            debug_print("KoboTouch:update_device_database_collections - about to remove a book from shelves book.title=%s" % book.title)
+                    if book.application_id is not None:
+                        books_in_library += 1
+                        show_debug = self.is_debugging_title(book.title)
+                        if show_debug:
+                            debug_print("KoboTouch:update_device_database_collections - book.title=%s" % book.title)
+                        if update_series_details:
+                            self.set_series(connection, book)
+                        if bookshelf_attribute:
+                            if show_debug:
+                                debug_print("KoboTouch:update_device_database_collections - about to remove a book from shelves book.title=%s" % book.title)
                             self.remove_book_from_device_bookshelves(connection, book)
                             book.device_collections.extend(book.kobo_collections)
                 if not prefs['manage_device_metadata'] == 'manual' and delete_empty_shelves:
                     debug_print("KoboTouch:update_device_database_collections - about to clear empty bookshelves")
                     self.delete_empty_bookshelves(connection)
+                debug_print("KoboTouch:update_device_database_collections - Number of series set=%d Number of books=%d" % (self.series_set, books_in_library))
 
                 self.dump_bookshelves(connection)
 
+
         debug_print('KoboTouch:update_device_database_collections - Finished ')
 
     def rebuild_collections(self, booklist, oncard):
@@ -2124,7 +2156,7 @@ class KOBOTOUCH(KOBO):
             debug_print('KoboTouch:remove_book_from_device_bookshelves - book.device_collections=', book.device_collections)
             debug_print('KoboTouch:remove_book_from_device_bookshelves - book.current_shelves=', book.current_shelves)
             debug_print('KoboTouch:remove_book_from_device_bookshelves - remove_shelf_list=', remove_shelf_list)
-        
+
         if len(remove_shelf_list) == 0:
             return
 
@@ -2310,9 +2342,70 @@ class KOBOTOUCH(KOBO):
 
         debug_print("KoboTouch:remove_from_bookshelf - end")
 
+    def set_series(self, connection, book):
+        show_debug = self.is_debugging_title(book.title)
+        if show_debug:
+            debug_print('KoboTouch:set_series book.kobo_series="%s"'%book.kobo_series)
+            debug_print('KoboTouch:set_series book.series="%s"'%book.series)
+            debug_print('KoboTouch:set_series book.series_index=', book.series_index)
+
+        if book.series == book.kobo_series and book.series_index == book.kobo_series_number:
+            if show_debug:
+                debug_print('KoboTouch:set_series - series info the same - not changing')
+            return
+
+        update_query = 'UPDATE content SET Series=?, SeriesNumber==? where BookID is Null and ContentID = ?'
+        if book.series is None:
+            update_values = (None, None, book.contentID, )
+        else:
+            update_values = (book.series, "%g"%book.series_index, book.contentID, )
+
+        cursor = connection.cursor()
+        try:
+            if show_debug:
+                debug_print('KoboTouch:set_series - about to set - parameters:', update_values)
+            cursor.execute(update_query, update_values)
+            self.series_set += 1
+        except:
+            debug_print('    Database Exception:  Unable to set series info')
+            raise
+        else:
+            connection.commit()
+        cursor.close()
+
+        if show_debug:
+            debug_print("KoboTouch:set_series - end")
+
+
+    @classmethod
+    def settings(cls):
+        opts = cls._config().parse()
+        if isinstance(cls.EXTRA_CUSTOMIZATION_DEFAULT, list):
+            if opts.extra_customization is None:
+                opts.extra_customization = []
+            if not isinstance(opts.extra_customization, list):
+                opts.extra_customization = [opts.extra_customization]
+            if len(cls.EXTRA_CUSTOMIZATION_DEFAULT) > len(opts.extra_customization):
+                extra_options_offset = 0
+                extra_customization = []
+                for i,d in enumerate(cls.EXTRA_CUSTOMIZATION_DEFAULT):
+                    if i >= len(opts.extra_customization) + extra_options_offset:
+                        extra_customization.append(d)
+                    elif d.__class__ != opts.extra_customization[i - extra_options_offset].__class__:
+                        extra_options_offset += 1
+                        extra_customization.append(d)
+                    else:
+                        extra_customization.append(opts.extra_customization[i - extra_options_offset])
+                opts.extra_customization = extra_customization
+        return opts
+
+
     def supports_bookshelves(self):
         return self.dbversion >= self.min_supported_dbversion
 
+    def supports_series(self):
+        return self.dbversion >= self.min_dbversion_series
+
 #    def is_debugging_title(self, title):
 ##        debug_print("KoboTouch:is_debugging - title=", title)
 #        is_debugging = False

src/calibre/devices/mtp/unix/devices.c

@@ -14,6 +14,10 @@ const calibre_device_entry_t calibre_mtp_device_table[] = {
     // Amazon Kindle Fire HD
     , { "Amazon", 0x1949, "Fire HD", 0x0007, DEVICE_FLAGS_ANDROID_BUGS}
 
+    // Nexus 10
+    , { "Google", 0x18d1, "Nexus 10", 0x4ee2, DEVICE_FLAGS_ANDROID_BUGS}
+    , { "Google", 0x18d1, "Nexus 10", 0x4ee1, DEVICE_FLAGS_ANDROID_BUGS}
+
     , { NULL, 0xffff, NULL, 0xffff, DEVICE_FLAG_NONE }
 };
 

src/calibre/devices/mtp/windows/content_enumeration.cpp

@@ -696,7 +696,7 @@ PyObject* wpd::put_file(IPortableDevice *device, const wchar_t *parent_id, const
         PyBytes_AsStringAndSize(raw, &buf, &bytes_read);
         if (bytes_read > 0) {
             Py_BEGIN_ALLOW_THREADS;
-            hr = dest->Write(buf, bytes_read, &bytes_written);
+            hr = dest->Write(buf, (ULONG)bytes_read, &bytes_written);
             Py_END_ALLOW_THREADS;
             Py_DECREF(raw);
             if (hr == STG_E_MEDIUMFULL) { PyErr_SetString(WPDError, "Cannot write to device as it is full"); break; }

src/calibre/devices/prst1/driver.py

@@ -380,7 +380,7 @@ class PRST1(USBMS):
         # Record what the max id being used is as well.
         db_books = {}
         for i, row in enumerate(cursor):
-            if row[0] is None:
+            if not hasattr(row[0], 'replace'):
                 continue
             lpath = row[0].replace('\\', '/')
             db_books[lpath] = row[1]

src/calibre/devices/udisks.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, dbus, re
+import os, re
 
 def node_mountpoint(node):
 
@@ -25,6 +25,7 @@ class NoUDisks1(Exception):
 class UDisks(object):
 
     def __init__(self):
+        import dbus
         self.bus = dbus.SystemBus()
         try:
             self.main = dbus.Interface(self.bus.get_object('org.freedesktop.UDisks',
@@ -35,6 +36,7 @@ class UDisks(object):
             raise
 
     def device(self, device_node_path):
+        import dbus
         devpath = self.main.FindDeviceByDeviceFile(device_node_path)
         return dbus.Interface(self.bus.get_object('org.freedesktop.UDisks',
                         devpath), 'org.freedesktop.UDisks.Device')
@@ -73,6 +75,7 @@ class UDisks2(object):
     DRIVE = 'org.freedesktop.UDisks2.Drive'
 
     def __init__(self):
+        import dbus
         self.bus = dbus.SystemBus()
         try:
             self.bus.get_object('org.freedesktop.UDisks2',

src/calibre/ebooks/compression/palmdoc.c

@@ -101,7 +101,7 @@ cpalmdoc_rfind(Byte *data, Py_ssize_t pos, Py_ssize_t chunk_length) {
 static Py_ssize_t
 cpalmdoc_do_compress(buffer *b, char *output) {
     Py_ssize_t i = 0, j, chunk_len, dist;
-    unsigned compound;
+    unsigned int compound;
     Byte c, n;
     bool found;
     char *head;
@@ -119,7 +119,7 @@ cpalmdoc_do_compress(buffer *b, char *output) {
                 dist = i - j;
                 if (j < i && dist <= 2047) {
                     found = true;
-                    compound = (dist << 3) + chunk_len-3;
+                    compound = (unsigned int)((dist << 3) + chunk_len-3);
                     *(output++) = CHAR(0x80 + (compound >> 8 ));
                     *(output++) = CHAR(compound & 0xFF);
                     i += chunk_len;
@@ -148,7 +148,7 @@ cpalmdoc_do_compress(buffer *b, char *output) {
                 temp.data[temp.len++] = c; j++;
             }
             i += temp.len - 1;
-            *(output++) = temp.len;
+            *(output++) = (char)temp.len;
             for (j=0; j < temp.len; j++) *(output++) = (char)temp.data[j];
         }
     }

src/calibre/ebooks/metadata/archive.py

@@ -48,12 +48,13 @@ class ArchiveExtract(FileTypePlugin):
     def run(self, archive):
         is_rar = archive.lower().endswith('.rar')
         if is_rar:
-            from calibre.libunrar import extract_member, names
+            from calibre.utils.unrar import extract_member, names
         else:
             zf = ZipFile(archive, 'r')
 
         if is_rar:
-            fnames = names(archive)
+            with open(archive, 'rb') as rf:
+                fnames = list(names(rf))
         else:
             fnames = zf.namelist()
 
@@ -76,7 +77,8 @@ class ArchiveExtract(FileTypePlugin):
         of = self.temporary_file('_archive_extract.'+ext)
         with closing(of):
             if is_rar:
-                data = extract_member(archive, match=None, name=fname)[1]
+                with open(archive, 'rb') as f:
+                    data = extract_member(f, match=None, name=fname)[1]
                 of.write(data)
             else:
                 of.write(zf.read(fname))
@@ -108,21 +110,44 @@ def get_comic_book_info(d, mi):
                 authors.append(x)
     if authors:
         mi.authors = authors
+    comments = d.get('comments', '')
+    if comments and comments.strip():
+        mi.comments = comments.strip()
+    pubm, puby = d.get('publicationMonth', None), d.get('publicationYear', None)
+    if puby is not None:
+        from calibre.utils.date import parse_only_date
+        from datetime import date
+        try:
+            dt = date(puby, 6 if pubm is None else pubm, 15)
+            dt = parse_only_date(str(dt))
+            mi.pubdate = dt
+        except:
+            pass
 
-
+def get_comic_metadata(stream, stream_type):
-
-def get_cbz_metadata(stream):
     # See http://code.google.com/p/comicbookinfo/wiki/Example
-    from calibre.utils.zipfile import ZipFile
     from calibre.ebooks.metadata import MetaInformation
-    import json
 
-    zf = ZipFile(stream)
+    comment = None
+
     mi = MetaInformation(None, None)
-    if zf.comment:
+
-        m = json.loads(zf.comment)
+    if stream_type == 'cbz':
-        if hasattr(m, 'keys'):
+        from calibre.utils.zipfile import ZipFile
-            for cat in m.keys():
+        zf = ZipFile(stream)
+        comment = zf.comment
+    elif stream_type == 'cbr':
+        from calibre.utils.unrar import RARFile
+        f = RARFile(stream, get_comment=True)
+        comment = f.comment
+
+    if comment:
+        import json
+        m = json.loads(comment)
+        if hasattr(m, 'iterkeys'):
+            for cat in m.iterkeys():
                 if cat.startswith('ComicBookInfo'):
                     get_comic_book_info(m[cat], mi)
+                    break
     return mi
+

src/calibre/ebooks/metadata/rar.py

@@ -8,35 +8,27 @@ Read metadata from RAR archives
 '''
 
 import os
+from io import BytesIO
 
-from calibre.ptempfile import PersistentTemporaryFile, TemporaryDirectory
+from calibre.utils.unrar import extract_member, names
-from calibre.libunrar import extract_member, names
-from calibre import CurrentDir
 
 def get_metadata(stream):
     from calibre.ebooks.metadata.archive import is_comic
     from calibre.ebooks.metadata.meta import get_metadata
 
-    path = getattr(stream, 'name', False)
+    file_names = list(names(stream))
-    if not path:
-        pt = PersistentTemporaryFile('_rar-meta.rar')
-        pt.write(stream.read())
-        pt.close()
-        path = pt.name
-    path = os.path.abspath(path)
-    file_names = list(names(path))
     if is_comic(file_names):
         return get_metadata(stream, 'cbr')
     for f in file_names:
         stream_type = os.path.splitext(f)[1].lower()
         if stream_type:
             stream_type = stream_type[1:]
-            if stream_type in ('lit', 'opf', 'prc', 'mobi', 'fb2', 'epub',
+            if stream_type in {'lit', 'opf', 'prc', 'mobi', 'fb2', 'epub',
-                               'rb', 'imp', 'pdf', 'lrf', 'azw', 'azw1', 'azw3'):
+                               'rb', 'imp', 'pdf', 'lrf', 'azw', 'azw1',
-                with TemporaryDirectory() as tdir:
+                               'azw3'}:
-                    with CurrentDir(tdir):
+                name, data = extract_member(stream, match=None, name=f)
-                       stream = extract_member(path, match=None, name=f,
+                stream = BytesIO(data)
-                               as_file=True)[1]
+                stream.name = os.path.basename(name)
                 return get_metadata(stream, stream_type)
     raise ValueError('No ebook found in RAR archive')
 

src/calibre/ebooks/mobi/mobiml.py

@@ -214,7 +214,11 @@ class MobiMLizer(object):
         if tag in CONTENT_TAGS:
             bstate.inline = para
             pstate = bstate.istate = None
-            etree.SubElement(para, XHTML(tag), attrib=istate.attrib)
+            try:
+                etree.SubElement(para, XHTML(tag), attrib=istate.attrib)
+            except:
+                print 'Invalid subelement:', para, tag, istate.attrib
+                raise
         elif tag in TABLE_TAGS:
             para.attrib['valign'] = 'top'
         if istate.ids:

src/calibre/ebooks/mobi/reader/markup.py

@@ -11,7 +11,7 @@ import re, os
 
 from calibre.ebooks.chardet import strip_encoding_declarations
 
-def update_internal_links(mobi8_reader):
+def update_internal_links(mobi8_reader, log):
     # need to update all links that are internal which
     # are based on positions within the xhtml files **BEFORE**
     # cutting and pasting any pieces into the xhtml text files
@@ -35,11 +35,16 @@ def update_internal_links(mobi8_reader):
                 for m in posfid_index_pattern.finditer(tag):
                     posfid = m.group(1)
                     offset = m.group(2)
-                    filename, idtag = mr.get_id_tag_by_pos_fid(int(posfid, 32),
+                    try:
-                            int(offset, 32))
+                        filename, idtag = mr.get_id_tag_by_pos_fid(
-                    suffix = (b'#' + idtag) if idtag else b''
+                            int(posfid, 32), int(offset, 32))
-                    replacement = filename.split('/')[-1].encode(
+                    except ValueError:
-                            mr.header.codec) + suffix
+                        log.warn('Invalid link, points to nowhere, ignoring')
+                        replacement = b'#'
+                    else:
+                        suffix = (b'#' + idtag) if idtag else b''
+                        replacement = filename.split('/')[-1].encode(
+                                mr.header.codec) + suffix
                     tag = posfid_index_pattern.sub(replacement, tag, 1)
                 srcpieces[j] = tag
         raw = b''.join(srcpieces)
@@ -198,7 +203,7 @@ def update_flow_links(mobi8_reader, resource_map, log):
     # All flows are now unicode and have links resolved
     return flows
 
-def insert_flows_into_markup(parts, flows, mobi8_reader):
+def insert_flows_into_markup(parts, flows, mobi8_reader, log):
     mr = mobi8_reader
 
     # kindle:flow:XXXX?mime=YYYY/ZZZ (used for style sheets, svg images, etc)
@@ -214,12 +219,17 @@ def insert_flows_into_markup(parts, flows, mobi8_reader):
             if tag.startswith('<'):
                 for m in flow_pattern.finditer(tag):
                     num = int(m.group(1), 32)
-                    fi = mr.flowinfo[num]
+                    try:
-                    if fi.format == 'inline':
+                        fi = mr.flowinfo[num]
-                        tag = flows[num]
+                    except IndexError:
+                        log.warn('Ignoring invalid flow reference: %s'%m.group())
+                        tag = ''
                     else:
-                        replacement = '"../' + fi.dir + '/' + fi.fname + '"'
+                        if fi.format == 'inline':
-                        tag = flow_pattern.sub(replacement, tag, 1)
+                            tag = flows[num]
+                        else:
+                            replacement = '"../' + fi.dir + '/' + fi.fname + '"'
+                            tag = flow_pattern.sub(replacement, tag, 1)
                 srcpieces[j] = tag
         part = "".join(srcpieces)
         # store away modified version
@@ -298,7 +308,7 @@ def upshift_markup(parts):
 
 def expand_mobi8_markup(mobi8_reader, resource_map, log):
     # First update all internal links that are based on offsets
-    parts = update_internal_links(mobi8_reader)
+    parts = update_internal_links(mobi8_reader, log)
 
     # Remove pointless markup inserted by kindlegen
     remove_kindlegen_markup(parts)
@@ -308,7 +318,7 @@ def expand_mobi8_markup(mobi8_reader, resource_map, log):
     flows = update_flow_links(mobi8_reader, resource_map, log)
 
     # Insert inline flows into the markup
-    insert_flows_into_markup(parts, flows, mobi8_reader)
+    insert_flows_into_markup(parts, flows, mobi8_reader, log)
 
     # Insert raster images into markup
     insert_images_into_markup(parts, resource_map, log)

src/calibre/ebooks/mobi/reader/mobi8.py

@@ -34,6 +34,16 @@ Elem = namedtuple('Elem',
 FlowInfo = namedtuple('FlowInfo',
         'type format dir fname')
 
+# locate beginning and ending positions of tag with specific aid attribute
+def locate_beg_end_of_tag(ml, aid):
+    pattern = br'''<[^>]*\said\s*=\s*['"]%s['"][^>]*>''' % aid
+    aid_pattern = re.compile(pattern, re.IGNORECASE)
+    for m in re.finditer(aid_pattern, ml):
+        plt = m.start()
+        pgt = ml.find(b'>', plt+1)
+        return plt, pgt
+    return 0, 0
+
 class Mobi8Reader(object):
 
     def __init__(self, mobi6_reader, log):
@@ -148,6 +158,7 @@ class Mobi8Reader(object):
         for skelnum, skelname, divcnt, skelpos, skellen in self.files:
             baseptr = skelpos + skellen
             skeleton = text[skelpos:baseptr]
+            inspos_warned = False
             for i in xrange(divcnt):
                 insertpos, idtext, filenum, seqnum, startpos, length = \
                                     self.elems[divptr]
@@ -156,6 +167,23 @@ class Mobi8Reader(object):
                     filename = 'part%04d.html' % filenum
                 part = text[baseptr:baseptr + length]
                 insertpos = insertpos - skelpos
+                head = skeleton[:insertpos]
+                tail = skeleton[insertpos:]
+                if (tail.find(b'>') < tail.find(b'<') or head.rfind(b'>') <
+                    head.rfind(b'<')):
+                    # There is an incomplete tag in either the head or tail.
+                    # This can happen for some badly formed KF8 files, see for
+                    # example, https://bugs.launchpad.net/bugs/1082669
+                    if not inspos_warned:
+                        self.log.warn(
+                            'The div table for %s has incorrect insert '
+                                'positions. Calculating manually.'%skelname)
+                        inspos_warned = True
+                    bp, ep = locate_beg_end_of_tag(skeleton, aidtext if
+                        isinstance(aidtext, bytes) else aidtext.encode('utf-8'))
+                    if bp != ep:
+                        insertpos = ep + 1 + startpos
+
                 skeleton = skeleton[0:insertpos] + part + skeleton[insertpos:]
                 baseptr = baseptr + length
                 divptr += 1
@@ -320,6 +348,7 @@ class Mobi8Reader(object):
     def create_ncx(self):
         index_entries = read_ncx(self.kf8_sections, self.header.ncxidx,
                 self.header.codec)
+        remove = []
 
         # Add href and anchor info to the index entries
         for entry in index_entries:
@@ -332,11 +361,20 @@ class Mobi8Reader(object):
                 idtag = self.get_id_tag(pos).decode(self.header.codec)
                 href = '%s/%s'%(fi.type, fi.filename)
             else:
-                href, idtag = self.get_id_tag_by_pos_fid(*pos_fid)
+                try:
+                    href, idtag = self.get_id_tag_by_pos_fid(*pos_fid)
+                except ValueError:
+                    self.log.warn('Invalid entry in NCX (title: %s), ignoring'
+                                  %entry['text'])
+                    remove.append(entry)
+                    continue
 
             entry['href'] = href
             entry['idtag'] = idtag
 
+        for e in remove:
+            index_entries.remove(e)
+
         # Build the TOC object
         return build_toc(index_entries)
 

src/calibre/ebooks/oeb/display/paged.coffee

@@ -116,6 +116,18 @@ class PagedDisplay
         # above the columns, which causes them to effectively be added to the
         # page margins (the margin collapse algorithm)
         bs.setProperty('-webkit-margin-collapse', 'separate')
+        # Remove any webkit specified default margin from the first child of body
+        # Otherwise, you could end up with an effective negative margin, I dont
+        # understand exactly why, but see:
+        # https://bugs.launchpad.net/calibre/+bug/1082640 for an example
+        c = document.body.firstChild
+        count = 0
+        while c?.nodeType != 1 and count < 20
+            c = c?.nextSibling
+            count += 1
+        if c?.nodeType == 1
+            c.style.setProperty('-webkit-margin-before', '0')
+
 
         bs.setProperty('overflow', 'visible')
         bs.setProperty('height', (window.innerHeight - this.margin_top - this.margin_bottom) + 'px')

src/calibre/ebooks/oeb/display/webview.py

@@ -28,7 +28,7 @@ def self_closing_sub(match):
     tag = match.group(1)
     if tag.lower().strip() == 'br':
         return match.group()
-    return '<%s %s></%s>'%(match.group(1), match.group(2), match.group(1))
+    return '<%s%s></%s>'%(match.group(1), match.group(2), match.group(1))
 
 def load_html(path, view, codec='utf-8', mime_type=None,
         pre_load_callback=lambda x:None, path_is_html=False):
@@ -45,12 +45,9 @@ def load_html(path, view, codec='utf-8', mime_type=None,
 
     html = EntityDeclarationProcessor(html).processed_html
     has_svg = re.search(r'<[:a-zA-Z]*svg', html) is not None
-    if 'xhtml' in mime_type:
+    self_closing_pat = re.compile(r'<\s*([A-Za-z1-6]+)([^>]*)/\s*>')
-        self_closing_pat = re.compile(r'<([a-z1-6]+)\s+([^>]+)/>',
+    html = self_closing_pat.sub(self_closing_sub, html)
-                re.IGNORECASE)
-        html = self_closing_pat.sub(self_closing_sub, html)
 
-    html = re.sub(ur'<\s*title\s*/\s*>', u'', html, flags=re.IGNORECASE)
     loading_url = QUrl.fromLocalFile(path)
     pre_load_callback(loading_url)
 

src/calibre/ebooks/oeb/transforms/guide.py

@@ -21,8 +21,13 @@ class Clean(object):
                     'other.ms-thumbimage', 'thumbimagestandard'):
                 if x in self.oeb.guide:
                     href = self.oeb.guide[x].href
-                    item = self.oeb.manifest.hrefs[href]
+                    try:
-                    covers.append([self.oeb.guide[x], len(item.data)])
+                        item = self.oeb.manifest.hrefs[href]
+                    except KeyError:
+                        continue
+                    else:
+                        covers.append([self.oeb.guide[x], len(item.data)])
+
             covers.sort(cmp=lambda x,y:cmp(x[1], y[1]), reverse=True)
             if covers:
                 ref = covers[0][0]

src/calibre/gui2/__init__.py

@@ -105,6 +105,7 @@ gprefs.defaults['ui_style'] = 'calibre' if iswindows or isosx else 'system'
 gprefs.defaults['tag_browser_old_look'] = False
 gprefs.defaults['book_list_tooltips'] = True
 gprefs.defaults['bd_show_cover'] = True
+gprefs.defaults['bd_overlay_cover_size'] = False
 # }}}
 
 NONE = QVariant() #: Null value to return from the data function of item models
@@ -465,6 +466,8 @@ class FileIconProvider(QFileIconProvider):
              'gif'     : 'gif',
              'png'     : 'png',
              'bmp'     : 'bmp',
+             'cbz'     : 'cbz',
+             'cbr'     : 'cbr',
              'svg'     : 'svg',
              'html'    : 'html',
              'htmlz'   : 'html',

src/calibre/gui2/actions/convert.py

@@ -132,6 +132,7 @@ class ConvertAction(InterfaceAction):
             num = len(jobs)
 
         if num > 0:
+            self.gui.jobs_pointer.start()
             self.gui.status_bar.show_message(_('Starting conversion of %d book(s)') %
                 num, 2000)
 

src/calibre/gui2/book_details.py

@@ -7,7 +7,8 @@ __docformat__ = 'restructuredtext en'
 
 from PyQt4.Qt import (QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, QIcon,
     QPropertyAnimation, QEasingCurve, QApplication, QFontInfo, QAction,
-    QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette, QMenu)
+    QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette, QMenu,
+    QPen, QColor)
 from PyQt4.QtWebKit import QWebView
 
 from calibre import fit_image, force_unicode, prepare_string_for_xml
@@ -324,6 +325,17 @@ class CoverView(QWidget): # {{{
         p.setRenderHints(QPainter.Antialiasing | QPainter.SmoothPixmapTransform)
         p.drawPixmap(target, self.pixmap.scaled(target.size(),
             Qt.KeepAspectRatio, Qt.SmoothTransformation))
+        if gprefs['bd_overlay_cover_size']:
+            sztgt = target.adjusted(0, 0, 0, -4)
+            f = p.font()
+            f.setBold(True)
+            p.setFont(f)
+            sz = u'\u00a0%d x %d\u00a0'%(self.pixmap.width(), self.pixmap.height())
+            flags = Qt.AlignBottom|Qt.AlignRight|Qt.TextSingleLine
+            szrect = p.boundingRect(sztgt, flags, sz)
+            p.fillRect(szrect.adjusted(0, 0, 0, 4), QColor(0, 0, 0, 200))
+            p.setPen(QPen(QColor(255,255,255)))
+            p.drawText(sztgt, flags, sz)
         p.end()
 
     current_pixmap_size = pyqtProperty('QSize',

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -88,7 +88,7 @@ class PluginWidget(QWidget,Ui_Form):
                              [{'ordinal':0,
                                'enabled':True,
                                'name':_('Catalogs'),
-                               'field':'Tags',
+                               'field':_('Tags'),
                                'pattern':'Catalog'},],
                              ['table_widget'])
 
@@ -97,13 +97,13 @@ class PluginWidget(QWidget,Ui_Form):
                              [{'ordinal':0,
                                'enabled':True,
                                'name':_('Read book'),
-                               'field':'Tags',
+                               'field':_('Tags'),
                                'pattern':'+',
                                'prefix':u'\u2713'},
                               {'ordinal':1,
                                'enabled':True,
                                'name':_('Wishlist item'),
-                               'field':'Tags',
+                               'field':_('Tags'),
                                'pattern':'Wishlist',
                                'prefix':u'\u00d7'},],
                              ['table_widget','table_widget'])
@@ -127,7 +127,7 @@ class PluginWidget(QWidget,Ui_Form):
             elif 'prefix' in rule and rule['prefix'] is None:
                 continue
             else:
-                if rule['field'] != 'Tags':
+                if rule['field'] != _('Tags'):
                     # Look up custom column friendly name
                     rule['field'] = self.eligible_custom_fields[rule['field']]['field']
                     if rule['pattern'] in [_('any value'),_('any date')]:
@@ -144,14 +144,14 @@ class PluginWidget(QWidget,Ui_Form):
         # Strip off the trailing '_tw'
         opts_dict[c_name[:-3]] = opt_value
 
-    def exclude_genre_changed(self, regex):
+    def exclude_genre_changed(self):
         """ Dynamically compute excluded genres.
 
-        Run exclude_genre regex against db.all_tags() to show excluded tags.
+        Run exclude_genre regex against selected genre_source_field to show excluded tags.
-        PROVISIONAL CODE, NEEDS TESTING
 
-        Args:
+        Inputs:
-         regex (QLineEdit.text()): regex to compile, compute
+            current regex
+            genre_source_field
 
         Output:
          self.exclude_genre_results (QLabel): updated to show tags to be excluded as genres
@@ -183,23 +183,31 @@ class PluginWidget(QWidget,Ui_Form):
                 return "%s  ...  %s" % (', '.join(start), ', '.join(end))
 
         results = _('No genres will be excluded')
+
+        regex = unicode(getattr(self, 'exclude_genre').text()).strip()
         if not regex:
             self.exclude_genre_results.clear()
             self.exclude_genre_results.setText(results)
             return
 
+        # Populate all_genre_tags from currently source
+        if self.genre_source_field_name == _('Tags'):
+            all_genre_tags = self.db.all_tags()
+        else:
+            all_genre_tags = list(self.db.all_custom(self.db.field_metadata.key_to_label(self.genre_source_field_name)))
+
         try:
             pattern = re.compile((str(regex)))
         except:
             results = _("regex error: %s") % sys.exc_info()[1]
         else:
             excluded_tags = []
-            for tag in self.all_tags:
+            for tag in all_genre_tags:
                 hit = pattern.search(tag)
                 if hit:
                     excluded_tags.append(hit.string)
             if excluded_tags:
-                if set(excluded_tags) == set(self.all_tags):
+                if set(excluded_tags) == set(all_genre_tags):
                     results = _("All genres will be excluded")
                 else:
                     results = _truncated_results(excluded_tags)
@@ -218,7 +226,7 @@ class PluginWidget(QWidget,Ui_Form):
     def fetch_eligible_custom_fields(self):
         self.all_custom_fields = self.db.custom_field_keys()
         custom_fields = {}
-        custom_fields['Tags'] = {'field':'tag', 'datatype':u'text'}
+        custom_fields[_('Tags')] = {'field':'tag', 'datatype':u'text'}
         for custom_field in self.all_custom_fields:
             field_md = self.db.metadata_for_field(custom_field)
             if field_md['datatype'] in ['bool','composite','datetime','enumeration','text']:
@@ -237,6 +245,34 @@ class PluginWidget(QWidget,Ui_Form):
         self.merge_after.setEnabled(enabled)
         self.include_hr.setEnabled(enabled)
 
+    def generate_genres_changed(self, enabled):
+        '''
+        Toggle Genres-related controls
+        '''
+        self.genre_source_field.setEnabled(enabled)
+
+    def genre_source_field_changed(self,new_index):
+        '''
+        Process changes in the genre_source_field combo box
+        Update Excluded genres preview
+        '''
+        new_source = str(self.genre_source_field.currentText())
+        self.genre_source_field_name = new_source
+        if new_source != _('Tags'):
+            genre_source_spec = self.genre_source_fields[unicode(new_source)]
+            self.genre_source_field_name = genre_source_spec['field']
+        self.exclude_genre_changed()
+
+    def header_note_source_field_changed(self,new_index):
+        '''
+        Process changes in the header_note_source_field combo box
+        '''
+        new_source = str(self.header_note_source_field.currentText())
+        self.header_note_source_field_name = new_source
+        if new_source > '':
+            header_note_source_spec = self.header_note_source_fields[unicode(new_source)]
+            self.header_note_source_field_name = header_note_source_spec['field']
+
     def initialize(self, name, db):
         '''
         CheckBoxControls (c_type: check_box):
@@ -245,8 +281,8 @@ class PluginWidget(QWidget,Ui_Form):
              'generate_recently_added','generate_descriptions',
              'include_hr']
         ComboBoxControls (c_type: combo_box):
-            ['exclude_source_field','header_note_source_field',
+            ['exclude_source_field','genre_source_field',
-             'merge_source_field']
+             'header_note_source_field','merge_source_field']
         LineEditControls (c_type: line_edit):
             ['exclude_genre']
         RadioButtonControls (c_type: radio_button):
@@ -261,11 +297,11 @@ class PluginWidget(QWidget,Ui_Form):
         '''
         self.name = name
         self.db = db
-        self.all_tags = db.all_tags()
+        self.all_genre_tags = []
         self.fetch_eligible_custom_fields()
         self.populate_combo_boxes()
 
-        # Update dialog fields from stored options
+        # Update dialog fields from stored options, validating options for combo boxes
         exclusion_rules = []
         prefix_rules = []
         for opt in self.OPTION_FIELDS:
@@ -273,13 +309,18 @@ class PluginWidget(QWidget,Ui_Form):
             opt_value = gprefs.get(self.name + '_' + c_name, c_def)
             if c_type in ['check_box']:
                 getattr(self, c_name).setChecked(eval(str(opt_value)))
-            elif c_type in ['combo_box'] and opt_value is not None:
+            elif c_type in ['combo_box']:
-                # *** Test this code with combo boxes ***
+                if opt_value is None:
-                #index = self.read_source_field.findText(opt_value)
+                    index = 0
-                index = getattr(self,c_name).findText(opt_value)
+                    if c_name == 'genre_source_field':
-                if index == -1 and c_name == 'read_source_field':
+                        index = self.genre_source_field.findText(_('Tags'))
-                    index = self.read_source_field.findText('Tag')
+                else:
-                #self.read_source_field.setCurrentIndex(index)
+                    index = getattr(self,c_name).findText(opt_value)
+                    if index == -1:
+                        if c_name == 'read_source_field':
+                            index = self.read_source_field.findText(_('Tags'))
+                        elif c_name == 'genre_source_field':
+                            index = self.genre_source_field.findText(_('Tags'))
                 getattr(self,c_name).setCurrentIndex(index)
             elif c_type in ['line_edit']:
                 getattr(self, c_name).setText(opt_value if opt_value else '')
@@ -320,6 +361,17 @@ class PluginWidget(QWidget,Ui_Form):
             header_note_source_spec = self.header_note_source_fields[cs]
             self.header_note_source_field_name = header_note_source_spec['field']
 
+        # Init self.genre_source_field_name
+        self.genre_source_field_name = _('Tags')
+        cs = unicode(self.genre_source_field.currentText())
+        if cs != _('Tags'):
+            genre_source_spec = self.genre_source_fields[cs]
+            self.genre_source_field_name = genre_source_spec['field']
+
+        # Hook Genres checkbox
+        self.generate_genres.clicked.connect(self.generate_genres_changed)
+        self.generate_genres_changed(self.generate_genres.isChecked())
+
         # Initialize exclusion rules
         self.exclusion_rules_table = ExclusionRules(self.exclusion_rules_gb,
             "exclusion_rules_tw",exclusion_rules, self.eligible_custom_fields,self.db)
@@ -329,7 +381,27 @@ class PluginWidget(QWidget,Ui_Form):
             "prefix_rules_tw",prefix_rules, self.eligible_custom_fields,self.db)
 
         # Initialize excluded genres preview
-        self.exclude_genre_changed(unicode(getattr(self, 'exclude_genre').text()).strip())
+        self.exclude_genre_changed()
+
+    def merge_source_field_changed(self,new_index):
+        '''
+        Process changes in the merge_source_field combo box
+        '''
+        new_source = str(self.merge_source_field.currentText())
+        self.merge_source_field_name = new_source
+        if new_source > '':
+            merge_source_spec = self.merge_source_fields[unicode(new_source)]
+            self.merge_source_field_name = merge_source_spec['field']
+            if not self.merge_before.isChecked() and not self.merge_after.isChecked():
+                self.merge_after.setChecked(True)
+            self.merge_before.setEnabled(True)
+            self.merge_after.setEnabled(True)
+            self.include_hr.setEnabled(True)
+
+        else:
+            self.merge_before.setEnabled(False)
+            self.merge_after.setEnabled(False)
+            self.include_hr.setEnabled(False)
 
     def options(self):
         # Save/return the current options
@@ -373,7 +445,7 @@ class PluginWidget(QWidget,Ui_Form):
             else:
                 opts_dict[c_name] = opt_value
 
-        # Generate specs for merge_comments, header_note_source_field
+        # Generate specs for merge_comments, header_note_source_field, genre_source_field
         checked = ''
         if self.merge_before.isChecked():
             checked = 'before'
@@ -385,6 +457,8 @@ class PluginWidget(QWidget,Ui_Form):
 
         opts_dict['header_note_source_field'] = self.header_note_source_field_name
 
+        opts_dict['genre_source_field'] = self.genre_source_field_name
+
         # Fix up exclude_genre regex if blank. Assume blank = no exclusions
         if opts_dict['exclude_genre'] == '':
             opts_dict['exclude_genre'] = 'a^'
@@ -457,35 +531,18 @@ class PluginWidget(QWidget,Ui_Form):
         self.merge_after.setEnabled(False)
         self.include_hr.setEnabled(False)
 
-    def header_note_source_field_changed(self,new_index):
+        # Populate the 'Genres' combo box
-        '''
+        custom_fields = {_('Tags'):{'field':None,'datatype':None}}
-        Process changes in the header_note_source_field combo box
+        for custom_field in self.all_custom_fields:
-        '''
+            field_md = self.db.metadata_for_field(custom_field)
-        new_source = str(self.header_note_source_field.currentText())
+            if field_md['datatype'] in ['text','enumeration']:
-        self.header_note_source_field_name = new_source
+                custom_fields[field_md['name']] = {'field':custom_field,
-        if new_source > '':
+                                                   'datatype':field_md['datatype']}
-            header_note_source_spec = self.header_note_source_fields[unicode(new_source)]
+        # Add the sorted eligible fields to the combo box
-            self.header_note_source_field_name = header_note_source_spec['field']
+        for cf in sorted(custom_fields, key=sort_key):
-
+            self.genre_source_field.addItem(cf)
-    def merge_source_field_changed(self,new_index):
+        self.genre_source_fields = custom_fields
-        '''
+        self.genre_source_field.currentIndexChanged.connect(self.genre_source_field_changed)
-        Process changes in the merge_source_field combo box
-        '''
-        new_source = str(self.merge_source_field.currentText())
-        self.merge_source_field_name = new_source
-        if new_source > '':
-            merge_source_spec = self.merge_source_fields[unicode(new_source)]
-            self.merge_source_field_name = merge_source_spec['field']
-            if not self.merge_before.isChecked() and not self.merge_after.isChecked():
-                self.merge_after.setChecked(True)
-            self.merge_before.setEnabled(True)
-            self.merge_after.setEnabled(True)
-            self.include_hr.setEnabled(True)
-
-        else:
-            self.merge_before.setEnabled(False)
-            self.merge_after.setEnabled(False)
-            self.include_hr.setEnabled(False)
 
     def show_help(self):
         '''
@@ -779,9 +836,10 @@ class GenericRulesTable(QTableWidget):
         # Populate the Pattern field based upon the Source field
 
         source_field = str(combo.currentText())
+
         if source_field == '':
             values = []
-        elif source_field == 'Tags':
+        elif source_field == _('Tags'):
             values = sorted(self.db.all_tags(), key=sort_key)
         else:
             if self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['enumeration', 'text']:

src/calibre/gui2/catalog/catalog_epub_mobi.ui

@@ -54,40 +54,71 @@
         </property>
        </widget>
       </item>
-      <item row="0" column="2">
+      <item row="1" column="0">
        <widget class="QCheckBox" name="generate_titles">
         <property name="text">
          <string>&amp;Titles</string>
         </property>
        </widget>
       </item>
-      <item row="0" column="3">
+      <item row="3" column="0">
        <widget class="QCheckBox" name="generate_series">
         <property name="text">
          <string>&amp;Series</string>
         </property>
        </widget>
       </item>
-      <item row="4" column="0">
+      <item row="0" column="2">
-       <widget class="QCheckBox" name="generate_genres">
+       <layout class="QHBoxLayout" name="horizontalLayout_2">
-        <property name="text">
+        <item>
-         <string>&amp;Genres</string>
+         <widget class="QCheckBox" name="generate_genres">
-        </property>
+          <property name="text">
-       </widget>
+           <string>&amp;Genres</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QComboBox" name="genre_source_field">
+          <property name="toolTip">
+           <string>Field containing Genre information</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
       </item>
-      <item row="4" column="2">
+      <item row="1" column="2">
-       <widget class="QCheckBox" name="generate_recently_added">
+       <layout class="QHBoxLayout" name="horizontalLayout_5">
-        <property name="text">
+        <item>
-         <string>&amp;Recently Added</string>
+         <widget class="QCheckBox" name="generate_recently_added">
-        </property>
+          <property name="minimumSize">
-       </widget>
+           <size>
+            <width>0</width>
+            <height>26</height>
+           </size>
+          </property>
+          <property name="text">
+           <string>&amp;Recently Added</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
       </item>
-      <item row="4" column="3">
+      <item row="3" column="2">
-       <widget class="QCheckBox" name="generate_descriptions">
+       <layout class="QHBoxLayout" name="horizontalLayout_7">
-        <property name="text">
+        <item>
-         <string>&amp;Descriptions</string>
+         <widget class="QCheckBox" name="generate_descriptions">
-        </property>
+          <property name="minimumSize">
-       </widget>
+           <size>
+            <width>0</width>
+            <height>26</height>
+           </size>
+          </property>
+          <property name="text">
+           <string>&amp;Descriptions</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
       </item>
      </layout>
     </widget>
@@ -177,7 +208,7 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book]
          </size>
         </property>
         <property name="text">
-         <string>Tags to &amp;exclude (regex):</string>
+         <string>Genres to &amp;exclude (regex):</string>
         </property>
         <property name="textFormat">
          <enum>Qt::AutoText</enum>

src/calibre/gui2/dialogs/choose_library.ui

@@ -59,13 +59,13 @@
      </item>
      <item>
       <widget class="QCheckBox" name="copy_structure">
-       <property name="text">
-        <string>&amp;Copy structure from the current library</string>
-       </property>
        <property name="toolTip">
         <string>Copy the custom columns, saved searches, column widths, plugboards,
 user categories, and other information from the old to the new library</string>
        </property>
+       <property name="text">
+        <string>&amp;Copy structure from the current library</string>
+       </property>
       </widget>
      </item>
     </layout>
@@ -138,7 +138,14 @@ user categories, and other information from the old to the new library</string>
     </widget>
    </item>
    <item row="2" column="1" colspan="2">
-    <widget class="HistoryLineEdit" name="location"/>
+    <widget class="HistoryLineEdit" name="location">
+     <property name="sizeAdjustPolicy">
+      <enum>QComboBox::AdjustToMinimumContentsLength</enum>
+     </property>
+     <property name="minimumContentsLength">
+      <number>40</number>
+     </property>
+    </widget>
    </item>
   </layout>
  </widget>

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -17,6 +17,7 @@ from calibre.gui2.custom_column_widgets import populate_metadata_page
 from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATETIME, \
     gprefs, question_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
+from calibre.gui2.metadata.basic_widgets import CalendarWidget
 from calibre.utils.config import dynamic, JSONConfig
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import sort_key, capitalize
@@ -339,6 +340,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.tag_editor_button.clicked.connect(self.tag_editor)
         self.autonumber_series.stateChanged[int].connect(self.auto_number_changed)
         self.pubdate.setMinimumDateTime(UNDEFINED_QDATETIME)
+        self.pubdate_cw = CalendarWidget(self.pubdate)
+        self.pubdate.setCalendarWidget(self.pubdate_cw)
         pubdate_format = tweaks['gui_pubdate_display_format']
         if pubdate_format is not None:
             self.pubdate.setDisplayFormat(pubdate_format)

src/calibre/gui2/job_indicator.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import (QPainter, Qt, QWidget, QPropertyAnimation, QRect, QPoint,
+                      QColor, QEasingCurve, QBrush, QPainterPath, QPointF,
+                      QPalette)
+
+from calibre.gui2 import config
+
+class Pointer(QWidget):
+
+    def __init__(self, gui):
+        QWidget.__init__(self, gui)
+        self.setObjectName('jobs_pointer')
+        self.setVisible(False)
+        self.resize(100, 80)
+        self.animation = QPropertyAnimation(self, "geometry", self)
+        self.animation.setDuration(750)
+        self.animation.setLoopCount(2)
+        self.animation.setEasingCurve(QEasingCurve.Linear)
+        self.animation.finished.connect(self.hide)
+
+        taily, heady = 0, 55
+        self.arrow_path = QPainterPath(QPointF(40, taily))
+        self.arrow_path.lineTo(40, heady)
+        self.arrow_path.lineTo(20, heady)
+        self.arrow_path.lineTo(50, self.height())
+        self.arrow_path.lineTo(80, heady)
+        self.arrow_path.lineTo(60, heady)
+        self.arrow_path.lineTo(60, taily)
+        self.arrow_path.closeSubpath()
+
+        c = self.palette().color(QPalette.Active, QPalette.WindowText)
+        self.color = QColor(c)
+        self.color.setAlpha(100)
+        self.brush = QBrush(self.color, Qt.SolidPattern)
+
+        # from PyQt4.Qt import QTimer
+        # QTimer.singleShot(1000, self.start)
+
+    @property
+    def gui(self):
+        return self.parent()
+
+    def point_at(self, frac):
+        return (self.path.pointAtPercent(frac).toPoint() -
+                QPoint(self.rect().center().x(), self.height()))
+
+    def rect_at(self, frac):
+        return QRect(self.point_at(frac), self.size())
+
+    def abspos(self, widget):
+        pos = widget.pos()
+        parent = widget.parent()
+        while parent is not self.gui:
+            pos += parent.pos()
+            parent = parent.parent()
+        return pos
+
+    def start(self):
+        if config['disable_animations']:
+            return
+        self.setVisible(True)
+        self.raise_()
+        end = self.abspos(self.gui.jobs_button)
+        end = QPointF( end.x() + self.gui.jobs_button.width()/3.0, end.y()+20)
+        start = QPointF(end.x(), end.y() - 0.5*self.height())
+        self.path = QPainterPath(QPointF(start))
+        self.path.lineTo(end)
+        self.path.closeSubpath()
+        self.animation.setStartValue(self.rect_at(0.0))
+        self.animation.setEndValue(self.rect_at(1.0))
+        self.animation.setDirection(self.animation.Backward)
+        num_keys = 100
+        for i in xrange(1, num_keys):
+            i /= num_keys
+            self.animation.setKeyValueAt(i, self.rect_at(i))
+        self.animation.start()
+
+    def paintEvent(self, ev):
+        p = QPainter(self)
+        p.setRenderHints(p.Antialiasing)
+        p.setBrush(self.brush)
+        p.setPen(Qt.NoPen)
+        p.drawPath(self.arrow_path)
+        p.end()
+

src/calibre/gui2/main_window.py

@@ -7,9 +7,8 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import StringIO, traceback, sys, gc
 
-from PyQt4.Qt import QMainWindow, QString, Qt, QFont, QTimer, \
+from PyQt4.Qt import (QMainWindow, QTimer, QAction, QMenu, QMenuBar, QIcon,
-                     QAction, QMenu, QMenuBar, QIcon, pyqtSignal, QObject
+                      pyqtSignal, QObject)
-from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog
 from calibre.utils.config import OptionParser
 from calibre.gui2 import error_dialog
 from calibre import prints
@@ -20,26 +19,8 @@ Usage: %prog [options]
 Launch the Graphical User Interface
 '''):
     parser = OptionParser(usage)
-    # The b is required because of a regression in optparse.py in python 2.7.0
-    parser.add_option(b'--redirect-console-output', default=False, action='store_true', dest='redirect',
-                      help=_('Redirect console output to a dialog window (both stdout and stderr). Useful on windows where GUI apps do not have a output streams.'))
     return parser
 
-class DebugWindow(ConversionErrorDialog):
-
-    def __init__(self, parent):
-        ConversionErrorDialog.__init__(self, parent, 'Console output', '')
-        self.setModal(Qt.NonModal)
-        font = QFont()
-        font.setStyleHint(QFont.TypeWriter)
-        self.text.setFont(font)
-
-    def write(self, msg):
-        self.text.setPlainText(self.text.toPlainText()+QString(msg))
-
-    def flush(self):
-        pass
-
 class GarbageCollector(QObject):
 
     '''
@@ -120,10 +101,6 @@ class MainWindow(QMainWindow):
         QMainWindow.__init__(self, parent)
         if disable_automatic_gc:
             self._gc = GarbageCollector(self, debug=False)
-        if getattr(opts, 'redirect', False):
-            self.__console_redirect = DebugWindow(self)
-            sys.stdout = sys.stderr = self.__console_redirect
-            self.__console_redirect.show()
 
     def unhandled_exception(self, type, value, tb):
         if type == KeyboardInterrupt:

src/calibre/gui2/notify.py

@@ -7,7 +7,7 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 
-from calibre.constants import islinux, isosx
+from calibre.constants import islinux, isosx, get_osx_version
 
 class Notifier(object):
 
@@ -101,32 +101,37 @@ class QtNotifier(Notifier):
             except:
                 pass
 
-class GrowlNotifier(Notifier):
+class AppleNotifier(Notifier):
-
-    notification_type = 'All notifications'
 
     def __init__(self):
+        self.ok = False
+        import os, sys
         try:
-            import Growl
+            self.exe = os.path.join(sys.console_binaries_path.replace(
-            self.icon = Growl.Image.imageFromPath(I('notify.png'))
+                'console.app', 'calibre-notifier.app'), 'Calibre')
-            self.growl = Growl.GrowlNotifier(applicationName='calibre',
+            self.ok = os.access(self.exe, os.X_OK)
-                    applicationIcon=self.icon, notifications=[self.notification_type])
+            import subprocess
-            self.growl.register()
+            self.call = subprocess.Popen
-            self.ok = True
         except:
-            self.ok = False
+            pass
 
-    def encode(self, msg):
+    def notify(self, body, summary):
-        if isinstance(msg, unicode):
+        def encode(x):
-            msg = msg.encode('utf-8')
+            if isinstance(x, unicode):
-        return msg
+                x = x.encode('utf-8')
+            return x
+
+        cmd = [self.exe, '-activate',
+               'net.kovidgoyal.calibre', '-message', encode(body)]
+        if summary:
+            cmd += ['-title', encode(summary)]
+        self.call(cmd)
 
     def __call__(self, body, summary=None, replaces_id=None, timeout=0):
         timeout, body, summary = self.get_msg_parms(timeout, body, summary)
         if self.ok:
             try:
-                self.growl.notify(self.notification_type, self.encode(summary),
+                self.notify(body, summary)
-                        self.encode(body))
             except:
                 import traceback
                 traceback.print_exc()
@@ -140,10 +145,10 @@ def get_notifier(systray=None):
             ans = FDONotifier()
             if not ans.ok:
                 ans = None
-    #if isosx:
+    elif isosx and get_osx_version() >= (10, 8, 0):
-    #    ans = GrowlNotifier()
+        ans = AppleNotifier()
-    #    if not ans.ok:
+        if not ans.ok:
-    #        ans = None
+            ans = None
     if ans is None:
         ans = QtNotifier(systray)
         if not ans.ok:

src/calibre/gui2/preferences/look_feel.py

@@ -107,6 +107,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('book_list_tooltips', gprefs)
         r('tag_browser_old_look', gprefs, restart_required=True)
         r('bd_show_cover', gprefs)
+        r('bd_overlay_cover_size', gprefs)
 
         r('cover_flow_queue_length', config, restart_required=True)
 

src/calibre/gui2/preferences/look_feel.ui

@@ -309,11 +309,38 @@ Manage Authors. You can use the values {author} and
         </layout>
        </item>
        <item row="1" column="0" colspan="2">
-        <widget class="QCheckBox" name="opt_bd_show_cover">
+        <layout class="QHBoxLayout">
-         <property name="text">
+         <item>
-          <string>Show &amp;cover in the book details panel</string>
+          <widget class="QCheckBox" name="opt_bd_show_cover">
-         </property>
+           <property name="text">
-        </widget>
+            <string>Show &amp;cover in the book details panel</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QCheckBox" name="opt_bd_overlay_cover_size">
+           <property name="toolTip">
+            <string>Show the size of the book's cover in pixels</string>
+           </property>
+           <property name="text">
+            <string>Show cover &amp;size</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <spacer name="horizontalSpacer_bd1">
+           <property name="orientation">
+            <enum>Qt::Horizontal</enum>
+           </property>
+           <property name="sizeHint" stdset="0">
+            <size>
+             <width>20</width>
+             <height>0</height>
+            </size>
+           </property>
+          </spacer>
+         </item>
+        </layout>
        </item>
       </layout>
      </widget>

src/calibre/gui2/store/stores/amazon_de_plugin.py

@@ -22,3 +22,4 @@ class AmazonDEKindleStore(AmazonUKKindleStore):
                           '&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742')
     search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords='
 
+    author_article = 'von '

src/calibre/gui2/store/stores/amazon_es_plugin.py

@@ -19,4 +19,6 @@ class AmazonESKindleStore(AmazonUKKindleStore):
     store_link_details = ('http://www.amazon.es/gp/redirect.html?ie=UTF8&'
                           'location=http://www.amazon.es/dp/%(asin)s&tag=%(tag)s'
                           '&linkCode=ur2&camp=3626&creative=24790')
-    search_url = 'http://www.amazon.es/s/?url=search-alias%3Ddigital-text&field-keywords='
+    search_url = 'http://www.amazon.es/s/?url=search-alias%3Ddigital-text&field-keywords='
+
+    author_article = 'de '

src/calibre/gui2/store/stores/amazon_fr_plugin.py

@@ -19,3 +19,4 @@ class AmazonFRKindleStore(AmazonUKKindleStore):
     store_link_details = 'http://www.amazon.fr/gp/redirect.html?ie=UTF8&location=http://www.amazon.fr/dp/%(asin)s&tag=%(tag)s&linkCode=ur2&camp=1634&creative=6738'
     search_url = 'http://www.amazon.fr/s/?url=search-alias%3Ddigital-text&field-keywords='
 
+    author_article = 'de '

src/calibre/gui2/store/stores/amazon_it_plugin.py

@@ -20,3 +20,5 @@ class AmazonITKindleStore(AmazonUKKindleStore):
                           'location=http://www.amazon.it/dp/%(asin)s&tag=%(tag)s&'
                           'linkCode=ur2&camp=3370&creative=23322')
     search_url = 'http://www.amazon.it/s/?url=search-alias%3Ddigital-text&field-keywords='
+
+    author_article = 'di '

src/calibre/gui2/store/stores/amazon_uk_plugin.py

@@ -29,6 +29,8 @@ class AmazonUKKindleStore(StorePlugin):
                           'linkCode=ur2&camp=1634&creative=6738')
     search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords='
 
+    author_article = 'by '
+
     '''
     For comments on the implementation, please see amazon_plugin.py
     '''
@@ -88,7 +90,8 @@ class AmazonUKKindleStore(StorePlugin):
                 title = ''.join(data.xpath(title_xpath))
                 author = ''.join(data.xpath(author_xpath))
                 try:
-                    author = author.split('by ', 1)[1].split(" (")[0]
+                    if self.author_article:
+                        author = author.split(self.author_article, 1)[1].split(" (")[0]
                 except:
                     pass
 

src/calibre/gui2/store/stores/publio_plugin.py

@@ -68,7 +68,7 @@ class PublioStore(BasicStoreConfig, StorePlugin):
                     counter -= 1
 
                     s = SearchResult()
-                    s.cover_url = 'http://www.publio.pl' + cover_url 
+                    s.cover_url = 'http://www.publio.pl' + cover_url
                     s.title = title.strip()
                     s.author = author
                     s.price = price

src/calibre/gui2/ui.py

@@ -45,6 +45,7 @@ from calibre.gui2.auto_add import AutoAdder
 from calibre.library.sqlite import sqlite, DatabaseException
 from calibre.gui2.proceed import ProceedQuestion
 from calibre.gui2.dialogs.message_box import JobError
+from calibre.gui2.job_indicator import Pointer
 
 class Listener(Thread): # {{{
 
@@ -109,6 +110,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
     def __init__(self, opts, parent=None, gui_debug=None):
         global _gui
         MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True)
+        self.jobs_pointer = Pointer(self)
         self.proceed_requested.connect(self.do_proceed,
                 type=Qt.QueuedConnection)
         self.proceed_question = ProceedQuestion(self)

src/calibre/gui2/update.py

@@ -67,6 +67,7 @@ class UpdateNotification(QDialog):
 
     def __init__(self, calibre_version, plugin_updates, parent=None):
         QDialog.__init__(self, parent)
+        self.setAttribute(Qt.WA_QuitOnClose, False)
         self.resize(400, 250)
         self.l = QGridLayout()
         self.setLayout(self.l)

src/calibre/gui2/viewer/config.py

@@ -287,6 +287,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
         c.set('fullscreen_scrollbar', self.opt_fullscreen_scrollbar.isChecked())
         c.set('show_fullscreen_help', self.opt_show_fullscreen_help.isChecked())
         c.set('cols_per_screen', int(self.opt_cols_per_screen.value()))
+        c.set('start_in_fullscreen', self.opt_start_in_fullscreen.isChecked())
         c.set('use_book_margins', not
                 self.opt_override_book_margins.isChecked())
         c.set('text_color', self.current_text_color)

src/calibre/gui2/viewer/javascript.py

@@ -80,7 +80,7 @@ class JavaScriptLoader(object):
             evaljs(src)
 
         if not lang:
-            lang = 'en'
+            lang = default_lang or 'en'
 
         def lang_name(l):
             l = l.lower()

src/calibre/gui2/viewer/main.py

@@ -510,17 +510,18 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
     def show_full_screen_label(self):
         f = self.full_screen_label
         self.esc_full_screen_action.setEnabled(True)
-        f.setVisible(True)
         height = 200
         width = int(0.7*self.view.width())
         f.resize(width, height)
         f.move((self.view.width() - width)//2, (self.view.height()-height)//2)
-        a = self.full_screen_label_anim
+        if self.view.document.show_fullscreen_help:
-        a.setDuration(500)
+            f.setVisible(True)
-        a.setStartValue(QSize(width, 0))
+            a = self.full_screen_label_anim
-        a.setEndValue(QSize(width, height))
+            a.setDuration(500)
-        a.start()
+            a.setStartValue(QSize(width, 0))
-        QTimer.singleShot(3500, self.full_screen_label.hide)
+            a.setEndValue(QSize(width, height))
+            a.start()
+            QTimer.singleShot(3500, self.full_screen_label.hide)
         self.view.document.switch_to_fullscreen_mode()
         if self.view.document.fullscreen_clock:
             self.show_clock()
@@ -591,8 +592,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             fs = self.window_mode_changed == 'fullscreen'
             self.window_mode_changed = None
             if fs:
-                if self.view.document.show_fullscreen_help:
+                self.show_full_screen_label()
-                    self.show_full_screen_label()
             else:
                 self.view.document.switch_to_window_mode()
             self.view.document.page_position.restore()

src/calibre/library/catalogs/__init__.py

@@ -19,4 +19,5 @@ TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate', '
 
 class AuthorSortMismatchException(Exception): pass
 class EmptyCatalogException(Exception): pass
+class InvalidGenresSourceFieldException(Exception): pass
 

src/calibre/library/catalogs/epub_mobi.py

@@ -121,6 +121,13 @@ class EPUB_MOBI(CatalogPlugin):
                           help=_("Include 'Recently Added' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: AZW3, ePub, MOBI output formats")),
+                   Option('--genre-source-field',
+                          default='Tags',
+                          dest='genre_source_field',
+                          action = None,
+                          help=_("Source field for Genres section.\n"
+                          "Default: '%default'\n"
+                          "Applies to: AZW3, ePub, MOBI output formats")),
                    Option('--header-note-source-field',
                           default='',
                           dest='header_note_source_field',
@@ -327,7 +334,7 @@ class EPUB_MOBI(CatalogPlugin):
             if key in ['catalog_title','author_clip','connected_kindle','creator',
                        'cross_reference_authors','description_clip','exclude_book_marker',
                        'exclude_genre','exclude_tags','exclusion_rules', 'fmt',
-                       'header_note_source_field','merge_comments_rule',
+                       'genre_source_field', 'header_note_source_field','merge_comments_rule',
                        'output_profile','prefix_rules','read_book_marker',
                        'search_text','sort_by','sort_descriptions_by_author','sync',
                        'thumb_width','use_existing_cover','wishlist_tag']:

src/calibre/library/catalogs/epub_mobi_builder.py

@@ -15,7 +15,8 @@ from calibre.customize.ui import output_profiles
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString
 from calibre.ebooks.chardet import substitute_entites
 from calibre.ebooks.metadata import author_to_author_sort
-from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException
+from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException, \
+                                     InvalidGenresSourceFieldException
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.config import config_dir
 from calibre.utils.date import format_date, is_date_undefined, now as nowf
@@ -134,7 +135,7 @@ class CatalogBuilder(object):
         self.generate_recently_read = False
         self.genres = []
         self.genre_tags_dict = \
-            self.filter_db_tags(max_len = 245 - len("%s/Genre_.html" % self.content_dir)) \
+            self.filter_genre_tags(max_len = 245 - len("%s/Genre_.html" % self.content_dir)) \
             if self.opts.generate_genres else None
         self.html_filelist_1 = []
         self.html_filelist_2 = []
@@ -938,6 +939,21 @@ class CatalogBuilder(object):
                 this_title['tags'] = self.filter_excluded_genres(record['tags'],
                                         self.opts.exclude_genre)
 
+            this_title['genres'] = []
+            if self.opts.genre_source_field == _('Tags'):
+                this_title['genres'] = this_title['tags']
+            else:
+                record_genres = self.db.get_field(record['id'],
+                                    self.opts.genre_source_field,
+                                    index_is_id=True)
+
+                if record_genres:
+                    if type(record_genres) is not list:
+                        record_genres = [record_genres]
+
+                    this_title['genres'] = self.filter_excluded_genres(record_genres,
+                                            self.opts.exclude_genre)
+
             if record['formats']:
                 formats = []
                 for format in record['formats']:
@@ -1104,7 +1120,7 @@ class CatalogBuilder(object):
 
             self.bookmarked_books = bookmarks
 
-    def filter_db_tags(self, max_len):
+    def filter_genre_tags(self, max_len):
         """ Remove excluded tags from data set, return normalized genre list.
 
         Filter all db tags, removing excluded tags supplied in opts.
@@ -1166,7 +1182,32 @@ class CatalogBuilder(object):
         normalized_tags = []
         friendly_tags = []
         excluded_tags = []
-        for tag in self.db.all_tags():
+
+        # Fetch all possible genres from source field
+        all_genre_tags = []
+        if self.opts.genre_source_field == _('Tags'):
+            all_genre_tags = self.db.all_tags()
+        else:
+            # Validate custom field is usable as a genre source
+            field_md = self.db.metadata_for_field(self.opts.genre_source_field)
+            if not field_md['datatype'] in ['enumeration','text']:
+                all_custom_fields = self.db.custom_field_keys()
+                eligible_custom_fields = []
+                for cf in all_custom_fields:
+                    if self.db.metadata_for_field(cf)['datatype'] in ['enumeration','text']:
+                        eligible_custom_fields.append(cf)
+                self.opts.log.error("Custom genre_source_field must be either:\n"
+                                    " 'Comma separated text, like tags, shown in the browser',\n"
+                                    " 'Text, column shown in the tag browser', or\n"
+                                    " 'Text, but with a fixed set of permitted values'.")
+                self.opts.log.error("Eligible custom fields: %s" % ', '.join(eligible_custom_fields))
+                raise InvalidGenresSourceFieldException, "invalid custom field specified for genre_source_field"
+
+            all_genre_tags = list(self.db.all_custom(self.db.field_metadata.key_to_label(self.opts.genre_source_field)))
+
+        all_genre_tags.sort()
+
+        for tag in all_genre_tags:
             if tag in self.excluded_tags:
                 excluded_tags.append(tag)
                 continue
@@ -1194,9 +1235,10 @@ class CatalogBuilder(object):
                     if genre_tags_dict[key] == normalized:
                         self.opts.log.warn("       %s" % key)
         if self.opts.verbose:
-            self.opts.log.info('%s' % _format_tag_list(genre_tags_dict, header="enabled genre tags in database"))
+            self.opts.log.info('%s' % _format_tag_list(genre_tags_dict, header="enabled genres"))
-            self.opts.log.info('%s' % _format_tag_list(excluded_tags, header="excluded genre tags"))
+            self.opts.log.info('%s' % _format_tag_list(excluded_tags, header="excluded genres"))
 
+        print("genre_tags_dict: %s" % genre_tags_dict)
         return genre_tags_dict
 
     def filter_excluded_genres(self, tags, regex):
@@ -1969,7 +2011,7 @@ class CatalogBuilder(object):
         create a separate HTML file. Normalize tags to flatten synonymous tags.
 
         Inputs:
-         db.all_tags() (list): all database tags
+         self.genre_tags_dict (list): all genre tags
 
         Output:
          (files): HTML file per genre
@@ -1987,7 +2029,7 @@ class CatalogBuilder(object):
             tag_list = {}
             for book in self.books_by_author:
                 # Scan each book for tag matching friendly_tag
-                if 'tags' in book and friendly_tag in book['tags']:
+                if 'genres' in book and friendly_tag in book['genres']:
                     this_book = {}
                     this_book['author'] = book['author']
                     this_book['title'] = book['title']
@@ -2577,18 +2619,18 @@ class CatalogBuilder(object):
 
         # Genres
         genres = ''
-        if 'tags' in book:
+        if 'genres' in book:
             _soup = BeautifulSoup('')
             genresTag = Tag(_soup,'p')
             gtc = 0
-            for (i, tag) in enumerate(sorted(book.get('tags', []))):
+            for (i, tag) in enumerate(sorted(book.get('genres', []))):
                 aTag = Tag(_soup,'a')
                 if self.opts.generate_genres:
                     aTag['href'] = "Genre_%s.html" % self.genre_tags_dict[tag]
                 aTag.insert(0,escape(NavigableString(tag)))
                 genresTag.insert(gtc, aTag)
                 gtc += 1
-                if i < len(book['tags'])-1:
+                if i < len(book['genres'])-1:
                     genresTag.insert(gtc, NavigableString(' &middot; '))
                     gtc += 1
             genres = genresTag.renderContents()
@@ -4382,7 +4424,7 @@ class CatalogBuilder(object):
         """ Return the first friendly_tag matching genre.
 
         Scan self.genre_tags_dict[] for first friendly_tag matching genre.
-        genre_tags_dict[] populated in filter_db_tags().
+        genre_tags_dict[] populated in filter_genre_tags().
 
         Args:
          genre (str): genre to match

src/calibre/library/cli.py

@@ -215,7 +215,7 @@ class DevNull(object):
 NULL = DevNull()
 
 def do_add(db, paths, one_book_per_directory, recurse, add_duplicates, otitle,
-        oauthors, oisbn, otags, oseries, oseries_index):
+        oauthors, oisbn, otags, oseries, oseries_index, ocover):
     orig = sys.stdout
     #sys.stdout = NULL
     try:
@@ -247,6 +247,8 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates, otitle,
                 if val: setattr(mi, x, val)
             if oseries:
                 mi.series_index = oseries_index
+            if ocover:
+                mi.cover = ocover
 
             formats.append(format)
             metadata.append(mi)
@@ -335,11 +337,12 @@ the directory related options below.
             help=_('Set the series of the added book(s)'))
     parser.add_option('-S', '--series-index', default=1.0, type=float,
             help=_('Set the series number of the added book(s)'))
-
+    parser.add_option('-c', '--cover', default=None,
+            help=_('Path to the cover to use for the added book'))
 
     return parser
 
-def do_add_empty(db, title, authors, isbn, tags, series, series_index):
+def do_add_empty(db, title, authors, isbn, tags, series, series_index, cover):
     from calibre.ebooks.metadata import MetaInformation
     mi = MetaInformation(None)
     if title is not None:
@@ -352,6 +355,8 @@ def do_add_empty(db, title, authors, isbn, tags, series, series_index):
         mi.tags = tags
     if series:
         mi.series, mi.series_index = series, series_index
+    if cover:
+        mi.cover = cover
     db.import_book(mi, [])
     write_dirtied(db)
     send_message()
@@ -364,7 +369,7 @@ def command_add(args, dbpath):
     tags = [x.strip() for x in opts.tags.split(',')] if opts.tags else []
     if opts.empty:
         do_add_empty(get_db(dbpath, opts), opts.title, aut, opts.isbn, tags,
-                opts.series, opts.series_index)
+                opts.series, opts.series_index, opts.cover)
         return 0
     if len(args) < 2:
         parser.print_help()
@@ -373,7 +378,7 @@ def command_add(args, dbpath):
         return 1
     do_add(get_db(dbpath, opts), args[1:], opts.one_book_per_directory,
             opts.recurse, opts.duplicates, opts.title, aut, opts.isbn,
-            tags, opts.series, opts.series_index)
+            tags, opts.series, opts.series_index, opts.cover)
     return 0
 
 def do_remove(db, ids):

src/calibre/library/save_to_disk.py

@@ -20,6 +20,7 @@ from calibre.ebooks.metadata import title_sort
 from calibre.utils.date import parse_date, as_local_time
 from calibre import strftime, prints, sanitize_file_name_unicode
 from calibre.ptempfile import SpooledTemporaryFile
+from calibre.db.lazy import FormatsList
 
 plugboard_any_device_value = 'any device'
 plugboard_any_format_value = 'any format'
@@ -159,7 +160,7 @@ class Formatter(TemplateFormatter):
                 return self.composite_values[key]
             if key in kwargs:
                 val = kwargs[key]
-                if isinstance(val, list):
+                if isinstance(val, list) or isinstance(val, FormatsList):
                     val = ','.join(val)
                 return val.replace('/', '_').replace('\\', '_')
             return ''

src/calibre/libunrar.py

@@ -1,292 +0,0 @@
-from __future__ import with_statement
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-"""
-This module provides a thin ctypes based wrapper around libunrar.
-
-See  ftp://ftp.rarlabs.com/rar/unrarsrc-3.7.5.tar.gz
-"""
-import os, ctypes, sys, re
-from ctypes import Structure as _Structure, c_char_p, c_uint, c_void_p, POINTER, \
-                    byref, c_wchar_p, c_int, c_char, c_wchar
-from tempfile import NamedTemporaryFile
-from StringIO import StringIO
-
-from calibre import iswindows, load_library, CurrentDir
-from calibre.ptempfile import TemporaryDirectory, PersistentTemporaryFile
-
-_librar_name = 'libunrar'
-cdll = ctypes.cdll
-if iswindows:
-    class Structure(_Structure):
-        _pack_ = 1
-    _librar_name = 'unrar'
-    cdll = ctypes.windll
-else:
-    Structure = _Structure
-if hasattr(sys, 'frozen') and iswindows:
-    lp = os.path.join(os.path.dirname(sys.executable), 'DLLs', 'unrar.dll')
-    _libunrar = cdll.LoadLibrary(lp)
-elif hasattr(sys, 'frozen_path'):
-    lp = os.path.join(sys.frozen_path, 'lib', 'libunrar.so')
-    _libunrar = cdll.LoadLibrary(lp)
-else:
-    _libunrar = load_library(_librar_name, cdll)
-
-RAR_OM_LIST    = 0
-RAR_OM_EXTRACT = 1
-
-ERAR_END_ARCHIVE      =  10
-ERAR_NO_MEMORY        =  11
-ERAR_BAD_DATA         =  12
-ERAR_BAD_ARCHIVE      =  13
-ERAR_UNKNOWN_FORMAT   =  14
-ERAR_EOPEN            =  15
-ERAR_ECREATE          =  16
-ERAR_ECLOSE           =  17
-ERAR_EREAD            =  18
-ERAR_EWRITE           =  19
-ERAR_SMALL_BUF        =  20
-ERAR_UNKNOWN          =  21
-ERAR_MISSING_PASSWORD =  22
-
-RAR_VOL_ASK           = 0
-RAR_VOL_NOTIFY        = 1
-
-RAR_SKIP              = 0
-RAR_TEST              = 1
-RAR_EXTRACT           = 2
-
-class UnRARException(Exception):
-    pass
-
-class RAROpenArchiveDataEx(Structure):
-    _fields_ = [
-                ('ArcName', c_char_p),
-                ('ArcNameW', c_wchar_p),
-                ('OpenMode', c_uint),
-                ('OpenResult', c_uint),
-                ('CmtBuf', c_char_p),
-                ('CmtBufSize', c_uint),
-                ('CmtSize', c_uint),
-                ('CmtState', c_uint),
-                ('Flags', c_uint),
-                ('Reserved', c_uint * 32)
-               ]
-
-class RARHeaderDataEx(Structure):
-    _fields_ = [
-                ('ArcName', c_char*1024),
-                ('ArcNameW', c_wchar*1024),
-                ('FileName', c_char*1024),
-                ('FileNameW', c_wchar*1024),
-                ('Flags', c_uint),
-                ('PackSize', c_uint),
-                ('PackSizeHigh', c_uint),
-                ('UnpSize', c_uint),
-                ('UnpSizeHigh', c_uint),
-                ('HostOS', c_uint),
-                ('FileCRC', c_uint),
-                ('FileTime', c_uint),
-                ('UnpVer', c_uint),
-                ('Method', c_uint),
-                ('FileAttr', c_uint),
-                ('CmtBuf', c_char_p),
-                ('CmtBufSize', c_uint),
-                ('CmtSize', c_uint),
-                ('CmtState', c_uint),
-                ('Reserved', c_uint*1024)
-                ]
-
-# Define a callback function
-#CALLBACK_FUNC = CFUNCTYPE(c_int, c_uint, c_long, c_char_p, c_long)
-#def py_callback_func(msg, user_data, p1, p2):
-#    return 0
-
-#callback_func = CALLBACK_FUNC(py_callback_func)
-
-_libunrar.RAROpenArchiveEx.argtypes = [POINTER(RAROpenArchiveDataEx)]
-_libunrar.RAROpenArchiveEx.restype  = c_void_p
-_libunrar.RARReadHeaderEx.argtypes  = [c_void_p, POINTER(RARHeaderDataEx)]
-_libunrar.RARReadHeaderEx.restype   = c_int
-_libunrar.RARProcessFileW.argtypes  = [c_void_p, c_int, c_wchar_p, c_wchar_p]
-_libunrar.RARProcessFileW.restype   = c_int
-_libunrar.RARCloseArchive.argtypes  = [c_void_p]
-_libunrar.RARCloseArchive.restype   = c_int
-_libunrar.RARSetPassword.argtypes   = [c_void_p, c_char_p]
-#_libunrar.RARSetCallback.argtypes   = [c_void_p,  CALLBACK_FUNC, c_long]
-
-
-def _interpret_open_error(code, path):
-    msg = 'Unknown error.'
-    if code == ERAR_NO_MEMORY:
-        msg = "Not enough memory to process " + path
-    elif code == ERAR_BAD_DATA:
-        msg = "Archive header broken: " + path
-    elif code == ERAR_BAD_ARCHIVE:
-        msg = path + ' is not a RAR archive.'
-    elif code == ERAR_EOPEN:
-        msg = 'Cannot open ' + path
-    return msg
-
-def _interpret_process_file_error(code):
-    msg = 'Unknown Error'
-    if code == ERAR_UNKNOWN_FORMAT:
-        msg = 'Unknown archive format'
-    elif  code == ERAR_BAD_ARCHIVE:
-        msg = 'Bad volume'
-    elif  code == ERAR_ECREATE:
-        msg = 'File create error'
-    elif  code == ERAR_EOPEN:
-        msg = 'Volume open error'
-    elif  code == ERAR_ECLOSE:
-        msg = 'File close error'
-    elif  code == ERAR_EREAD:
-        msg = 'Read error'
-    elif  code == ERAR_EWRITE:
-        msg = 'Write error'
-    elif  code == ERAR_BAD_DATA:
-        msg = 'CRC error'
-    elif  code == ERAR_MISSING_PASSWORD:
-        msg = 'Password is required.'
-    return msg
-
-def get_archive_info(flags):
-    ios = StringIO()
-    print >>ios, 'Volume:\t\t', 'yes' if (flags & 1) else 'no'
-    print >>ios, 'Comment:\t', 'yes' if (flags & 2) else 'no'
-    print >>ios, 'Locked:\t\t', 'yes' if (flags & 4) else 'no'
-    print >>ios, 'Solid:\t\t', 'yes' if (flags & 8) else 'no'
-    print >>ios, 'New naming:\t', 'yes' if (flags & 16) else 'no'
-    print >>ios, 'Authenticity:\t', 'yes' if (flags & 32) else 'no'
-    print >>ios, 'Recovery:\t', 'yes' if (flags & 64) else 'no'
-    print >>ios, 'Encr.headers:\t', 'yes' if (flags & 128) else 'no'
-    print >>ios, 'First Volume:\t', 'yes' if (flags & 256) else 'no or older than 3.0'
-    return ios.getvalue()
-
-def extract(path, dir):
-    """
-    Extract archive C{filename} into directory C{dir}
-    """
-    open_archive_data = RAROpenArchiveDataEx(ArcName=path, OpenMode=RAR_OM_EXTRACT, CmtBuf=None)
-    arc_data = _libunrar.RAROpenArchiveEx(byref(open_archive_data))
-    cwd = os.getcwdu()
-    if not os.path.isdir( dir ):
-        os.mkdir( dir )
-    os.chdir( dir )
-    try:
-        if open_archive_data.OpenResult != 0:
-            raise UnRARException(_interpret_open_error(open_archive_data.OpenResult, path))
-        #prints('Archive:', path)
-        #print get_archive_info(open_archive_data.Flags)
-        header_data = RARHeaderDataEx(CmtBuf=None)
-        #_libunrar.RARSetCallback(arc_data, callback_func, mode)
-        while True:
-            RHCode = _libunrar.RARReadHeaderEx(arc_data, byref(header_data))
-            if RHCode != 0:
-                break
-            PFCode = _libunrar.RARProcessFileW(arc_data, RAR_EXTRACT, None, None)
-            if PFCode != 0:
-                raise UnRARException(_interpret_process_file_error(PFCode))
-        if RHCode == ERAR_BAD_DATA:
-            raise UnRARException('File header broken')
-    finally:
-        os.chdir(cwd)
-        _libunrar.RARCloseArchive(arc_data)
-
-def names(path):
-    if hasattr(path, 'read'):
-        data = path.read()
-        f = NamedTemporaryFile(suffix='.rar')
-        f.write(data)
-        f.flush()
-        path = f.name
-    open_archive_data = RAROpenArchiveDataEx(ArcName=path, OpenMode=RAR_OM_LIST, CmtBuf=None)
-    arc_data = _libunrar.RAROpenArchiveEx(byref(open_archive_data))
-    try:
-        if open_archive_data.OpenResult != 0:
-            raise UnRARException(_interpret_open_error(open_archive_data.OpenResult, path))
-        header_data = RARHeaderDataEx(CmtBuf=None)
-        while True:
-            if _libunrar.RARReadHeaderEx(arc_data, byref(header_data)) != 0:
-                break
-            PFCode = _libunrar.RARProcessFileW(arc_data, RAR_SKIP, None, None)
-            if PFCode != 0:
-                raise UnRARException(_interpret_process_file_error(PFCode))
-            yield header_data.FileNameW
-    finally:
-        _libunrar.RARCloseArchive(arc_data)
-
-def _extract_member(path, match, name):
-
-    def is_match(fname):
-        return (name is not None and fname == name) or \
-               (match is not None and match.search(fname) is not None)
-
-    open_archive_data = RAROpenArchiveDataEx(ArcName=path, OpenMode=RAR_OM_EXTRACT, CmtBuf=None)
-    arc_data = _libunrar.RAROpenArchiveEx(byref(open_archive_data))
-    try:
-        if open_archive_data.OpenResult != 0:
-            raise UnRARException(_interpret_open_error(open_archive_data.OpenResult, path))
-        header_data = RARHeaderDataEx(CmtBuf=None)
-        first = True
-        while True:
-            if _libunrar.RARReadHeaderEx(arc_data, byref(header_data)) != 0:
-                raise UnRARException('%s has no files'%path if first
-                        else 'No match found in %s'%path)
-            file_name = header_data.FileNameW
-            if is_match(file_name):
-                PFCode = _libunrar.RARProcessFileW(arc_data, RAR_EXTRACT, None, None)
-                if PFCode != 0:
-                    raise UnRARException(_interpret_process_file_error(PFCode))
-                abspath = os.path.abspath(os.path.join(*file_name.split('/')))
-                return abspath
-            else:
-                PFCode = _libunrar.RARProcessFileW(arc_data, RAR_SKIP, None, None)
-                if PFCode != 0:
-                    raise UnRARException(_interpret_process_file_error(PFCode))
-            first = False
-
-    finally:
-        _libunrar.RARCloseArchive(arc_data)
-
-def extract_member(path, match=re.compile(r'\.(jpg|jpeg|gif|png)\s*$', re.I),
-        name=None, as_file=False):
-    if hasattr(path, 'read'):
-        data = path.read()
-        f = NamedTemporaryFile(suffix='.rar')
-        f.write(data)
-        f.flush()
-        path = f.name
-
-    path = os.path.abspath(path)
-    if as_file:
-        path = _extract_member(path, match, name)
-        return path, open(path, 'rb')
-    else:
-        with TemporaryDirectory('_libunrar') as tdir:
-            with CurrentDir(tdir):
-                path = _extract_member(path, match, name)
-                return path, open(path, 'rb').read()
-
-def extract_first_alphabetically(path):
-    remove_path = False
-    if hasattr(path, 'read'):
-        data = path.read()
-        with PersistentTemporaryFile('.rar') as f:
-            f.write(data)
-        path = f.name
-        remove_path = True
-
-    names_ = [x for x in names(path) if os.path.splitext(x)[1][1:].lower() in
-            ('png', 'jpg', 'jpeg', 'gif')]
-    names_.sort()
-    ans = extract_member(path, name=names_[0], match=None)
-    try:
-        if remove_path:
-            os.remove(path)
-    except:
-        pass
-    return ans
-
-

src/calibre/test_build.py

@@ -87,9 +87,8 @@ def test_imaging():
     print ('PIL OK!')
 
 def test_unrar():
-    from calibre.libunrar import _libunrar
+    from calibre.utils.unrar import test_basic
-    if not _libunrar:
+    test_basic()
-        raise RuntimeError('Failed to load libunrar')
     print ('Unrar OK!')
 
 def test_icu():