Merge branch 'master' of https://github.com/kovidgoyal/calibre

Changelog.yaml

@@ -20,6 +20,81 @@
 #   new recipes:
 #     - title: 
 
+- version: 1.6.0
+  date: 2013-10-11
+
+  new features:
+    - title: "Temporary marking of books in the library"
+      description: "This allows you to select books from your calibre library manually and mark them. This 'mark' will remain until you restart calibre, or clear the marks. You can easily work with only the marked subset of books by right clicking the Mark Books button. To use this feature, go to Preferences->Toolbars and add the 'Mark Books' tool to the main toolbar."
+      type: major
+
+    - title: "Get Books: Add Wolne Lektury and Amazon (Canada) ebook stores"
+
+    - title: "DOCX Input: Handle hyperlinks in footnotes and endnotes"
+      tickets: [1232790]
+
+    - title: "Driver for Sunstech reader"
+      tickets: [1231590]
+
+    - title: "Allow using both uri: and url: identifiers to create two different arbitrary links instead of just one in the Book details panel"
+
+    - title: "E-book viewer: Make all keyboard shortcuts configurable"
+      tickets: [1232019]
+
+    - title: "Conversion: Add an option to not condense CSS rules for margin, padding, border, etc. Option is under the Look & Feel section of the conversion dialog."
+      tickets: [1233220]
+
+    - title: "calibredb: Allow setting of title sort field"
+      tickets: [1233711]
+
+    - title: "ebook-meta: Add an --identifier option to set identifiers."
+ 
+  bug fixes:
+    - title: "Fix a locking error when composite columns containing formats are used and formats are added/deleted."
+      tickets: [1233330]
+
+    - title: "EPUB Output: Do not strip <object> tags with type application/svg+xml in addition to those that use image/svg+xml."
+      tickets: [1236845]
+
+    - title: "Cover grid: Fix selecting all books with Ctrl+A causing subsequent deselects to not fully work."
+      tickets: [1236348]
+
+    - title: "HTMLZ Output: Fix long titles causing error when converting on windows."
+      tickets: [1235815]
+
+    - title: "Content server: Fix OPDS category links to composite columns"
+
+    - title: "E-book viewer: Fix regression that broke import/export of bookmarks"
+      tickets: [1231980]
+
+    - title: "E-book viewer: Use the default font size setting for the dictionary view as well."
+      tickets: [1232025]
+
+    - title: "DOCX Input: Avoid using the value attribute for simple numbered lists, to silence the asinine epubcheck"
+
+    - title: "HTML Input: Images linked by the poster attribute of the <video> tag are now recognized and processed."
+
+    - title: "DOCX Input: Fix erorr when converting docx files that have numbering defined with no associated character style."
+      tickets: [1232100]
+
+    - title: "EPUB Metadata: Implementing updating identifiers other than isbn in the epub file from calibre when polishing or exporting the epub"
+
+    - title: "Amazon metadata download: Fix parsing of some dates on amazon.de"
+      tickets: [1238125]
+
+  improved recipes:
+    - National Geographic Magazine
+    - New York Review of Books
+    - Focus (PL)
+    - Carta Capital
+    - AM 730 
+    - Ming Pao (HK)
+    - Neu Osnabrucker Zeitung
+
+  new recipes:
+    - title: Various Uruguayan news sources 
+      author: Carlos Alves
+     
 - version: 1.5.0
   date: 2013-09-26
 

imgsrc/marked.svg

@@ -0,0 +1,162 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="128"
+   height="128"
+   id="svg2"
+   version="1.1"
+   inkscape:version="0.48.4 r9939"
+   sodipodi:docname="marked.svg"
+   inkscape:export-filename="/home/kovid/work/calibre/resources/images/marked.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <title
+     id="title3847">Pushpin Icon</title>
+  <defs
+     id="defs4">
+    <linearGradient
+       id="linearGradient3782">
+      <stop
+         style="stop-color:#000000;stop-opacity:1;"
+         offset="0"
+         id="stop3784" />
+      <stop
+         style="stop-color:#c3c3c0;stop-opacity:1;"
+         offset="1"
+         id="stop3786" />
+    </linearGradient>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient3782"
+       id="linearGradient3813"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(0.70710678,-0.70710678,0.70710678,0.70710678,-18.805519,996.21376)"
+       x1="58"
+       y1="91"
+       x2="73"
+       y2="91" />
+    <filter
+       id="filter3014"
+       inkscape:label="Ridged border"
+       inkscape:menu="Bevels"
+       inkscape:menu-tooltip="Ridged border with inner bevel"
+       color-interpolation-filters="sRGB">
+      <feMorphology
+         id="feMorphology3016"
+         radius="4.3"
+         in="SourceAlpha"
+         result="result91" />
+      <feComposite
+         id="feComposite3018"
+         in2="result91"
+         operator="out"
+         in="SourceGraphic" />
+      <feGaussianBlur
+         id="feGaussianBlur3020"
+         result="result0"
+         stdDeviation="1.2" />
+      <feDiffuseLighting
+         id="feDiffuseLighting3022"
+         diffuseConstant="1"
+         result="result92">
+        <feDistantLight
+           id="feDistantLight3024"
+           elevation="66"
+           azimuth="225" />
+      </feDiffuseLighting>
+      <feBlend
+         id="feBlend3026"
+         in2="SourceGraphic"
+         mode="multiply"
+         result="result93" />
+      <feComposite
+         id="feComposite3028"
+         in2="SourceAlpha"
+         operator="in" />
+    </filter>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="5.6568542"
+     inkscape:cx="30.580486"
+     inkscape:cy="63.624717"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="true"
+     inkscape:snap-smooth-nodes="false"
+     inkscape:window-width="1920"
+     inkscape:window-height="1058"
+     inkscape:window-x="0"
+     inkscape:window-y="22"
+     inkscape:window-maximized="0"
+     inkscape:snap-bbox="false"
+     inkscape:object-paths="true"
+     inkscape:snap-midpoints="false"
+     inkscape:snap-global="true">
+    <inkscape:grid
+       empspacing="5"
+       visible="true"
+       enabled="true"
+       snapvisiblegridlinesonly="true"
+       type="xygrid"
+       id="grid2985" />
+  </sodipodi:namedview>
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title>Pushpin Icon</dc:title>
+        <dc:creator>
+          <cc:Agent>
+            <dc:title>Kovid Goyal</dc:title>
+          </cc:Agent>
+        </dc:creator>
+        <dc:rights>
+          <cc:Agent>
+            <dc:title>Public domain</dc:title>
+          </cc:Agent>
+        </dc:rights>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-924.36218)">
+    <path
+       style="fill:#f39509;fill-opacity:1;stroke:#7a6822;stroke-opacity:1;stroke-width:0;stroke-miterlimit:4;stroke-dasharray:none;filter:url(#filter3014)"
+       d="m 1.9128912,974.70018 49.4974748,-49.49747 -7.071068,21.2132 31.819805,17.67767 24.433067,-3.85121 -63.639613,63.63963 3.851207,-24.43308 -17.677669,-31.81981 z"
+       id="path3088"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="ccccccccc"
+       inkscape:export-xdpi="90"
+       inkscape:export-ydpi="90" />
+    <path
+       style="fill:url(#linearGradient3813);fill-opacity:1;stroke:none"
+       d="M 63.925974,996.92087 120,1042.5389 74.532576,986.31427"
+       id="path3097"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="ccc"
+       inkscape:export-xdpi="90"
+       inkscape:export-ydpi="90" />
+  </g>
+</svg>

manual/server.rst

@@ -104,7 +104,7 @@ Save this adapter as :file:`calibre-wsgi-adpater.py` somewhere your server will
 
 Let's suppose that we want to use WSGI in Apache. First enable WSGI in Apache by adding the following to :file:`httpd.conf`::
 
-    LoadModule proxy_module modules/mod_wsgi.so
+    LoadModule wsgi_module modules/mod_wsgi.so
 
 The exact technique for enabling the wsgi module will vary depending on your Apache installation. Once you have the proxy modules enabled, add the following rules to httpd.conf (or if you are using virtual hosts to the conf file for the virtual host in question::
 

recipes/10minutos.recipe

@@ -0,0 +1,50 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__author__ = '2013, Carlos Alves <carlosalves90@gmail.com>'
+'''
+10minutos.com.uy
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class General(BasicNewsRecipe):
+    title                 = '10minutos'
+    __author__            = 'Carlos Alves'
+    description           = 'Noticias de Salto - Uruguay'
+    tags = 'news, sports'
+    language       = 'es_UY'
+    timefmt        = '[%a, %d %b, %Y]'
+    use_embedded_content  = False
+    recursion             = 5
+    encoding = 'utf8'
+    remove_javascript = True
+    no_stylesheets = True
+
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    keep_only_tags = [dict(name='div', attrs={'class':'post-content'})]
+
+    remove_tags = [
+             dict(name='div', attrs={'class':['hr', 'titlebar', 'navigation']}),
+             dict(name='p', attrs={'class':'post-meta'}),
+             dict(name=['object','link'])
+                  ]
+
+    extra_css = '''
+                h1{font-family:Geneva, Arial, Helvetica, sans-serif;color:#154B7A;}
+                h3{font-size: 14px;color:#999999; font-family:Geneva, Arial, Helvetica, sans-serif;font-weight: bold;}
+                h2{color:#666666; font-family:Geneva, Arial, Helvetica, sans-serif;font-size:small;}
+                p {font-family:Arial,Helvetica,sans-serif;}
+                '''
+    feeds = [
+           (u'Articulos', u'http://10minutos.com.uy/feed/')
+        ]
+
+    def get_cover_url(self):
+        return 'http://10minutos.com.uy/a/img/logo.png'
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

recipes/am730.recipe

@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 __license__   = 'GPL v3'
 __copyright__ = '2013, Eddie Lau'
 __Date__ = ''
-__HiResImg__ = True
 
 '''
 Change Log:
+2013/09/28 -- update due to website redesign, add cover
 2013/03/30 -- first version
 '''
 
@@ -15,7 +15,7 @@ from calibre.utils.date import now as nowf
 import os, datetime, re
 from calibre.web.feeds.recipes import BasicNewsRecipe
 from contextlib import nested
-from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata import MetaInformation
@@ -32,18 +32,17 @@ class AppleDaily(BasicNewsRecipe):
     encoding = 'utf-8'
     auto_cleanup = False
     remove_javascript = True
-    use_embedded_content   = False
+    use_embedded_content = False
     no_stylesheets = True
     description = 'http://www.am730.com.hk'
     category    = 'Chinese, News, Hong Kong'
     masthead_url = 'http://www.am730.com.hk/images/logo.jpg'
-
+    extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 20px; margin-bottom: 20px; max-height:70%;} div[id=articleHeader] {font-size:200%; text-align:left; font-weight:bold;} li {font-size:50%; margin-left:auto; margin-right:auto;}'
-    extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px; max-height:90%;} div[id=articleHeader] {font-size:200%; text-align:left; font-weight:bold;} photocaption {font-size:50%; margin-left:auto; margin-right:auto;}'
+    keep_only_tags = [dict(name='h2', attrs={'class':'printTopic'}),
-    keep_only_tags = [dict(name='div', attrs={'id':'articleHeader'}),
+                      dict(name='div', attrs={'id':'article_content'}),
-                      dict(name='div', attrs={'class':'thecontent wordsnap'}),
+                      dict(name='div', attrs={'id':'slider'})]
-                      dict(name='a', attrs={'class':'lightboximg'})]
+    remove_tags = [dict(name='img', attrs={'src':'images/am730_article_logo.jpg'}),
-    remove_tags = [dict(name='img', attrs={'src':'/images/am730_article_logo.jpg'}),
+                   dict(name='img', attrs={'src':'images/am_endmark.gif'})]
-                   dict(name='img', attrs={'src':'/images/am_endmark.gif'})]
 
     def get_dtlocal(self):
         dt_utc = datetime.datetime.utcnow()
@@ -84,6 +83,16 @@ class AppleDaily(BasicNewsRecipe):
     def get_weekday(self):
         return self.get_dtlocal().weekday()
 
+    def get_cover_url(self):
+        soup = self.index_to_soup('http://www.am730.com.hk')
+        cover = 'http://www.am730.com.hk/' + soup.find(attrs={'id':'mini_news_img'}).find('img').get('src', False)
+        br = BasicNewsRecipe.get_browser(self)
+        try:
+            br.open(cover)
+        except:
+            cover = None
+        return cover
+
     def populate_article_metadata(self, article, soup, first):
         if first and hasattr(self, 'add_toc_thumbnail'):
             picdiv = soup.find('img')
@@ -93,48 +102,17 @@ class AppleDaily(BasicNewsRecipe):
     def parse_index(self):
         feeds = []
         soup = self.index_to_soup('http://www.am730.com.hk/')
-        ul = soup.find(attrs={'class':'nav-section'})
+        optgroups = soup.findAll('optgroup')
-        sectionList = []
+        for optgroup in optgroups:
-        for li in ul.findAll('li'):
+            sectitle = optgroup.get('label')
-            a = 'http://www.am730.com.hk/' + li.find('a', href=True).get('href', False)
+            articles = []
-            title = li.find('a').get('title', False).strip()
+            for option in optgroup.findAll('option'):
-            sectionList.append((title, a))
+                articlelink = "http://www.am730.com.hk/" + option.get('value')
-        for title, url in sectionList:
+                title = option.string
-            articles = self.parse_section(url)
+                articles.append({'title': title, 'url': articlelink})
-            if articles:
+            feeds.append((sectitle, articles))
-                feeds.append((title, articles))
         return feeds
 
-    def parse_section(self, url):
-        soup = self.index_to_soup(url)
-        items = soup.findAll(attrs={'style':'padding-bottom: 15px;'})
-        current_articles = []
-        for item in items:
-            a = item.find(attrs={'class':'t6 f14'}).find('a', href=True)
-            articlelink = 'http://www.am730.com.hk/' + a.get('href', True)
-            title = self.tag_to_string(a)
-            description = self.tag_to_string(item.find(attrs={'class':'t3 f14'}))
-            current_articles.append({'title': title, 'url': articlelink, 'description': description})
-        return current_articles
-
-    def preprocess_html(self, soup):
-        multia = soup.findAll('a')
-        for a in multia:
-            if not (a == None):
-                image = a.find('img')
-                if not (image == None):
-                    if __HiResImg__:
-                        image['src'] = image.get('src').replace('/thumbs/', '/')
-                    caption = image.get('alt')
-                    tag = Tag(soup, "photo", [])
-                    tag2 = Tag(soup, "photocaption", [])
-                    tag.insert(0, image)
-                    if not caption == None:
-                        tag2.insert(0, caption)
-                        tag.insert(1, tag2)
-                    a.replaceWith(tag)
-        return soup
-
     def create_opf(self, feeds, dir=None):
         if dir is None:
             dir = self.output_dir
@@ -288,3 +266,4 @@ class AppleDaily(BasicNewsRecipe):
         with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file):
             opf.render(opf_file, ncx_file)
 
+

recipes/carta_capital.recipe

@@ -1,23 +1,29 @@
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class AdvancedUserRecipe1312361378(BasicNewsRecipe):
+class AdvancedUserRecipe1380852962(BasicNewsRecipe):
-    title          = u'Carta capital'
+    title          = u'Carta Capital'
-    __author__ = 'Pablo Aldama'
+    __author__ = 'Erico Lisboa'
     language = 'pt_BR'
-    oldest_article = 9
+    oldest_article = 15
     max_articles_per_feed = 100
+    auto_cleanup = True
+    use_embedded_content = False
 
-    feeds          = [(u'Politica', u'http://www.cartacapital.com.br/category/politica/feed')
+    feeds          = [(u'Pol\xedtica',
-                      ,(u'Economia', u'http://www.cartacapital.com.br/category/economia/feed')
+u'http://www.cartacapital.com.br/politica/politica/rss'), (u'Economia',
-                      ,(u'Cultura', u'http://www.cartacapital.com.br/category/cultura/feed')
+u'http://www.cartacapital.com.br/economia/economia/atom.xml'),
-                      ,(u'Internacional', u'http://www.cartacapital.com.br/category/internacional/feed')
+(u'Sociedade',
-                      ,(u'Saude', u'http://www.cartacapital.com.br/category/saude/feed')
+u'http://www.cartacapital.com.br/sociedade/sociedade/atom.xml'),
-                      ,(u'Sociedade', u'http://www.cartacapital.com.br/category/sociedade/feed')
+(u'Internacional',
-                      ,(u'Tecnologia', u'http://www.cartacapital.com.br/category/tecnologia/feed')
+u'http://www.cartacapital.com.br/internacional/internacional/atom.xml'),
-                      ,(u'Carta na escola', u'http://www.cartacapital.com.br/category/carta-na-escola/feed')
+(u'Tecnologia',
-                      ,(u'Carta fundamental', u'http://www.cartacapital.com.br/category/carta-fundamental/feed')
+u'http://www.cartacapital.com.br/tecnologia/tecnologia/atom.xml'),
-                      ,(u'Carta verde', u'http://www.cartacapital.com.br/category/carta-verde/feed')
+(u'Cultura',
-
+u'http://www.cartacapital.com.br/cultura/cultura/atom.xml'),
-]
+(u'Sa\xfade', u'http://www.cartacapital.com.br/saude/saude/atom.xml'),
-    def print_version(self, url):
+(u'Educa\xe7\xe3o',
-        return url + '/print'
+u'http://www.cartacapital.com.br/educacao/educacao/atom.xml')]

recipes/diario_el_pueblo.recipe

@@ -0,0 +1,51 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__author__ = '2013, Carlos Alves <carlosalves90@gmail.com>'
+'''
+diarioelpueblo.com.uy
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class General(BasicNewsRecipe):
+    title                 = 'Diario El Pueblo'
+    __author__            = 'Carlos Alves'
+    description           = 'Noticias de Salto - Uruguay'
+    tags = 'news, sports'
+    language       = 'es_UY'
+    timefmt        = '[%a, %d %b, %Y]'
+    use_embedded_content  = False
+    recursion             = 5
+    encoding = 'utf8'
+    remove_javascript = True
+    no_stylesheets = True
+
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    keep_only_tags = [dict(name='div', attrs={'class':'post-alt blog'})]
+
+    remove_tags = [
+             dict(name='div', attrs={'class':['hr', 'titlebar', 'volver-arriba-right','navigation']}),
+             dict(name='div', attrs={'id':'comment','id':'suckerfish','id':'crp_related'}),
+             dict(name='h3', attrs={'class':['post_date']}),
+             dict(name=['object','link'])
+                  ]
+
+    extra_css = '''
+                h1{font-family:Geneva, Arial, Helvetica, sans-serif;color:#154B7A;}
+                h3{font-size: 14px;color:#999999; font-family:Geneva, Arial, Helvetica, sans-serif;font-weight: bold;}
+                h2{color:#666666; font-family:Geneva, Arial, Helvetica, sans-serif;font-size:small;}
+                p {font-family:Arial,Helvetica,sans-serif;}
+                '''
+    feeds = [
+           (u'Articulos', u'http://www.diarioelpueblo.com.uy/feed')
+        ]
+
+    def get_cover_url(self):
+        return 'http://www.diarioelpueblo.com.uy/wp-content/uploads/2013/06/Cabezal_Web1.jpg'
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

recipes/diario_salto.recipe

@@ -0,0 +1,50 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__author__ = '2013, Carlos Alves <carlosalves90@gmail.com>'
+'''
+diarisalto.com.uy
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class General(BasicNewsRecipe):
+    title                 = 'Diario Salto'
+    __author__            = 'Carlos Alves'
+    description           = 'Noticias de Salto - Uruguay'
+    tags = 'news, sports'
+    language       = 'es_UY'
+    timefmt        = '[%a, %d %b, %Y]'
+    use_embedded_content  = False
+    recursion             = 5
+    encoding = 'utf8'
+    remove_javascript = True
+    no_stylesheets = True
+
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    keep_only_tags = [dict(name='div', attrs={'class':'post'})]
+
+    remove_tags = [
+             dict(name='div', attrs={'class':['hr', 'titlebar', 'navigation']}),
+             dict(name='div', attrs={'id':'comment'}),
+             dict(name=['object','link'])
+                  ]
+
+    extra_css = '''
+                h1{font-family:Geneva, Arial, Helvetica, sans-serif;color:#154B7A;}
+                h3{font-size: 14px;color:#999999; font-family:Geneva, Arial, Helvetica, sans-serif;font-weight: bold;}
+                h2{color:#666666; font-family:Geneva, Arial, Helvetica, sans-serif;font-size:small;}
+                p {font-family:Arial,Helvetica,sans-serif;}
+                '''
+    feeds = [
+           (u'Articulos', u'http://www.diariosalto.com.uy/feed/atom')
+        ]
+
+    def get_cover_url(self):
+        return 'http://diariosalto.com.uy/demo/wp-content/uploads/2011/12/diario-salto_logo-final-b-b.png'
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

recipes/el_observador.recipe

@@ -1,18 +1,23 @@
 #!/usr/bin/env  python
+##
+## Last Edited:  2013-09-29 Carlos Alves <carlosalves90@gmail.com>
+##
 
 __license__   = 'GPL v3'
 __author__ = '2010, Yuri Alvarez<me at yurialvarez.com>'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+
 '''
-observa.com.uy
+elobservador.com.uy
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class ObservaDigital(BasicNewsRecipe):
+class Noticias(BasicNewsRecipe):
-    title                 = 'Observa Digital'
+    title                 = 'El Observador'
-    __author__            = 'yrvn'
+    __author__ = 'yrvn'
-    description           = 'Noticias de Uruguay'
+    description           = 'Noticias desde Uruguay'
+    tags = 'news, sports, entretainment'
     language       = 'es_UY'
     timefmt        = '[%a, %d %b, %Y]'
     use_embedded_content  = False
@@ -23,13 +28,18 @@ class ObservaDigital(BasicNewsRecipe):
 
     oldest_article        = 2
     max_articles_per_feed = 100
-    keep_only_tags = [dict(id=['contenido'])]
+    keep_only_tags = [
+             dict(name='div', attrs={'class':'story collapsed'})
+             ]
     remove_tags = [
-                     dict(name='div', attrs={'id':'contenedorVinculadas'}),
+             dict(name='div', attrs={'class':['fecha', 'copyright', 'story_right']}),
-             dict(name='p', attrs={'id':'nota_firma'}),
+             dict(name='div', attrs={'class':['photo', 'social']}),
+             dict(name='div', attrs={'id':'widget'}),
              dict(name=['object','link'])
                   ]
 
+    remove_attributes = ['width','height', 'style', 'font', 'color']
+
     extra_css = '''
                 h1{font-family:Geneva, Arial, Helvetica, sans-serif;color:#154B7A;}
                 h3{font-size: 14px;color:#999999; font-family:Geneva, Arial, Helvetica, sans-serif;font-weight: bold;}
@@ -37,19 +47,9 @@ class ObservaDigital(BasicNewsRecipe):
                 p {font-family:Arial,Helvetica,sans-serif;}
                 '''
     feeds = [
-               (u'Actualidad', u'http://www.observa.com.uy/RSS/actualidad.xml'),
+           (u'Portada', u'http://elobservador.com.uy/rss/portada/'),
-           (u'Deportes', u'http://www.observa.com.uy/RSS/deportes.xml'),
-           (u'Vida', u'http://www.observa.com.uy/RSS/vida.xml'),
-           (u'Ciencia y Tecnologia', u'http://www.observa.com.uy/RSS/ciencia.xml')
         ]
 
-    def get_cover_url(self):
-        index = 'http://www.observa.com.uy/'
-        soup = self.index_to_soup(index)
-        for image in soup.findAll('img',alt=True):
-           if image['alt'].startswith('Tapa El Observador'):
-              return image['src'].rstrip('b.jpg') + '.jpg'
-        return None
 
     def preprocess_html(self, soup):
         for item in soup.findAll(style=True):

recipes/focus_pl.recipe

@@ -1,85 +1,51 @@
-#!/usr/bin/env python
+from calibre.web.feeds.recipes import BasicNewsRecipe
-__license__ = 'GPL v3'
 
-import re
-from calibre.web.feeds.news import BasicNewsRecipe
 
-class FocusRecipe(BasicNewsRecipe):
+class NYTimes(BasicNewsRecipe):
 
-    __author__ = u'Artur Stachecki <artur.stachecki@gmail.com>'
+    title = 'Focus'
+    __author__ = 'Krittika Goyal'
     language = 'pl'
-    version = 1
+    description = 'Polish scientific monthly magazine'
-
+    timefmt = ' [%d %b, %Y]'
-    title = u'Focus'
+    needs_subscription = False
-    publisher = u'Gruner + Jahr Polska'
-    category = u'News'
-    description = u'Focus.pl - pierwszy w Polsce portal społecznościowy dla miłośników nauki. Tematyka: nauka, historia, cywilizacja, technika, przyroda, sport, gadżety'
-    category = 'magazine'
-    cover_url = ''
-    remove_empty_feeds = True
-    no_stylesheets = True
-    oldest_article = 7
-    max_articles_per_feed = 100000
-    recursions = 0
 
     no_stylesheets = True
-    remove_javascript = True
+    keep_only_tags = dict(name='article', attrs={'class': 'content'})
-    encoding = 'utf-8'
+    remove_tags_after = dict(name='div', attrs={'class': 'inner_article'})
-    # Seems to work best, but YMMV
+    remove_tags = [
-    simultaneous_downloads = 5
+        dict(name='div', attrs={'class': ['social_btns']}),
-
-    r = re.compile('.*(?P<url>http:\/\/(www.focus.pl)|(rss.feedsportal.com\/c)\/.*\.html?).*')
-    keep_only_tags = []
-    keep_only_tags.append(dict(name='div', attrs={'id': 'cll'}))
-
-    remove_tags = []
-    remove_tags.append(dict(name='div', attrs={'class': 'ulm noprint'}))
-    remove_tags.append(dict(name='div', attrs={'class': 'txb'}))
-    remove_tags.append(dict(name='div', attrs={'class': 'h2'}))
-    remove_tags.append(dict(name='ul', attrs={'class': 'txu'}))
-    remove_tags.append(dict(name='div', attrs={'class': 'ulc'}))
-
-    extra_css = '''
-                    body {font-family: verdana, arial, helvetica, geneva, sans-serif ;}
-                    h1{text-align: left;}
-                    h2{font-size: medium; font-weight: bold;}
-                    p.lead {font-weight: bold; text-align: left;}
-                    .authordate {font-size: small; color: #696969;}
-                    .fot{font-size: x-small; color: #666666;}
-                    '''
-
-    feeds = [
-        ('Nauka', 'http://www.focus.pl/nauka/rss/'),
-        ('Historia', 'http://www.focus.pl/historia/rss/'),
-        ('Cywilizacja', 'http://www.focus.pl/cywilizacja/rss/'),
-        ('Sport', 'http://www.focus.pl/sport/rss/'),
-        ('Technika', 'http://www.focus.pl/technika/rss/'),
-        ('Przyroda', 'http://www.focus.pl/przyroda/rss/'),
-        ('Technologie', 'http://www.focus.pl/gadzety/rss/')
     ]
 
-    def skip_ad_pages(self, soup):
+    # TO GET ARTICLE TOC
-        if ('advertisement' in soup.find('title').string.lower()):
+    def nejm_get_index(self):
-            href = soup.find('a').get('href')
+            return self.index_to_soup('http://www.focus.pl/')
-            return self.index_to_soup(href, raw=True)
-        else:
-            return None
 
-    def get_cover_url(self):
+    # To parse artice toc
-        soup = self.index_to_soup('http://www.focus.pl/magazyn/')
+    def parse_index(self):
-        tag = soup.find(name='div', attrs={'class': 'clr fl'})
+            soup = self.nejm_get_index()
-        if tag:
-            self.cover_url = 'http://www.focus.pl/' + tag.a['href']
-            return getattr(self, 'cover_url', self.cover_url)
 
-    def print_version(self, url):
+            toc = soup.find('div', id='wrapper')
-        if url.count('focus.pl.feedsportal.com'):
+
-            u = url.find('focus0Bpl')
+            articles = []
-            u = 'http://www.focus.pl/' + url[u + 11:]
+            feeds = []
-            u = u.replace('0C', '/')
+            section_title = 'Focus Articles'
-            u = u.replace('A', '')
+            for x in toc.findAll(True):
-            u = u.replace('0E', '-')
+                if x.name == 'h1':
-            u = u.replace('/nc/1//story01.htm', '/do-druku/1')
+                    # Article found
-        else:
+                    a = x.find('a')
-            u = url.replace('/nc/1', '/do-druku/1')
+                    if a is None:
-        return u
+                        continue
+                    title = self.tag_to_string(a)
+                    url = a.get('href', False)
+                    if not url or not title:
+                        continue
+                    # if url.startswith('story'):
+                    url = 'http://www.focus.pl' + url
+                    self.log('\t\tFound article:', title)
+                    self.log('\t\t\t', url)
+                    articles.append({'title': title, 'url': url,
+                                     'description': '', 'date': ''})
+            feeds.append((section_title, articles))
+
+            return feeds

recipes/ming_pao.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2010-2011, Eddie Lau'
+__copyright__ = '2010-2013, Eddie Lau'
 
 # Region - Hong Kong, Vancouver, Toronto
 __Region__ = 'Hong Kong'
@@ -32,6 +32,7 @@ __Date__ = ''
 
 '''
 Change Log:
+2013/09/28: allow thumbnails even with hi-res images
 2012/04/24: improved parsing of news.mingpao.com content
 2011/12/18: update the overridden create_odf(.) routine with the one from Calibre version 0.8.31. Move __UseChineseTitle__ usage away
             from create_odf(.). Optional support of text_summary and thumbnail images in Kindle's article view. Start new day
@@ -846,8 +847,7 @@ class MPRecipe(BasicNewsRecipe):
         return soup
 
     def populate_article_metadata(self, article, soup, first):
-        # thumbnails shouldn't be available if using hi-res images
+        if __IncludeThumbnails__ and first and hasattr(self, 'add_toc_thumbnail'):
-        if __IncludeThumbnails__ and __HiResImg__ == False and first and hasattr(self, 'add_toc_thumbnail'):
             img = soup.find('img')
             if img is not None:
                 self.add_toc_thumbnail(article, img['src'])
@@ -1071,3 +1071,4 @@ class MPRecipe(BasicNewsRecipe):
 
 
 
+

recipes/national_geographic_mag.recipe

@@ -1,46 +1,49 @@
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
-class NatGeoMag(BasicNewsRecipe):
+class NGM(BasicNewsRecipe):
-    title                  = 'National Geographic Mag'
-    __author__             = 'Terminal Veracity'
-    description            = 'The National Geographic Magazine'
-    publisher              = 'National Geographic'
-    oldest_article         = 31
-    max_articles_per_feed  = 50
-    category               = 'geography, magazine'
-    language               = 'en'
-    publication_type       = 'magazine'
-    cover_url              = 'http://www.yourlogoresources.com/wp-content/uploads/2011/09/national-geographic-logo.jpg'
-    use_embedded_content   = False
-    no_stylesheets         = True
-    remove_javascript      = True
-    recursions             = 1
-    remove_empty_feeds     = True
-    feeds                  = [('National Geographic Magazine', 'http://feeds.nationalgeographic.com/ng/NGM/NGM_Magazine')]
-    remove_tags            = [dict(name='div', attrs={'class':['nextpage_continue', 'subscribe']})]
-    keep_only_tags         = [dict(attrs={'class':'main_3narrow'})]
-    extra_css              = """
-                                h1 {font-size: large; font-weight: bold; margin: .5em 0; }
-                                h2 {font-size: large; font-weight: bold; margin: .5em 0; }
-                                h3 {font-size: medium; font-weight: bold; margin: 0 0; }
-                                .article_credits_author {font-size: small; font-style: italic; }
-                                .article_credits_photographer {font-size: small; font-style: italic; display: inline }
-                             """
 
-    def parse_feeds(self):
+    title       = 'National Geographic Magazine'
-        feeds = BasicNewsRecipe.parse_feeds(self)
+    __author__  = 'Krittika Goyal'
-        for feed in feeds:
+    description = 'National Geographic Magazine'
-            for article in feed.articles[:]:
+    timefmt = ' [%d %b, %Y]'
-                if 'Flashback' in article.title:
+
-                    feed.articles.remove(article)
+    no_stylesheets = True
-                elif 'Desktop Wallpaper' in article.title:
+    auto_cleanup = True
-                    feed.articles.remove(article)
+    auto_cleanup_keep = '//div[@class="featurepic"]'
-                elif 'Visions of Earth' in article.title:
+
-                    feed.articles.remove(article)
+    def nejm_get_index(self):
-                elif 'Your Shot' in article.title:
+        return self.index_to_soup('http://ngm.nationalgeographic.com/2013/10/table-of-contents')
-                    feed.articles.remove(article)
+
-                elif 'MyShot' in article.title:
+    # To parse artice toc
-                    feed.articles.remove(article)
+    def parse_index(self):
-                elif 'Field Test' in article.title:
+            soup = self.nejm_get_index()
-                    feed.articles.remove(article)
+            tocfull = soup.find('div', attrs={'class':'coltoc'})
-        return feeds
+
+            toc = tocfull.find('div', attrs={'class':'more_section'})
+
+            articles = []
+            feeds = []
+            section_title = 'Features'
+            for x in toc.findAll(True):
+                if x.name == 'a':
+                    # Article found
+                    title = self.tag_to_string(x)
+                    url = x.get('href', False)
+                    if not url or not title:
+                        continue
+                    url = 'http://ngm.nationalgeographic.com' + url
+                    self.log('\t\tFound article:', title)
+                    self.log('\t\t\t', url)
+                    articles.append({'title': title, 'url':url,
+                        'description':'', 'date':''})
+            feeds.append((section_title, articles))
+
+            art1 = tocfull.findAll('a')[1]
+            art1_title = self.tag_to_string(art1.find('div', attrs={'class': 'toched'}))
+            art1_url = art1.get('href', False)
+            art1_url = 'http://ngm.nationalgeographic.com' + art1_url
+            art1feed = {'title': art1_title, 'url':art1_url,
+                        'description':'', 'date':''}
+            feeds.append(('Cover Story', [art1feed]))
+
+            return feeds

recipes/neu_osnabrucker_zeitung.recipe

@@ -1,49 +1,108 @@
 # vim:fileencoding=utf-8
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class AdvancedUserRecipe1344926684(BasicNewsRecipe):
+class AdvancedUserRecipe1380105782(BasicNewsRecipe):
-    title          = u'Neue Osnabrücker Zeitung'
+    title                  = u'Neue Osnabrücker Zeitung'
-    __author__ = 'Krittika Goyal'
+    __author__             = 'vo_he'
-    oldest_article = 7
+    description            = 'Online auch ohne IPhone'
-    max_articles_per_feed = 100
+    encoding               = 'utf-8'
-    # auto_cleanup = True
+    language = 'de'
-    no_stylesheets         = True
-    use_embedded_content   = False
-    language               = 'de'
     remove_javascript      = True
+    no_stylesheets         = True
+
+    oldest_article = 2
+    max_articles_per_feed = 100
+    cover_url              = 'http://www.noz.de/bundles/nozplatform/images/logos/osnabruecker-zeitung.png'
+
+    remove_tags_before =dict(id='feedContent')
+    remove_tags_before =dict(id='headline')
+
+    remove_tags_after =dict(id='article-authorbox')
+    remove_tags_after =dict(id='footer-start')
+    remove_tags_after =dict(name='div', attrs={'class':'morelinks'})
 
-    keep_only_tags = [
-        dict(name='div', attrs={'class':'article'}),
-        dict(name='span', attrs={'id':'articletext'})
-        ]
     remove_tags = [
-        dict(name='div', attrs={'id':'retresco-title'}),
+        dict(name='div', attrs={'id':'ui-datepicker-div'}),
-        dict(name='div', attrs={'class':'retresco-item s1 relative'}),
+        dict(name='div', attrs={'class':'nav-second'}),
-        dict(name='a', attrs={'class':'medium2 largeSpaceTop icon'}),
+        dict(name='div', attrs={'class':'nav-first'}),
-        dict(name='div', attrs={'class':'articleFunctions inlineTeaserRight'}),
+        dict(name='div', attrs={'class':'icon-print'}),
-        dict(name='div', attrs={'class':'imageContainer '}),
+        dict(name='div', attrs={'class':'social-button'}),
-        dict(name='div', attrs={'class':'imageContainer centerContainer'}),
+        dict(name='div', attrs={'class':'social-media-bar'}),
-        dict(name='div', attrs={'class':'grid singleCol articleTeaser'}),
+        dict(name='div', attrs={'class':'pull-right'}),
-        dict(name='h3', attrs={'class':'teaserRow'}),
+        dict(name='div', attrs={'class':'btn btn-primary flat-button'}),
-        dict(name='div', attrs={'class':'related-comments'}),
+        dict(name='div', attrs={'class':'carousel-wrapper'}),
-        dict(name='a', attrs={'class':' icon'}),
+        dict(name='a', attrs={'class':'right-content merchandising hidden-tablet'}),
-        dict(name='a', attrs={'class':'right small'}),
+        dict(name='div', attrs={'class':'border-circle pull-left'}),
-        dict(name='span', attrs={'class':'small block spaceBottom rectangleAd'}),
+        dict(name='div', attrs={'class':'row show-grid general-infoimageContainer '}),
+        dict(name='div', attrs={'class':'location-list'}),
+        dict(name='div', attrs={'class':'block'}),
         dict(name='div', attrs={'class':'furtherGalleries largeSpaceTop'})
         ]
 
-    feeds          = [(u'Lokales', u'http://www.noz.de/rss/Lokales'),
+    feeds          = [(u'Melle Mitte', u'http://www.noz.de/rss/ressort/Melle%20Mitte'),
-(u'Vermischtes', u'http://www.noz.de/rss/Vermischtes'),
+                        (u'Melle Nord', u'http://www.noz.de/rss/ressort/Melle%20Nord'),
-(u'Politik', u'http://www.noz.de/rss/Politik'),
+                        (u'Melle Sued', u'http://www.noz.de/rss/ressort/Melle%20S%C3%BCd'),
-(u'Wirtschaft', u'http://www.noz.de/rss/Wirtschaft'),
+                        (u'Nordrhein Westfalen', u'http://www.noz.de/rss/ressort/Nordrhein-Westfalen'),
-(u'Kultur', u'http://www.noz.de/rss/Kultur'),
+                        (u'Niedersachsen', u'http://www.noz.de/rss/ressort/Niedersachsen'),
-(u'Medien', u'http://www.noz.de/rss/Medien'),
+                        (u'Vermischtes', u'http://www.noz.de/rss/ressort/Vermischtes'),
-(u'Wissenschaft', u'http://www.noz.de/rss/wissenschaft'),
+                        (u'GutzuWissen', u'http://www.noz.de/rss/ressort/Gut%20zu%20Wissen'),
-(u'Sport', u'http://www.noz.de/rss/Sport'),
+                        (u'Sport', u'http://www.noz.de/rss/ressort/Sport'),
-(u'Computer', u'http://www.noz.de/rss/Computer'),
+                        (u'Kultur', u'http://www.noz.de/rss/ressort/Kultur'),
-(u'Musik', u'http://www.noz.de/rss/Musik'),
+                        (u'Medien', u'http://www.noz.de/rss/ressort/Medien'),
-(u'Szene', u'http://www.noz.de/rss/Szene'),
+                        (u'Belm', u'http://www.noz.de/rss/ressort/Belm'),
-(u'Niedersachsen', u'http://www.noz.de/rss/Niedersachsen'),
+                        (u'Bissendorf', u' [url]http://www.noz.de/rss/ressort/Bissendorf[/url]'),
-(u'Kino', u'http://www.noz.de/rss/Kino')]
+                        (u'Osnabrueck', u'http://www.noz.de/rss/ressort/Osnabr%C3%BCck'),
-
+                        (u'Bad Essen', u'http://www.noz.de/rss/ressort/Bad%20Essen'),
+                        (u'Politik', u'http://www.noz.de/rss/ressort/Politik'),
+                        (u'Wirtschaft', u'http://www.noz.de/rss/ressort/Wirtschaft'),
+#(u'Fussball', u'http:/www.noz.de/rss/ressort/Fußball'),
+#(u'VfL Osnabrueck', u'http://www.noz.de/rss/ressort/VfL%20Osnabr%C3%BCck'),
+#(u'SF Lotte', u'http://www.noz.de/rss/ressort/SF%20Lotte'),
+#(u'SV Meppen', u'http://www.noz.de/rss/ressort/SV%20Meppen'),
+#(u'Artland Dragons', u'http://www.noz.de/rss/ressort/Artland%20Dragons'),
+#(u'Panthers', u'http://www.noz.de/rss/ressort/Panthers'),
+(u'OS-Sport', u'http://www.noz.de/rss/ressort/OS-Sport'),
+#(u'Emsland Sport', u'http://www.noz.de/rss/ressort/EL-Sport'),
+#(u'Lingen', u'http://www.noz.de/rss/ressort/Lingen'),
+#(u'Lohne', u'http://www.noz.de/rss/ressort/Lohne'),
+#(u'Emsbueren', u'http://www.noz.de/rss/ressort/Emsb%C3%BCren'),
+#(u'Salzbergen', u'http://www.noz.de/rss/ressort/Salzbergen'),
+#(u'Spelle', u'http://www.noz.de/rss/ressort/Spelle'),
+#(u'Freren', u'http://www.noz.de/rss/ressort/Freren'),
+#(u'Lengerich', u'http://www.noz.de/rss/ressort/Lengerich'),
+#(u'Bad Iburg', u'http://www.noz.de/rss/ressort/Bad%20Iburg'),
+#(u'Bad Laer', u'http://www.noz.de/rss/ressort/Bad%20Laer'),
+#(u'Bad Rothenfelde', u'http://www.noz.de/rss/ressort/Bad%20Rothenfelde'),
+#(u'GMHütte', u'http://www.noz.de/rss/ressort/Georgsmarienh%C3%BCtte'),
+#(u'Glandorf', u'http://www.noz.de/rss/ressort/Glandorf'),
+#(u'Hagen', u'http://www.noz.de/rss/ressort/Hagen'),
+#(u'Hasbergen', u'http://www.noz.de/rss/ressort/Hasbergen'),
+#(u'Hilter', u'http://www.noz.de/rss/ressort/Hilter'),
+#(u'Lotte', u'http://www.noz.de/rss/ressort/Lotte'),
+#(u'Wallenhorst', u'http://www.noz.de/rss/ressort/Wallenhorst'),
+#(u'Westerkappeln', u'http://www.noz.de/rss/ressort/Westerkappeln'),
+#(u'Artland', u'http://www.noz.de/rss/ressort/Artland'),
+#(u'Bersenbrück', u'http://www.noz.de/rss/ressort/Bersenbr%C3%BCck'),
+#(u'Fürstenau', u'http://www.noz.de/rss/ressort/F%C3%BCrstenau'),
+#(u'Neuenkirchen', u'http://www.noz.de/rss/ressort/Neuenkirchen'),
+#(u'Lokalsport', u'http://www.noz.de/rss/ressort/Lokalsport%20Nordkreis'),
+#(u'Bramsche', u'http://www.noz.de/rss/ressort/Bramsche'),
+#(u'Bramsche Ortsteile', u'http://www.noz.de/rss/ressort/Bramscher%20Ortsteile'),
+#(u'Neuenkirchen Vörden', u'http://www.noz.de/rss/ressort/Neuenkirchen-V%C3%B6rden'),
+#(u'Papenburg', u'http://www.noz.de/rss/ressort/Papenburg'),
+#(u'Dörpen', u'http://www.noz.de/rss/ressort/D%C3%B6rpen'),
+#(u'Rhede', u'http://www.noz.de/rss/ressort/Rhede'),
+#(u'Lathen', u'http://www.noz.de/rss/ressort/Lathen'),
+#(u'Sögel', u'http://www.noz.de/rss/ressort/S%C3%B6gel'),
+#(u'Nordhümmling', u'http://www.noz.de/rss/ressort/Nordh%C3%BCmmling'),
+#(u'Werlte', u'http://www.noz.de/rss/ressort/Werlte'),
+#(u'Westoverledingen', u'http://www.noz.de/rss/ressort/Westoverledingen'),
+#(u'Geeste', u'http://www.noz.de/rss/ressort/Geeste'),
+#(u'Haren', u'http://www.noz.de/rss/ressort/Haren'),
+#(u'Haselünne', u'http://www.noz.de/rss/ressort/Hasel%C3%BCnne'),
+#(u'Herzlake', u'http://www.noz.de/rss/ressort/Herzlake'),
+#(u'Meppen', u'http://www.noz.de/rss/ressort/Meppen'),
+#(u'Twist', u'http://www.noz.de/rss/ressort/Twist'),
+#(u'Bohmte', u'http://www.noz.de/rss/ressort/Bohmte'),
+#(u'Ostercappeln', u'http://www.noz.de/rss/ressort/Ostercappeln')
+]

recipes/new_york_review_of_books.recipe

@@ -1,4 +1,3 @@
-
 #!/usr/bin/env  python
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
@@ -11,6 +10,9 @@ import re
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
+def find_header(tag):
+    return tag.name == 'header' and tag.parent['class'] == 'article'
+
 class NewYorkReviewOfBooks(BasicNewsRecipe):
 
     title = u'New York Review of Books'
@@ -23,65 +25,70 @@ class NewYorkReviewOfBooks(BasicNewsRecipe):
     no_javascript = True
     needs_subscription = True
 
-    keep_only_tags = [dict(id=['article-body','page-title'])]
+    keep_only_tags = [
-    remove_tags = [dict(attrs={'class':['article-tools', 'article-links',
+        dict(name='section', attrs={'class':'article_body'}),
-        'center advertisement']})]
+        dict(name=find_header),
+        dict(name='div', attrs={'class':'for-subscribers-only'}),
+    ]
 
     preprocess_regexps = [(re.compile(r'<head>.*?</head>', re.DOTALL), lambda
         m:'<head></head>')]
 
+    def print_version(self, url):
+        return url+'?pagination=false'
+
     def get_browser(self):
         br = BasicNewsRecipe.get_browser(self)
         br.open('http://www.nybooks.com/account/signin/')
-        br.select_form(nr = 1)
+        br.select_form(nr=2)
         br['username'] = self.username
         br['password'] = self.password
         br.submit()
         return br
 
-    def print_version(self, url):
+    def preprocess_html(self, soup):
-        return url+'?pagination=false'
+        header = soup.find('header')
+        body = soup.find('body')
+        body.insert(0, header)
+        header.find('div', attrs={'class':'details'}).extract()
+        for i in soup.findAll('input'):
+            i.extract()
+        return soup
 
     def parse_index(self):
         soup = self.index_to_soup('http://www.nybooks.com/current-issue')
 
         # Find cover
-        sidebar = soup.find(id='sidebar')
+        sidebar = soup.find('div', attrs={'class':'issue_cover'})
         if sidebar is not None:
-            a = sidebar.find('a', href=lambda x: x and 'view-photo' in x)
+            img = sidebar.find('img', src=True)
-            if a is not None:
+            self.cover_url = 'http://www.nybooks.com' + img['src']
-                psoup = self.index_to_soup('http://www.nybooks.com'+a['href'])
+            self.log('Found cover at:', self.cover_url)
-                cover = psoup.find('img', src=True)
-                self.cover_url = cover['src']
-                self.log('Found cover at:', self.cover_url)
 
         # Find date
-        div = soup.find(id='page-title')
+        div = soup.find('time', pubdate='pubdate')
         if div is not None:
-            h5 = div.find('h5')
+            text = self.tag_to_string(div)
-            if h5 is not None:
+            date = text.partition(u'\u2022')[0].strip()
-                text = self.tag_to_string(h5)
+            self.timefmt = u' [%s]'%date
-                date = text.partition(u'\u2022')[0].strip()
+            self.log('Issue date:', date)
-                self.timefmt = u' [%s]'%date
-                self.log('Issue date:', date)
 
         # Find TOC
-        tocs = soup.findAll('ul', attrs={'class':'issue-article-list'})
+        toc = soup.find('div', attrs={'class':'current_issue'}).find('div', attrs={'class':'articles_list'})
         articles = []
-        for toc in tocs:
+        for div in toc.findAll('div', attrs={'class':'row'}):
-            for li in toc.findAll('li'):
+            h2 = div.find('h2')
-                h3 = li.find('h3')
+            title = self.tag_to_string(h2).strip()
-                title = self.tag_to_string(h3)
+            author = self.tag_to_string(div.find('div', attrs={'class':'author'})).strip()
-                author = self.tag_to_string(li.find('h4'))
+            title = title + u' (%s)'%author
-                title = title + u' (%s)'%author
+            url = 'http://www.nybooks.com' + h2.find('a', href=True)['href']
-                url = 'http://www.nybooks.com'+h3.find('a', href=True)['href']
+            desc = ''
-                desc = ''
+            for p in div.findAll('p', attrs={'class':lambda x: x and 'quiet' in x}):
-                for p in li.findAll('p'):
+                desc += self.tag_to_string(p)
-                    desc += self.tag_to_string(p)
+            self.log('Found article:', title)
-                self.log('Found article:', title)
+            self.log('\t', url)
-                self.log('\t', url)
+            self.log('\t', desc)
-                self.log('\t', desc)
+            articles.append({'title':title, 'url':url, 'date':'',
-                articles.append({'title':title, 'url':url, 'date':'',
                 'description':desc})
 
         return [('Current Issue', articles)]

recipes/new_york_review_of_books_no_sub.recipe

@@ -10,6 +10,9 @@ import re
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
+def find_header(tag):
+    return tag.name == 'header' and tag.parent['class'] == 'article'
+
 class NewYorkReviewOfBooks(BasicNewsRecipe):
 
     title = u'New York Review of Books (no subscription)'
@@ -21,9 +24,11 @@ class NewYorkReviewOfBooks(BasicNewsRecipe):
     no_stylesheets = True
     no_javascript = True
 
-    keep_only_tags = [dict(id=['article-body', 'page-title'])]
+    keep_only_tags = [
-    remove_tags = [dict(attrs={'class':['article-tools', 'article-links',
+        dict(name='section', attrs={'class':'article_body'}),
-        'center advertisement']})]
+        dict(name=find_header),
+        dict(name='div', attrs={'class':'for-subscribers-only'}),
+    ]
 
     preprocess_regexps = [(re.compile(r'<head>.*?</head>', re.DOTALL), lambda
         m:'<head></head>')]
@@ -31,40 +36,44 @@ class NewYorkReviewOfBooks(BasicNewsRecipe):
     def print_version(self, url):
         return url+'?pagination=false'
 
+    def preprocess_html(self, soup):
+        header = soup.find('header')
+        body = soup.find('body')
+        body.insert(0, header)
+        header.find('div', attrs={'class':'details'}).extract()
+        for i in soup.findAll('input'):
+            i.extract()
+        return soup
+
     def parse_index(self):
         soup = self.index_to_soup('http://www.nybooks.com/current-issue')
 
         # Find cover
-        sidebar = soup.find(id='sidebar')
+        sidebar = soup.find('div', attrs={'class':'issue_cover'})
         if sidebar is not None:
-            a = sidebar.find('a', href=lambda x: x and 'view-photo' in x)
+            img = sidebar.find('img', src=True)
-            if a is not None:
+            self.cover_url = 'http://www.nybooks.com' + img['src']
-                psoup = self.index_to_soup('http://www.nybooks.com'+a['href'])
+            self.log('Found cover at:', self.cover_url)
-                cover = psoup.find('img', src=True)
-                self.cover_url = cover['src']
-                self.log('Found cover at:', self.cover_url)
 
         # Find date
-        div = soup.find(id='page-title')
+        div = soup.find('time', pubdate='pubdate')
         if div is not None:
-            h5 = div.find('h5')
+            text = self.tag_to_string(div)
-            if h5 is not None:
+            date = text.partition(u'\u2022')[0].strip()
-                text = self.tag_to_string(h5)
+            self.timefmt = u' [%s]'%date
-                date = text.partition(u'\u2022')[0].strip()
+            self.log('Issue date:', date)
-                self.timefmt = u' [%s]'%date
-                self.log('Issue date:', date)
 
         # Find TOC
-        toc = soup.find('ul', attrs={'class':'issue-article-list'})
+        toc = soup.find('div', attrs={'class':'current_issue'}).find('div', attrs={'class':'articles_list'})
         articles = []
-        for li in toc.findAll('li'):
+        for div in toc.findAll('div', attrs={'class':'row'}):
-            h3 = li.find('h3')
+            h2 = div.find('h2')
-            title = self.tag_to_string(h3)
+            title = self.tag_to_string(h2).strip()
-            author = self.tag_to_string(li.find('h4'))
+            author = self.tag_to_string(div.find('div', attrs={'class':'author'})).strip()
             title = title + u' (%s)'%author
-            url = 'http://www.nybooks.com'+h3.find('a', href=True)['href']
+            url = 'http://www.nybooks.com' + h2.find('a', href=True)['href']
             desc = ''
-            for p in li.findAll('p'):
+            for p in div.findAll('p', attrs={'class':lambda x: x and 'quiet' in x}):
                 desc += self.tag_to_string(p)
             self.log('Found article:', title)
             self.log('\t', url)

recipes/padreydecano.recipe

@@ -0,0 +1,50 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__author__ = '2013, Carlos Alves <carlosalves90@gmail.com>'
+'''
+padreydecano.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class General(BasicNewsRecipe):
+    title                 = 'Padre y Decano'
+    __author__            = 'Carlos Alves'
+    description           = 'El sitio del pueblo'
+    tags = 'soccer, futbol, Peñarol'
+    language       = 'es_UY'
+    timefmt        = '[%a, %d %b, %Y]'
+    use_embedded_content  = False
+    recursion             = 5
+    encoding = None
+    remove_javascript = True
+    no_stylesheets = True
+
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    keep_only_tags = [
+             dict(name='h1', attrs={'class':'entry-title'}),
+             dict(name='div', attrs={'class':'entry-content clearfix'})
+             ]
+
+    remove_tags = [
+             dict(name='div', attrs={'class':['br', 'hr', 'titlebar', 'navigation']}),
+             dict(name='dl', attrs={'class':'gallery-item'}),
+             dict(name=['object','link'])
+                  ]
+
+    extra_css = '''
+                h1{font-family:Geneva, Arial, Helvetica, sans-serif;color:#154B7A;}
+                h3{font-size: 14px;color:#999999; font-family:Geneva, Arial, Helvetica, sans-serif;font-weight: bold;}
+                h2{color:#666666; font-family:Geneva, Arial, Helvetica, sans-serif;font-size:small;}
+                p {font-family:Arial,Helvetica,sans-serif;}
+                '''
+    feeds = [
+           (u'Padre y Decano | Club Atlético Peñarol', u'http://www.padreydecano.com/cms/feed/')
+        ]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

recipes/slate.recipe

@@ -20,6 +20,7 @@ class Slate(BasicNewsRecipe):
     masthead_url = 'http://img.slate.com/images/redesign2008/slate_logo.gif'
     remove_attributes = ['style']
     INDEX = 'http://slate.com'
+    compress_news_images = True
 
     keep_only_tags = [
         dict(name='header', attrs={'class':'article-header'}),

recipes/the_h.recipe

@@ -1,39 +0,0 @@
-#!/usr/bin/env  python
-
-__license__   = 'GPL v3'
-__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
-'''
-www.h-online.com
-'''
-
-from calibre.web.feeds.news import BasicNewsRecipe
-
-class TheHeiseOnline(BasicNewsRecipe):
-    title          = u'The H'
-    __author__     = 'Hiroshi Miura'
-    oldest_article = 3
-    description    = 'In association with Heise Online'
-    publisher      = 'Heise Media UK Ltd.'
-    category       = 'news, technology, security, OSS, internet'
-    max_articles_per_feed = 100
-    language       = 'en'
-    encoding       = 'utf-8'
-    conversion_options = {
-                      'comment'  : description
-                     ,'tags'     : category
-                     ,'publisher': publisher
-                     ,'language' : language
-                     }
-    feeds          = [
-                      (u'The H News Feed', u'http://www.h-online.com/news/atom.xml')
-                     ]
-    cover_url = 'http://www.h-online.com/icons/logo_theH.gif'
-
-    remove_tags = [
-                        dict(id="logo"),
-                        dict(id="footer")
-                        ]
-
-    def print_version(self, url):
-        return url + '?view=print'
-

recipes/unoticias.recipe

@@ -0,0 +1,56 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__author__ = '2013, Carlos Alves <carlosalves90@gmail.com>'
+'''
+unoticias.com.uy
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class General(BasicNewsRecipe):
+    title                 = 'UNoticias'
+    __author__            = 'Carlos Alves'
+    description           = 'Noticias Uruguay'
+    tags = 'news, sports, politics'
+    language       = 'es_UY'
+    timefmt        = '[%a, %d %b, %Y]'
+    use_embedded_content  = False
+    recursion             = 5
+    encoding = 'ISO-8859-1'
+    remove_javascript = True
+    no_stylesheets = True
+
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    keep_only_tags = [
+             dict(name='h1', attrs={'class':'nombre'}),
+             dict(name='h2', attrs={'class':'copete t20'}),
+             dict(name='div', attrs={'class':'desc'})
+             ]
+
+    remove_tags = [
+             dict(name='div', attrs={'class':['br', 'hr', 'titlebar', 'navigation']}),
+             dict(name='div', attrs={'id':'comment'}),
+             dict(name=['object','link'])
+                  ]
+
+    extra_css = '''
+                h1{font-family:Geneva, Arial, Helvetica, sans-serif;color:#154B7A;}
+                h3{font-size: 14px;color:#999999; font-family:Geneva, Arial, Helvetica, sans-serif;font-weight: bold;}
+                h2{color:#666666; font-family:Geneva, Arial, Helvetica, sans-serif;font-size:small;}
+                p {font-family:Arial,Helvetica,sans-serif;}
+                '''
+    feeds = [
+           (u'Nacionales', u'http://www.unoticias.com.uy/RSS/nacionales.xml'),
+           (u'Deportes', u'http://www.unoticias.com.uy/RSS/deportes.xml'),
+           (u'Sociedad', u'http://www.unoticias.com.uy/RSS/Sociedad.xml')
+        ]
+
+    def get_cover_url(self):
+        return 'http://www.unoticias.com.uy/artworks/logos/logo_small.gif'
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

recipes/vfr_magazine.recipe

@@ -1,9 +1,9 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
-class HindustanTimes(BasicNewsRecipe):
+class VFR(BasicNewsRecipe):
     title = u'VFR Magazine'
-    language = 'fr'
+    language = 'it'
     __author__ = 'Krittika Goyal'
     oldest_article = 31  # days
     max_articles_per_feed = 25

resources/default_tweaks.py

@@ -550,3 +550,10 @@ highlight_virtual_library = 'yellow'
 # all available output formats to be present.
 restrict_output_formats = None
 
+#: Set the thumbnail image quality used by the content server
+# The quality of a thumbnail is largely controlled by the compression quality
+# used when creating it. Set this to a larger number to improve the quality.
+# Note that the thumbnails get much larger with larger compression quality
+# numbers.
+# The value can be between 50 and 99
+content_server_thumbnail_compression_quality = 75

setup/hosting.py

@@ -508,12 +508,14 @@ def upload_to_servers(files, version):  # {{{
 
 def upload_to_dbs(files, version):  # {{{
     print('Uploading to fosshub.com')
+    sys.stdout.flush()
     server = 'mirror1.fosshub.com'
     rdir = 'release/'
     check_call(['ssh', 'kovid@%s' % server, 'rm -f release/*'])
     for x in files:
         start = time.time()
         print ('Uploading', x)
+        sys.stdout.flush()
         for i in range(5):
             try:
                 check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
@@ -522,10 +524,12 @@ def upload_to_dbs(files, version):  # {{{
                 raise SystemExit(1)
             except:
                 print ('\nUpload failed, trying again in 30 seconds')
+                sys.stdout.flush()
                 time.sleep(30)
             else:
                 break
         print ('Uploaded in', int(time.time() - start), 'seconds\n\n')
+        sys.stdout.flush()
     check_call(['ssh', 'kovid@%s' % server, '/home/kovid/uploadFiles'])
 # }}}
 

setup/resources.py

@@ -18,12 +18,14 @@ def get_opts_from_parser(parser):
         for x in opt._short_opts:
             yield x
     for o in parser.option_list:
-        for x in do_opt(o): yield x
+        for x in do_opt(o):
+            yield x
     for g in parser.option_groups:
         for o in g.option_list:
-            for x in do_opt(o): yield x
+            for x in do_opt(o):
+                yield x
 
-class Coffee(Command): # {{{
+class Coffee(Command):  # {{{
 
     description = 'Compile coffeescript files into javascript'
     COFFEE_DIRS = ('ebooks/oeb/display', 'ebooks/oeb/polish')
@@ -112,7 +114,7 @@ class Coffee(Command): # {{{
             os.remove(x)
 # }}}
 
-class Kakasi(Command): # {{{
+class Kakasi(Command):  # {{{
 
     description = 'Compile resources for unihandecode'
 
@@ -155,29 +157,29 @@ class Kakasi(Command): # {{{
         dic = {}
         for line in open(src, "r"):
             line = line.decode("utf-8").strip()
-            if line.startswith(';;'): # skip comment
+            if line.startswith(';;'):  # skip comment
                 continue
             if re.match(r"^$",line):
                 continue
             pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:unichr(int(x.group(1),16)), line)
             dic[pair[0]] = pair[1]
-        cPickle.dump(dic, open(dst, 'wb'), protocol=-1) #pickle
+        cPickle.dump(dic, open(dst, 'wb'), protocol=-1)  # pickle
 
     def mkkanadict(self, src, dst):
         dic = {}
         for line in open(src, "r"):
             line = line.decode("utf-8").strip()
-            if line.startswith(';;'): # skip comment
+            if line.startswith(';;'):  # skip comment
                 continue
             if re.match(r"^$",line):
                 continue
             (alpha, kana) = line.split(' ')
             dic[kana] = alpha
-        cPickle.dump(dic, open(dst, 'wb'), protocol=-1) #pickle
+        cPickle.dump(dic, open(dst, 'wb'), protocol=-1)  # pickle
 
     def parsekdict(self, line):
         line = line.decode("utf-8").strip()
-        if line.startswith(';;'): # skip comment
+        if line.startswith(';;'):  # skip comment
             return
         (yomi, kanji) = line.split(' ')
         if ord(yomi[-1:]) <= ord('z'):
@@ -193,7 +195,7 @@ class Kakasi(Command): # {{{
             if kanji in self.records[key]:
                 rec = self.records[key][kanji]
                 rec.append((yomi,tail))
-                self.records[key].update( {kanji: rec} )
+                self.records[key].update({kanji: rec})
             else:
                 self.records[key][kanji]=[(yomi, tail)]
         else:
@@ -213,7 +215,7 @@ class Kakasi(Command): # {{{
             shutil.rmtree(kakasi)
 # }}}
 
-class Resources(Command): # {{{
+class Resources(Command):  # {{{
 
     description = 'Compile various needed calibre resources'
     sub_commands = ['kakasi', 'coffee']
@@ -255,7 +257,6 @@ class Resources(Command): # {{{
                     with open(n, 'rb') as f:
                         zf.writestr(os.path.basename(n), f.read())
 
-
         dest = self.j(self.RESOURCES, 'ebook-convert-complete.pickle')
         files = []
         for x in os.walk(self.j(self.SRC, 'calibre')):

setup/upload.py

@@ -70,6 +70,10 @@ class ReUpload(Command):  # {{{
 
     def pre_sub_commands(self, opts):
         opts.replace = True
+        exists = {x for x in installers() if os.path.exists(x)}
+        if not exists:
+            print ('There appear to be no installers!')
+            raise SystemExit(1)
 
     def run(self, opts):
         upload_signatures()

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (1, 5, 0)
+numeric_version = (1, 6, 0)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -435,7 +435,7 @@ class EPUBMetadataWriter(MetadataWriterPlugin):
 
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.metadata.epub import set_metadata
-        set_metadata(stream, mi, apply_null=self.apply_null)
+        set_metadata(stream, mi, apply_null=self.apply_null, force_identifiers=self.force_identifiers)
 
 class FB2MetadataWriter(MetadataWriterPlugin):
 
@@ -923,6 +923,11 @@ class ActionSortBy(InterfaceActionBase):
     actual_plugin = 'calibre.gui2.actions.sort:SortByAction'
     description = _('Sort the list of books')
 
+class ActionMarkBooks(InterfaceActionBase):
+    name = 'Mark Books'
+    actual_plugin = 'calibre.gui2.actions.mark_books:MarkBooksAction'
+    description = _('Temporarily mark books')
+
 class ActionStore(InterfaceActionBase):
     name = 'Store'
     author = 'John Schember'
@@ -953,7 +958,8 @@ plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
         ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks,
         ActionAddToLibrary, ActionEditCollections, ActionMatchBooks, ActionChooseLibrary,
         ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore,
-        ActionPluginUpdater, ActionPickRandom, ActionEditToC, ActionSortBy]
+        ActionPluginUpdater, ActionPickRandom, ActionEditToC, ActionSortBy,
+        ActionMarkBooks]
 
 # }}}
 
@@ -1245,6 +1251,15 @@ class StoreSonyAUStore(StoreSonyStore):
     actual_plugin = 'calibre.gui2.store.stores.sony_au_plugin:SonyStore'
     headquarters = 'AU'
 
+class StoreAmazonCAKindleStore(StoreBase):
+    name = 'Amazon CA Kindle'
+    author = u'Tomasz Długosz'
+    description = u'Kindle books from Amazon.'
+    actual_plugin = 'calibre.gui2.store.stores.amazon_ca_plugin:AmazonCAKindleStore'
+
+    headquarters = 'CA'
+    formats = ['KINDLE']
+    # affiliate = True
 
 class StoreAmazonDEKindleStore(StoreBase):
     name = 'Amazon DE Kindle'
@@ -1342,16 +1357,6 @@ class StoreBiblioStore(StoreBase):
     headquarters = 'BG'
     formats = ['EPUB, PDF']
 
-class StoreBookotekaStore(StoreBase):
-    name = 'Bookoteka'
-    author = u'Tomasz Długosz'
-    description = u'E-booki w Bookotece dostępne są w formacie EPUB oraz PDF. Publikacje sprzedawane w Bookotece są objęte prawami autorskimi. Zobowiązaliśmy się chronić te prawa, ale bez ograniczania dostępu do książki użytkownikowi, który nabył ją w legalny sposób. Dlatego też Bookoteka stosuje tak zwany „watermarking transakcyjny” czyli swego rodzaju znaki wodne.'  # noqa
-    actual_plugin = 'calibre.gui2.store.stores.bookoteka_plugin:BookotekaStore'
-
-    drm_free_only = True
-    headquarters = 'PL'
-    formats = ['EPUB', 'PDF']
-
 class StoreCdpStore(StoreBase):
     name = 'Cdp.pl'
     author = u'Tomasz Długosz'
@@ -1687,6 +1692,15 @@ class StoreWHSmithUKStore(StoreBase):
     headquarters = 'UK'
     formats = ['EPUB', 'PDF']
 
+class StoreWolneLekturyStore(StoreBase):
+    name = 'Wolne Lektury'
+    author = u'Tomasz Długosz'
+    description = u'Wolne Lektury to biblioteka internetowa czynna 24 godziny na dobę, 365 dni w roku, której zasoby dostępne są całkowicie za darmo. Wszystkie dzieła są odpowiednio opracowane - opatrzone przypisami, motywami i udostępnione w kilku formatach - HTML, TXT, PDF, EPUB, MOBI, FB2.'  # noqa
+    actual_plugin = 'calibre.gui2.store.stores.wolnelektury_plugin:WolneLekturyStore'
+
+    headquarters = 'PL'
+    formats = ['EPUB', 'MOBI', 'PDF', 'TXT', 'FB2']
+
 class StoreWoblinkStore(StoreBase):
     name = 'Woblink'
     author = u'Tomasz Długosz'
@@ -1709,6 +1723,7 @@ plugins += [
     StoreAllegroStore,
     StoreArchiveOrgStore,
     StoreAmazonKindleStore,
+    StoreAmazonCAKindleStore,
     StoreAmazonDEKindleStore,
     StoreAmazonESKindleStore,
     StoreAmazonFRKindleStore,
@@ -1718,7 +1733,6 @@ plugins += [
     StoreBNStore,
     StoreBeamEBooksDEStore,
     StoreBiblioStore,
-    StoreBookotekaStore,
     StoreChitankaStore,
     StoreCdpStore,
     StoreDieselEbooksStore,
@@ -1754,6 +1768,7 @@ plugins += [
     StoreWaterstonesUKStore,
     StoreWeightlessBooksStore,
     StoreWHSmithUKStore,
+    StoreWolneLekturyStore,
     StoreWoblinkStore,
     XinXiiStore
 ]

src/calibre/customize/ui.py

@@ -319,6 +319,19 @@ class ApplyNullMetadata(object):
 
 apply_null_metadata = ApplyNullMetadata()
 
+class ForceIdentifiers(object):
+
+    def __init__(self):
+        self.force_identifiers = False
+
+    def __enter__(self):
+        self.force_identifiers = True
+
+    def __exit__(self, *args):
+        self.force_identifiers = False
+
+force_identifiers = ForceIdentifiers()
+
 def get_file_type_metadata(stream, ftype):
     mi = MetaInformation(None, None)
 
@@ -346,6 +359,7 @@ def set_file_type_metadata(stream, mi, ftype):
                 with plugin:
                     try:
                         plugin.apply_null = apply_null_metadata.apply_null
+                        plugin.force_identifiers = force_identifiers.force_identifiers
                         plugin.set_metadata(stream, mi, ftype.lower().strip())
                         break
                     except:

src/calibre/db/cache.py

@@ -18,7 +18,7 @@ from calibre.constants import iswindows, preferred_encoding
 from calibre.customize.ui import run_plugins_on_import, run_plugins_on_postimport
 from calibre.db import SPOOL_SIZE, _get_next_series_num_for_list
 from calibre.db.categories import get_categories
-from calibre.db.locking import create_locks
+from calibre.db.locking import create_locks, DowngradeLockError
 from calibre.db.errors import NoSuchFormat
 from calibre.db.fields import create_field, IDENTITY, InvalidLinkTable
 from calibre.db.search import Search
@@ -51,10 +51,19 @@ def write_api(f):
 
 def wrap_simple(lock, func):
     @wraps(func)
-    def ans(*args, **kwargs):
+    def call_func_with_lock(*args, **kwargs):
-        with lock:
+        try:
+            with lock:
+                return func(*args, **kwargs)
+        except DowngradeLockError:
+            # We already have an exclusive lock, no need to acquire a shared
+            # lock. This can happen when updating the search cache in the
+            # presence of composite columns. Updating the search cache holds an
+            # exclusive lock, but searching a composite column involves
+            # reading field values via ProxyMetadata which tries to get a
+            # shared lock.
             return func(*args, **kwargs)
-    return ans
+    return call_func_with_lock
 
 def run_import_plugins(path_or_stream, fmt):
     fmt = fmt.lower()

src/calibre/db/lazy.py

@@ -234,11 +234,11 @@ def composite_getter(mi, field, metadata, book_id, cache, formatter, template_ca
 
 def virtual_libraries_getter(dbref, book_id, cache):
     try:
-        return cache[field]
+        return cache['virtual_libraries']
     except KeyError:
         db = dbref()
         vls = db.virtual_libraries_for_books((book_id,))[book_id]
-        ret = cache[field] = ', '.join(vls)
+        ret = cache['virtual_libraries'] = ', '.join(vls)
         return ret
 
 getters = {

src/calibre/db/locking.py

@@ -17,6 +17,9 @@ class LockingError(RuntimeError):
         RuntimeError.__init__(self, msg)
         self.locking_debug_msg = extra
 
+class DowngradeLockError(LockingError):
+    pass
+
 def create_locks():
     '''
     Return a pair of locks: (read_lock, write_lock)
@@ -150,7 +153,7 @@ class SHLock(object):  # {{{
         #  to the shared queue and it will give us the lock eventually.
         if self.is_exclusive or self._exclusive_queue:
             if self._exclusive_owner is me:
-                raise LockingError("can't downgrade SHLock object")
+                raise DowngradeLockError("can't downgrade SHLock object")
             if not blocking:
                 return False
             waiter = self._take_waiter()

src/calibre/db/search.py

@@ -464,7 +464,7 @@ class Parser(SearchQueryParser):  # {{{
         return self.all_book_ids
 
     def field_iter(self, name, candidates):
-        get_metadata = partial(self.dbcache._get_metadata, get_user_categories=False)
+        get_metadata = self.dbcache._get_proxy_metadata
         try:
             field = self.dbcache.fields[name]
         except KeyError:

src/calibre/db/tests/main.py

@@ -61,11 +61,7 @@ class TestResult(unittest.TextTestResult):
 def find_tests():
     return unittest.defaultTestLoader.discover(os.path.dirname(os.path.abspath(__file__)), pattern='*.py')
 
-if __name__ == '__main__':
+def run_tests(find_tests=find_tests):
-    from calibre.utils.config_base import reset_tweaks_to_default
-    from calibre.ebooks.metadata.book.base import reset_field_metadata
-    reset_tweaks_to_default()
-    reset_field_metadata()
     parser = argparse.ArgumentParser()
     parser.add_argument('name', nargs='?', default=None,
                         help='The name of the test to run, for e.g. writing.WritingTest.many_many_basic or .many_many_basic for a shortcut')
@@ -95,3 +91,10 @@ if __name__ == '__main__':
     r.resultclass = TestResult
     r(verbosity=4).run(tests)
 
+if __name__ == '__main__':
+    from calibre.utils.config_base import reset_tweaks_to_default
+    from calibre.ebooks.metadata.book.base import reset_field_metadata
+    reset_tweaks_to_default()
+    reset_field_metadata()
+    run_tests()
+

src/calibre/db/tests/reading.py

@@ -583,6 +583,8 @@ class ReadingTest(BaseTest):
                                    display={'composite_template': '{pubdate:format_date(d-M-yy)}', 'composite_sort':'date'})
         cache.create_custom_column('bool', 'CC6', 'composite', False, display={'composite_template': '{#yesno}', 'composite_sort':'bool'})
         cache.create_custom_column('ccm', 'CC7', 'composite', True, display={'composite_template': '{#tags}'})
+        cache.create_custom_column('ccp', 'CC8', 'composite', True, display={'composite_template': '{publisher}'})
+        cache.create_custom_column('ccf', 'CC9', 'composite', True, display={'composite_template': "{:'approximate_formats()'}"})
 
         cache = self.init_cache()
         # Test searching
@@ -607,5 +609,14 @@ class ReadingTest(BaseTest):
         # Test is_multiple sorting
         cache.set_field('#tags', {1:'b, a, c', 2:'a, b, c', 3:'a, c, b'})
         self.assertEqual([1, 2, 3], cache.multisort([('#ccm', True)]))
+
+        # Test that lock downgrading during update of search cache works
+        self.assertEqual(cache.search('#ccp:One'), {2})
+        cache.set_field('publisher', {2:'One', 1:'One'})
+        self.assertEqual(cache.search('#ccp:One'), {1, 2})
+
+        self.assertEqual(cache.search('#ccf:FMT1'), {1, 2})
+        cache.remove_formats({1:('FMT1',)})
+        self.assertEqual('FMT2', cache.field_for('#ccf', 1))
     # }}}
 

src/calibre/db/view.py

@@ -13,7 +13,7 @@ from itertools import izip, imap
 from future_builtins import map
 
 from calibre.ebooks.metadata import title_sort
-from calibre.utils.config_base import tweaks
+from calibre.utils.config_base import tweaks, prefs
 from calibre.db.write import uniq
 
 def sanitize_sort_field_name(field_metadata, field):
@@ -77,6 +77,7 @@ class View(object):
     def __init__(self, cache):
         self.cache = cache
         self.marked_ids = {}
+        self.marked_listeners = {}
         self.search_restriction_book_count = 0
         self.search_restriction = self.base_restriction = ''
         self.search_restriction_name = self.base_restriction_name = ''
@@ -127,6 +128,9 @@ class View(object):
         self.full_map_is_sorted = True
         self.sort_history = [('id', True)]
 
+    def add_marked_listener(self, func):
+        self.marked_listeners[id(func)] = weakref.ref(func)
+
     def add_to_sort_history(self, items):
         self.sort_history = uniq((list(items) + list(self.sort_history)),
                                  operator.itemgetter(0))[:tweaks['maximum_resort_levels']]
@@ -370,7 +374,19 @@ class View(object):
                 id_dict.itervalues())))
         # This invalidates all searches in the cache even though the cache may
         # be shared by multiple views. This is not ideal, but...
-        self.cache.clear_search_caches(old_marked_ids | set(self.marked_ids))
+        cmids = set(self.marked_ids)
+        self.cache.clear_search_caches(old_marked_ids | cmids)
+        if old_marked_ids != cmids:
+            for funcref in self.marked_listeners.itervalues():
+                func = funcref()
+                if func is not None:
+                    func(old_marked_ids, cmids)
+
+    def toggle_marked_ids(self, book_ids):
+        book_ids = set(book_ids)
+        mids = set(self.marked_ids)
+        common = mids.intersection(book_ids)
+        self.set_marked_ids((mids | book_ids) - common)
 
     def refresh(self, field=None, ascending=True, clear_caches=True, do_search=True):
         self._map = tuple(sorted(self.cache.all_book_ids()))
@@ -410,4 +426,6 @@ class View(object):
         ids = tuple(ids)
         self._map = ids + self._map
         self._map_filtered = ids + self._map_filtered
+        if prefs['mark_new_books']:
+            self.toggle_marked_ids(ids)
 

src/calibre/devices/kobo/driver.py

@@ -63,7 +63,7 @@ class KOBO(USBMS):
     gui_name = 'Kobo Reader'
     description = _('Communicate with the Kobo Reader')
     author = 'Timothy Legge and David Forrester'
-    version = (2, 1, 3)
+    version = (2, 1, 4)
 
     dbversion = 0
     fwversion = 0
@@ -660,9 +660,10 @@ class KOBO(USBMS):
                     ' selecting "Configure this device" and then the '
                     ' "Attempt to support newer firmware" option.'
                     ' Doing so may require you to perform a factory reset of'
-                    ' your Kobo.'
+                    ' your Kobo.') + ((
-                     ),
+                    '\nDevice database version: %s.'
-                     UserFeedback.WARN)
+                    '\nDevice firmware version: %s') % (self.dbversion, self.fwversion))
+                    , UserFeedback.WARN)
 
                 return False
             else:
@@ -2833,8 +2834,10 @@ class KOBOTOUCH(KOBO):
                     ' selecting "Configure this device" and then the '
                     ' "Attempt to support newer firmware" option.'
                     ' Doing so may require you to perform a factory reset of'
-                    ' your Kobo.'
+                    ' your Kobo.') + (
-                     ),
+                    '\nDevice database version: %s.'
+                    '\nDevice firmware version: %s'
+                     ) % (self.dbversion, self.fwversion),
                      UserFeedback.WARN)
 
                 return False

src/calibre/devices/smart_device_app/driver.py

@@ -7,7 +7,7 @@ Created on 29 Jun 2012
 
 @author: charles
 '''
-import socket, select, json, os, traceback, time, sys, random, cPickle
+import socket, select, json, os, traceback, time, sys, random
 import posixpath
 from collections import defaultdict
 import hashlib, threading
@@ -682,32 +682,36 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             return None
 
     def _metadata_in_cache(self, uuid, ext, lastmod):
-        from calibre.utils.date import parse_date, now
+        try:
-        key = uuid+ext
+            from calibre.utils.date import parse_date, now
-        if isinstance(lastmod, unicode):
+            key = uuid+ext
-            lastmod = parse_date(lastmod)
+            if isinstance(lastmod, unicode):
-#         if key in self.known_uuids:
+                if lastmod == 'None':
-#             self._debug(key, lastmod, self.known_uuids[key].last_modified)
+                    return None
-#         else:
+                lastmod = parse_date(lastmod)
-#             self._debug(key, 'not in known uuids')
+            if key in self.known_uuids and self.known_uuids[key]['book'].last_modified == lastmod:
-        if key in self.known_uuids and self.known_uuids[key]['book'].last_modified == lastmod:
+                self.known_uuids[key]['last_used'] = now()
-            self.known_uuids[key]['last_used'] = now()
+                return self.known_uuids[key]['book'].deepcopy()
-            return self.known_uuids[key]['book'].deepcopy()
+        except:
+            traceback.print_exc()
         return None
 
     def _metadata_already_on_device(self, book):
-        v = self.known_metadata.get(book.lpath, None)
+        try:
-        if v is not None:
+            v = self.known_metadata.get(book.lpath, None)
-            # Metadata is the same if the uuids match, if the last_modified dates
+            if v is not None:
-            # match, and if the height of the thumbnails is the same. The last
+                # Metadata is the same if the uuids match, if the last_modified dates
-            # is there to allow a device to demand a different thumbnail size
+                # match, and if the height of the thumbnails is the same. The last
-            if (v.get('uuid', None) == book.get('uuid', None) and
+                # is there to allow a device to demand a different thumbnail size
-                    v.get('last_modified', None) == book.get('last_modified', None)):
+                if (v.get('uuid', None) == book.get('uuid', None) and
-                v_thumb = v.get('thumbnail', None)
+                        v.get('last_modified', None) == book.get('last_modified', None)):
-                b_thumb = book.get('thumbnail', None)
+                    v_thumb = v.get('thumbnail', None)
-                if bool(v_thumb) != bool(b_thumb):
+                    b_thumb = book.get('thumbnail', None)
-                    return False
+                    if bool(v_thumb) != bool(b_thumb):
-                return not v_thumb or v_thumb[1] == b_thumb[1]
+                        return False
+                    return not v_thumb or v_thumb[1] == b_thumb[1]
+        except:
+            traceback.print_exc()
         return False
 
     def _uuid_already_on_device(self, uuid, ext):
@@ -717,32 +721,74 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             return None
 
     def _read_metadata_cache(self):
-        cache_file_name = os.path.join(cache_dir(),
+        from calibre.utils.config import from_json
+        try:
+            old_cache_file_name = os.path.join(cache_dir(),
                            'device_drivers_' + self.__class__.__name__ +
                                 '_metadata_cache.pickle')
-        if os.path.exists(cache_file_name):
+            if os.path.exists(old_cache_file_name):
-            with open(cache_file_name, mode='rb') as fd:
+                os.remove(old_cache_file_name)
-                json_metadata = cPickle.load(fd)
+        except:
-            for uuid,json_book in json_metadata.iteritems():
+            pass
-                book = self.json_codec.raw_to_book(json_book['book'], SDBook, self.PREFIX)
+
-                self.known_uuids[uuid]['book'] = book
+        cache_file_name = os.path.join(cache_dir(),
-                self.known_uuids[uuid]['last_used'] = json_book['last_used']
+                           'device_drivers_' + self.__class__.__name__ +
-                lpath = book.get('lpath')
+                                '_metadata_cache.json')
-                if lpath in self.known_metadata:
+        self.known_uuids = defaultdict(dict)
-                    self.known_uuids.pop(uuid, None)
+        self.known_metadata = {}
-                else:
+        try:
-                    self.known_metadata[lpath] = book
+            if os.path.exists(cache_file_name):
+                with open(cache_file_name, mode='rb') as fd:
+                    while True:
+                        rec_len = fd.readline()
+                        if len(rec_len) != 8:
+                            break
+                        raw = fd.read(int(rec_len))
+                        book = json.loads(raw.decode('utf-8'), object_hook=from_json)
+                        uuid = book.keys()[0]
+                        metadata = self.json_codec.raw_to_book(book[uuid]['book'],
+                                                            SDBook, self.PREFIX)
+                        book[uuid]['book'] = metadata
+                        self.known_uuids.update(book)
+
+                        lpath = metadata.get('lpath')
+                        if lpath in self.known_metadata:
+                            self.known_uuids.pop(uuid, None)
+                        else:
+                            self.known_metadata[lpath] = metadata
+        except:
+            traceback.print_exc()
+            self.known_uuids = defaultdict(dict)
+            self.known_metadata = {}
+            try:
+                if os.path.exists(cache_file_name):
+                    os.remove(cache_file_name)
+            except:
+                traceback.print_exc()
+
 
     def _write_metadata_cache(self):
+        from calibre.utils.config import to_json
         cache_file_name = os.path.join(cache_dir(),
                            'device_drivers_' + self.__class__.__name__ +
-                                '_metadata_cache.pickle')
+                                '_metadata_cache.json')
-        json_metadata = defaultdict(dict)
+        try:
-        for uuid,book in self.known_uuids.iteritems():
+            with open(cache_file_name, mode='wb') as fd:
-            json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book'])
+                for uuid,book in self.known_uuids.iteritems():
-            json_metadata[uuid]['last_used'] = book['last_used']
+                    json_metadata = defaultdict(dict)
-        with open(cache_file_name, mode='wb') as fd:
+                    json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book'])
-            cPickle.dump(json_metadata, fd, -1)
+                    json_metadata[uuid]['last_used'] = book['last_used']
+                    result = json.dumps(json_metadata, indent=2, default=to_json)
+                    fd.write("%0.7d\n"%(len(result)+1))
+                    fd.write(result)
+                    fd.write('\n')
+        except:
+            traceback.print_exc()
+            try:
+                if os.path.exists(cache_file_name):
+                    os.remove(cache_file_name)
+            except:
+                traceback.print_exc()
 
     def _set_known_metadata(self, book, remove=False):
         from calibre.utils.date import now
@@ -757,7 +803,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             if key:
                 self.known_uuids.pop(key, None)
         else:
-            new_book = self.known_metadata[lpath] = book.deepcopy()
+            # Check if we have another UUID with the same lpath. If so, remove it
+            existing_uuid = self.known_metadata.get(lpath, {}).get('uuid', None)
+            if existing_uuid:
+                self.known_uuids.pop(existing_uuid + ext, None)
+
+            new_book = book.deepcopy()
+            self.known_metadata[lpath] = new_book
             if key:
                 self.known_uuids[key]['book'] = new_book
                 self.known_uuids[key]['last_used'] = now()
@@ -815,8 +867,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             self.is_connected = False
         if self.is_connected:
             self.noop_counter += 1
-            if only_presence and (
+            if (only_presence and
-                    self.noop_counter % self.SEND_NOOP_EVERY_NTH_PROBE) != 1:
+                    self.noop_counter > self.SEND_NOOP_EVERY_NTH_PROBE and
+                    (self.noop_counter % self.SEND_NOOP_EVERY_NTH_PROBE) != 1):
                 try:
                     ans = select.select((self.device_socket,), (), (), 0)
                     if len(ans[0]) == 0:

src/calibre/devices/teclast/driver.py

@@ -20,9 +20,9 @@ class TECLAST_K3(USBMS):
     BCD         = [0x0000, 0x0100]
 
     VENDOR_NAME      = ['TECLAST', 'IMAGIN', 'RK28XX', 'PER3274B', 'BEBOOK',
-                        'RK2728', 'MR700']
+                        'RK2728', 'MR700', 'CYBER']
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['DIGITAL_PLAYER', 'TL-K5',
-            'EREADER', 'USB-MSC', 'PER3274B', 'BEBOOK', 'USER']
+            'EREADER', 'USB-MSC', 'PER3274B', 'BEBOOK', 'USER', 'BOOK']
 
     MAIN_MEMORY_VOLUME_LABEL  = 'K3 Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'K3 Storage Card'

src/calibre/ebooks/conversion/cli.py

@@ -9,6 +9,7 @@ Command line interface to conversion sub-system
 
 import sys, os
 from optparse import OptionGroup, Option
+from collections import OrderedDict
 
 from calibre.utils.config import OptionParser
 from calibre.utils.logging import Log
@@ -126,14 +127,14 @@ def add_input_output_options(parser, plumber):
         parser.add_option_group(oo)
 
 def add_pipeline_options(parser, plumber):
-    groups = {
+    groups = OrderedDict((
-              '' : ('',
+              ('' , ('',
                     [
                      'input_profile',
                      'output_profile',
                      ]
-                    ),
+                    )),
-              'LOOK AND FEEL' : (
+              (_('LOOK AND FEEL') , (
                   _('Options to control the look and feel of the output'),
                   [
                       'base_font_size', 'disable_font_rescaling',
@@ -141,7 +142,7 @@ def add_pipeline_options(parser, plumber):
                       'subset_embedded_fonts', 'embed_all_fonts',
                       'line_height', 'minimum_line_height',
                       'linearize_tables',
-                      'extra_css', 'filter_css',
+                      'extra_css', 'filter_css', 'expand_css',
                       'smarten_punctuation', 'unsmarten_punctuation',
                       'margin_top', 'margin_left', 'margin_right',
                       'margin_bottom', 'change_justification',
@@ -150,17 +151,17 @@ def add_pipeline_options(parser, plumber):
                       'remove_paragraph_spacing_indent_size',
                       'asciiize', 'keep_ligatures',
                   ]
-                  ),
+                  )),
 
-              'HEURISTIC PROCESSING' : (
+              (_('HEURISTIC PROCESSING') , (
                   _('Modify the document text and structure using common'
                      ' patterns. Disabled by default. Use %(en)s to enable. '
                      ' Individual actions can be disabled with the %(dis)s options.')
                   % dict(en='--enable-heuristics', dis='--disable-*'),
                   ['enable_heuristics'] + HEURISTIC_OPTIONS
-                  ),
+                  )),
 
-              'SEARCH AND REPLACE' : (
+              (_('SEARCH AND REPLACE') , (
                  _('Modify the document text and structure using user defined patterns.'),
                  [
                      'sr1_search', 'sr1_replace',
@@ -168,9 +169,9 @@ def add_pipeline_options(parser, plumber):
                      'sr3_search', 'sr3_replace',
                      'search_replace',
                  ]
-              ),
+              )),
 
-              'STRUCTURE DETECTION' : (
+              (_('STRUCTURE DETECTION') , (
                   _('Control auto-detection of document structure.'),
                   [
                       'chapter', 'chapter_mark',
@@ -178,9 +179,9 @@ def add_pipeline_options(parser, plumber):
                       'insert_metadata', 'page_breaks_before',
                       'remove_fake_margins', 'start_reading_at',
                   ]
-                  ),
+                  )),
 
-              'TABLE OF CONTENTS' : (
+              (_('TABLE OF CONTENTS') , (
                   _('Control the automatic generation of a Table of Contents. By '
                   'default, if the source file has a Table of Contents, it will '
                   'be used in preference to the automatically generated one.'),
@@ -189,26 +190,20 @@ def add_pipeline_options(parser, plumber):
                     'toc_threshold', 'max_toc_links', 'no_chapters_in_toc',
                     'use_auto_toc', 'toc_filter', 'duplicate_links_in_toc',
                   ]
-                  ),
+                  )),
 
-              'METADATA' : (_('Options to set metadata in the output'),
+              (_('METADATA') , (_('Options to set metadata in the output'),
                             plumber.metadata_option_names + ['read_metadata_from_opf'],
-                            ),
+                            )),
-              'DEBUG': (_('Options to help with debugging the conversion'),
+              (_('DEBUG'), (_('Options to help with debugging the conversion'),
                         [
                          'verbose',
                          'debug_pipeline',
-                         ]),
+                         ])),
 
+              ))
 
-              }
+    for group, (desc, options) in groups.iteritems():
-
-    group_order = ['', 'LOOK AND FEEL', 'HEURISTIC PROCESSING',
-            'SEARCH AND REPLACE', 'STRUCTURE DETECTION',
-            'TABLE OF CONTENTS', 'METADATA', 'DEBUG']
-
-    for group in group_order:
-        desc, options = groups[group]
         if group:
             group = OptionGroup(parser, group, desc)
             parser.add_option_group(group)

src/calibre/ebooks/conversion/plugins/epub_output.py

@@ -410,7 +410,7 @@ class EPUBOutput(OutputFormatPlugin):
             for tag in XPath('//h:embed')(root):
                 tag.getparent().remove(tag)
             for tag in XPath('//h:object')(root):
-                if tag.get('type', '').lower().strip() in ('image/svg+xml',):
+                if tag.get('type', '').lower().strip() in {'image/svg+xml', 'application/svg+xml'}:
                     continue
                 tag.getparent().remove(tag)
 

src/calibre/ebooks/conversion/plugins/htmlz_output.py

@@ -67,7 +67,8 @@ class HTMLZOutput(OutputFormatPlugin):
 
             fname = u'index'
             if opts.htmlz_title_filename:
-                fname = ascii_filename(unicode(oeb_book.metadata.title[0]))
+                from calibre.utils.filenames import shorten_components_to
+                fname = shorten_components_to(100, (ascii_filename(unicode(oeb_book.metadata.title[0])),))[0]
             with open(os.path.join(tdir, fname+u'.html'), 'wb') as tf:
                 tf.write(html)
 

src/calibre/ebooks/conversion/plugins/oeb_output.py

@@ -54,7 +54,7 @@ class OEBOutput(OutputFormatPlugin):
                         f.write(raw)
 
             for item in oeb_book.manifest:
-                if item.media_type in OEB_STYLES and hasattr(item.data, 'cssText'):
+                if not self.opts.expand_css and item.media_type in OEB_STYLES and hasattr(item.data, 'cssText'):
                     condense_sheet(item.data)
                 path = os.path.abspath(unquote(item.href))
                 dir = os.path.dirname(path)

src/calibre/ebooks/conversion/plumber.py

@@ -363,6 +363,14 @@ OptionRecommendation(name='filter_css',
                 'font-family,color,margin-left,margin-right')
         ),
 
+OptionRecommendation(name='expand_css',
+            recommended_value=False, level=OptionRecommendation.LOW,
+            help=_(
+                'By default, calibre will use the shorthand form for various'
+                ' css properties such as margin, padding, border, etc. This'
+                ' option will cause it to use the full expanded form instead.')
+        ),
+
 OptionRecommendation(name='page_breaks_before',
             recommended_value="//*[name()='h1' or name()='h2']",
             level=OptionRecommendation.LOW,

src/calibre/ebooks/docx/numbering.py

@@ -90,7 +90,7 @@ class Level(object):
                 self.is_numbered = False
                 cs = self.character_style
                 if lt in {'\uf0a7', 'o'} or (
-                    cs.font_family is not inherit and cs.font_family.lower() in {'wingdings', 'symbol'}):
+                    cs is not None and cs.font_family is not inherit and cs.font_family.lower() in {'wingdings', 'symbol'}):
                     self.fmt = {'\uf0a7':'square', 'o':'circle'}.get(lt, 'disc')
                 else:
                     self.bullet_template = lt
@@ -298,7 +298,7 @@ class Numbering(object):
                 for attr in ('list-lvl', 'list-id', 'list-template'):
                     child.attrib.pop(attr, None)
                 val = int(child.get('value'))
-                if last_val == val - 1 or wrap.tag == 'ul':
+                if last_val == val - 1 or wrap.tag == 'ul' or (last_val is None and val == 1):
                     child.attrib.pop('value')
                 last_val = val
             current_run[-1].tail = '\n'

src/calibre/ebooks/docx/to_html.py

@@ -93,10 +93,12 @@ class Convert(object):
         self.framed_map = {}
         self.anchor_map = {}
         self.link_map = defaultdict(list)
+        self.link_source_map = {}
         paras = []
 
         self.log.debug('Converting Word markup to HTML')
         self.read_page_properties(doc)
+        self.current_rels = relationships_by_id
         for wp, page_properties in self.page_map.iteritems():
             self.current_page = page_properties
             if wp.tag.endswith('}p'):
@@ -123,7 +125,7 @@ class Convert(object):
                 dl[-1][0].tail = ']'
                 dl.append(DD())
                 paras = []
-                self.images.rid_map = note.rels[0]
+                self.images.rid_map = self.current_rels = note.rels[0]
                 for wp in note:
                     if wp.tag.endswith('}tbl'):
                         self.tables.register(wp, self.styles)
@@ -157,7 +159,7 @@ class Convert(object):
 
         self.images.rid_map = orig_rid_map
 
-        self.resolve_links(relationships_by_id)
+        self.resolve_links()
 
         self.styles.cascade(self.layers)
 
@@ -378,6 +380,7 @@ class Convert(object):
                     try:
                         hl = hl_xpath(x)[0]
                         self.link_map[hl].append(span)
+                        self.link_source_map[hl] = self.current_rels
                         x.set('is-link', '1')
                     except IndexError:
                         current_hyperlink = None
@@ -455,9 +458,10 @@ class Convert(object):
             wrapper.append(elem)
         return wrapper
 
-    def resolve_links(self, relationships_by_id):
+    def resolve_links(self):
         self.resolved_link_map = {}
         for hyperlink, spans in self.link_map.iteritems():
+            relationships_by_id = self.link_source_map[hyperlink]
             span = spans[0]
             if len(spans) > 1:
                 span = self.wrap_elems(spans, SPAN())

src/calibre/ebooks/metadata/cli.py

@@ -9,7 +9,7 @@ ebook-meta
 import sys, os
 
 from calibre.utils.config import StringConfig
-from calibre.customize.ui import metadata_readers, metadata_writers
+from calibre.customize.ui import metadata_readers, metadata_writers, force_identifiers
 from calibre.ebooks.metadata.meta import get_metadata, set_metadata
 from calibre.ebooks.metadata import string_to_authors, authors_to_sort_string, \
                     title_sort, MetaInformation
@@ -63,6 +63,11 @@ def config():
               help=_('Set the rating. Should be a number between 1 and 5.'))
     c.add_opt('isbn', ['--isbn'],
               help=_('Set the ISBN of the book.'))
+    c.add_opt('identifiers', ['--identifier'], action='append',
+              help=_('Set the identifiers for the book, can be specified multiple times.'
+                     ' For example: --identifier uri:http://acme.com --identifier isbn:12345'
+                     ' To remove an identifier, specify no value, --identifier isbn:'
+                     ' Note that for EPUB files, an identifier marked as the package identifier cannot be removed.'))
     c.add_opt('tags', ['--tags'],
               help=_('Set the tags for the book. Should be a comma separated list.'))
     c.add_opt('book_producer', ['-k', '--book-producer'],
@@ -114,7 +119,7 @@ def do_set_metadata(opts, mi, stream, stream_type):
     for pref in config().option_set.preferences:
         if pref.name in ('to_opf', 'from_opf', 'authors', 'title_sort',
                          'author_sort', 'get_cover', 'cover', 'tags',
-                         'lrf_bookid'):
+                         'lrf_bookid', 'identifiers'):
             continue
         val = getattr(opts, pref.name, None)
         if val is not None:
@@ -136,12 +141,20 @@ def do_set_metadata(opts, mi, stream, stream_type):
         mi.series_index = float(opts.series_index.strip())
     if getattr(opts, 'pubdate', None) is not None:
         mi.pubdate = parse_date(opts.pubdate, assume_utc=False, as_utc=False)
+    if getattr(opts, 'identifiers', None):
+        val = {k.strip():v.strip() for k, v in (x.partition(':')[0::2] for x in opts.identifiers)}
+        if val:
+            orig = mi.get_identifiers()
+            orig.update(val)
+            val = {k:v for k, v in orig.iteritems() if k and v}
+            mi.set_identifiers(val)
 
     if getattr(opts, 'cover', None) is not None:
         ext = os.path.splitext(opts.cover)[1].replace('.', '').upper()
         mi.cover_data = (ext, open(opts.cover, 'rb').read())
 
-    set_metadata(stream, mi, stream_type)
+    with force_identifiers:
+        set_metadata(stream, mi, stream_type)
 
 
 def main(args=sys.argv):

src/calibre/ebooks/metadata/epub.py

@@ -250,7 +250,7 @@ def _write_new_cover(new_cdata, cpath):
     save_cover_data_to(new_cdata, new_cover.name)
     return new_cover
 
-def update_metadata(opf, mi, apply_null=False, update_timestamp=False):
+def update_metadata(opf, mi, apply_null=False, update_timestamp=False, force_identifiers=False):
     for x in ('guide', 'toc', 'manifest', 'spine'):
         setattr(mi, x, None)
     if mi.languages:
@@ -274,10 +274,16 @@ def update_metadata(opf, mi, apply_null=False, update_timestamp=False):
             opf.isbn = None
         if not getattr(mi, 'comments', None):
             opf.comments = None
+    if apply_null or force_identifiers:
+        opf.set_identifiers(mi.get_identifiers())
+    else:
+        orig = opf.get_identifiers()
+        orig.update(mi.get_identifiers())
+        opf.set_identifiers({k:v for k, v in orig.iteritems() if k and v})
     if update_timestamp and mi.timestamp is not None:
         opf.timestamp = mi.timestamp
 
-def set_metadata(stream, mi, apply_null=False, update_timestamp=False):
+def set_metadata(stream, mi, apply_null=False, update_timestamp=False, force_identifiers=False):
     stream.seek(0)
     reader = get_zip_reader(stream, root=os.getcwdu())
     raster_cover = reader.opf.raster_cover
@@ -308,7 +314,7 @@ def set_metadata(stream, mi, apply_null=False, update_timestamp=False):
             traceback.print_exc()
 
     update_metadata(reader.opf, mi, apply_null=apply_null,
-                    update_timestamp=update_timestamp)
+                    update_timestamp=update_timestamp, force_identifiers=force_identifiers)
 
     newopf = StringIO(reader.opf.render())
     if isinstance(reader.archive, LocalZipFile):

src/calibre/ebooks/metadata/opf2.py

@@ -959,6 +959,33 @@ class OPF(object):  # {{{
                         identifiers['isbn'] = val
         return identifiers
 
+    def set_identifiers(self, identifiers):
+        identifiers = identifiers.copy()
+        uuid_id = None
+        for attr in self.root.attrib:
+            if attr.endswith('unique-identifier'):
+                uuid_id = self.root.attrib[attr]
+                break
+
+        for x in self.XPath(
+            'descendant::*[local-name() = "identifier"]')(
+                    self.metadata):
+            xid = x.get('id', None)
+            is_package_identifier = uuid_id is not None and uuid_id == xid
+            typ = {val for attr, val in x.attrib.iteritems() if attr.endswith('scheme')}
+            if is_package_identifier:
+                typ = tuple(typ)
+                if typ and typ[0].lower() in identifiers:
+                    self.set_text(x, identifiers.pop(typ[0].lower()))
+                continue
+            if typ and not (typ & {'calibre', 'uuid'}):
+                x.getparent().remove(x)
+
+        for typ, val in identifiers.iteritems():
+            attrib = {'{%s}scheme'%self.NAMESPACES['opf']: typ.upper()}
+            self.set_text(self.create_metadata_element(
+                'identifier', attrib=attrib), unicode(val))
+
     @dynamic_property
     def application_id(self):
 

src/calibre/ebooks/metadata/sources/amazon.py

@@ -46,14 +46,14 @@ class Worker(Thread):  # Get details {{{
 
         months = {
                 'de': {
-            1: ['jän'],
+            1: ['jän', 'januar'],
             2: ['februar'],
             3: ['märz'],
             5: ['mai'],
             6: ['juni'],
             7: ['juli'],
-            10: ['okt'],
+            10: ['okt', 'oktober'],
-            12: ['dez']
+            12: ['dez', 'dezember']
             },
                 'it': {
             1: ['enn'],

src/calibre/ebooks/metadata/sources/identify.py

@@ -78,7 +78,6 @@ class xISBN(Thread):
             self.tb = traceback.format_exception()
 
 
-
 class ISBNMerge(object):
 
     def __init__(self, log):
@@ -361,7 +360,7 @@ def merge_identify_results(result_map, log):
 
 # }}}
 
-def identify(log, abort, # {{{
+def identify(log, abort,  # {{{
         title=None, authors=None, identifiers={}, timeout=30):
     if title == _('Unknown'):
         title = None
@@ -492,7 +491,6 @@ def identify(log, abort, # {{{
     log('We have %d merged results, merging took: %.2f seconds' %
             (len(results), time.time() - start_time))
 
-
     max_tags = msprefs['max_tags']
     for r in results:
         r.tags = r.tags[:max_tags]
@@ -514,7 +512,7 @@ def identify(log, abort, # {{{
     return results
 # }}}
 
-def urls_from_identifiers(identifiers): # {{{
+def urls_from_identifiers(identifiers):  # {{{
     identifiers = dict([(k.lower(), v) for k, v in identifiers.iteritems()])
     ans = []
     for plugin in all_metadata_plugins():
@@ -539,18 +537,17 @@ def urls_from_identifiers(identifiers): # {{{
     if oclc:
         ans.append(('OCLC', 'oclc', oclc,
             'http://www.worldcat.org/oclc/'+oclc))
-    url = identifiers.get('uri', None)
+    for x in ('uri', 'url'):
-    if url is None:
+        url = identifiers.get(x, None)
-        url = identifiers.get('url', None)
+        if url and url.startswith('http'):
-    if url and url.startswith('http'):
+            url = url[:8].replace('|', ':') + url[8:].replace('|', ',')
-        url = url[:8].replace('|', ':') + url[8:].replace('|', ',')
+            parts = urlparse(url)
-        parts = urlparse(url)
+            name = parts.netloc
-        name = parts.netloc
+            ans.append((name, x, url, url))
-        ans.append((name, 'url', url, url))
     return ans
 # }}}
 
-if __name__ == '__main__': # tests {{{
+if __name__ == '__main__':  # tests {{{
     # To run these test use: calibre-debug -e
     # src/calibre/ebooks/metadata/sources/identify.py
     from calibre.ebooks.metadata.sources.test import (test_identify,
@@ -563,7 +560,7 @@ if __name__ == '__main__': # tests {{{
             ),
 
 
-            ( # An e-book ISBN not on Amazon, one of the authors is
+            (  # An e-book ISBN not on Amazon, one of the authors is
               # unknown to Amazon
                 {'identifiers':{'isbn': '9780307459671'},
                     'title':'Invisible Gorilla', 'authors':['Christopher Chabris']},
@@ -580,7 +577,7 @@ if __name__ == '__main__': # tests {{{
 
             ),
 
-            ( # Sophisticated comment formatting
+            (  # Sophisticated comment formatting
                 {'identifiers':{'isbn': '9781416580829'}},
                 [title_test('Angels & Demons',
                     exact=True), authors_test(['Dan Brown'])]
@@ -594,7 +591,7 @@ if __name__ == '__main__': # tests {{{
             ),
 
         ]
-    #test_identify(tests[1:2])
+    # test_identify(tests[1:2])
     test_identify(tests)
 # }}}
 

src/calibre/ebooks/mobi/writer8/main.py

@@ -151,7 +151,7 @@ class KF8Writer(object):
 
         for item in self.oeb.manifest:
             if item.media_type in OEB_STYLES:
-                if hasattr(item.data, 'cssText'):
+                if not self.opts.expand_css and hasattr(item.data, 'cssText'):
                     condense_sheet(self.data(item))
                 data = self.data(item).cssText
                 sheets[item.href] = len(self.flows)

src/calibre/ebooks/oeb/base.py

@@ -107,8 +107,7 @@ def iterlinks(root, find_links_in_css=True):
     :param root: A valid lxml.etree element.
     '''
     assert etree.iselement(root)
-    link_attrs = set(html.defs.link_attrs)
+    link_attrs = set(html.defs.link_attrs) | {XLINK('href'), 'poster'}
-    link_attrs.add(XLINK('href'))
 
     for el in root.iter():
         attribs = el.attrib

src/calibre/ebooks/oeb/polish/container.py

@@ -7,11 +7,12 @@ __license__   = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, logging, sys, hashlib, uuid, re
+import os, logging, sys, hashlib, uuid, re, shutil, copy
 from collections import defaultdict
 from io import BytesIO
 from urllib import unquote as urlunquote, quote as urlquote
 from urlparse import urlparse
+from future_builtins import zip
 
 from lxml import etree
 
@@ -30,6 +31,7 @@ from calibre.ebooks.oeb.base import (
 from calibre.ebooks.oeb.polish.errors import InvalidBook, DRMError
 from calibre.ebooks.oeb.parse_utils import NotHTML, parse_html, RECOVER_PARSER
 from calibre.ptempfile import PersistentTemporaryDirectory, PersistentTemporaryFile
+from calibre.utils.filenames import nlinks_file, hardlink_file
 from calibre.utils.ipc.simple_worker import fork_job, WorkerError
 from calibre.utils.logging import default_log
 from calibre.utils.zipfile import ZipFile
@@ -47,7 +49,30 @@ class CSSPreProcessor(cssp):
     def __call__(self, data):
         return self.MS_PAT.sub(self.ms_sub, data)
 
-class Container(object):
+def clone_dir(src, dest):
+    ' Clone a directory using hard links for the files, dest must already exist '
+    for x in os.listdir(src):
+        dpath = os.path.join(dest, x)
+        spath = os.path.join(src, x)
+        if os.path.isdir(spath):
+            os.mkdir(dpath)
+            clone_dir(spath, dpath)
+        else:
+            try:
+                hardlink_file(spath, dpath)
+            except:
+                shutil.copy2(spath, dpath)
+
+def clone_container(container, dest_dir):
+    ' Efficiently clone a container using hard links '
+    dest_dir = os.path.abspath(os.path.realpath(dest_dir))
+    clone_data = container.clone_data(dest_dir)
+    cls = type(container)
+    if cls is Container:
+        return cls(None, None, container.log, clone_data=clone_data)
+    return cls(None, container.log, clone_data=clone_data)
+
+class Container(object):  # {{{
 
     '''
     A container represents an Open EBook as a directory full of files and an
@@ -67,8 +92,8 @@ class Container(object):
 
     book_type = 'oeb'
 
-    def __init__(self, rootpath, opfpath, log):
+    def __init__(self, rootpath, opfpath, log, clone_data=None):
-        self.root = os.path.abspath(rootpath)
+        self.root = clone_data['root'] if clone_data is not None else os.path.abspath(rootpath)
         self.log = log
         self.html_preprocessor = HTMLPreProcessor()
         self.css_preprocessor = CSSPreProcessor()
@@ -79,6 +104,14 @@ class Container(object):
         self.dirtied = set()
         self.encoding_map = {}
         self.pretty_print = set()
+        self.cloned = False
+
+        if clone_data is not None:
+            self.cloned = True
+            for x in ('name_path_map', 'opf_name', 'mime_map', 'pretty_print', 'encoding_map'):
+                setattr(self, x, clone_data[x])
+            self.opf_dir = os.path.dirname(self.name_path_map[self.opf_name])
+            return
 
         # Map of relative paths with '/' separators from root of unzipped ePub
         # to absolute paths on filesystem with os-specific separators
@@ -105,6 +138,21 @@ class Container(object):
             if name in self.mime_map:
                 self.mime_map[name] = item.get('media-type')
 
+    def clone_data(self, dest_dir):
+        Container.commit(self, keep_parsed=True)
+        self.cloned = True
+        clone_dir(self.root, dest_dir)
+        return {
+            'root': dest_dir,
+            'opf_name': self.opf_name,
+            'mime_map': self.mime_map.copy(),
+            'pretty_print': set(self.pretty_print),
+            'encoding_map': self.encoding_map.copy(),
+            'name_path_map': {
+                name:os.path.join(dest_dir, os.path.relpath(path, self.root))
+                for name, path in self.name_path_map.iteritems()}
+        }
+
     def abspath_to_name(self, fullpath):
         return self.relpath(os.path.abspath(fullpath)).replace(os.sep, '/')
 
@@ -181,6 +229,14 @@ class Container(object):
         data, self.used_encoding = xml_to_unicode(data)
         return fix_data(data)
 
+    @property
+    def names_that_need_not_be_manifested(self):
+        return {self.opf_name}
+
+    @property
+    def names_that_must_not_be_removed(self):
+        return {self.opf_name}
+
     def parse_xml(self, data):
         data, self.used_encoding = xml_to_unicode(
             data, strip_encoding_pats=True, assume_utf8=True, resolve_entities=True)
@@ -262,21 +318,42 @@ class Container(object):
             for item in self.opf_xpath('//opf:guide/opf:reference[@href and @type]')}
 
     @property
-    def spine_items(self):
+    def spine_iter(self):
         manifest_id_map = self.manifest_id_map
-
+        non_linear = []
-        linear, non_linear = [], []
         for item in self.opf_xpath('//opf:spine/opf:itemref[@idref]'):
             idref = item.get('idref')
             name = manifest_id_map.get(idref, None)
             path = self.name_path_map.get(name, None)
             if path:
                 if item.get('linear', 'yes') == 'yes':
-                    yield path
+                    yield item, name, True
                 else:
-                    non_linear.append(path)
+                    non_linear.append((item, name))
-        for path in non_linear:
+        for item, name in non_linear:
-            yield path
+            yield item, name, False
+
+    @property
+    def spine_names(self):
+        for item, name, linear in self.spine_iter:
+            yield name, linear
+
+    @property
+    def spine_items(self):
+        for name, linear in self.spine_names:
+            yield self.name_path_map[name]
+
+    def remove_from_spine(self, spine_items, remove_if_no_longer_in_spine=True):
+        nixed = set()
+        for (name, remove), (item, xname, linear) in zip(spine_items, self.spine_iter):
+            if remove and name == xname:
+                self.remove_from_xml(item)
+                nixed.add(name)
+        if remove_if_no_longer_in_spine:
+            # Remove from the book if no longer in spine
+            nixed -= {name for name, linear in self.spine_names}
+            for name in nixed:
+                self.remove_item(name)
 
     def remove_item(self, name):
         '''
@@ -293,12 +370,23 @@ class Container(object):
                 self.remove_from_xml(elem)
                 self.dirty(self.opf_name)
         if removed:
+            for spine in self.opf_xpath('//opf:spine'):
+                tocref = spine.attrib.get('toc', None)
+                if tocref and tocref in removed:
+                    spine.attrib.pop('toc', None)
+                    self.dirty(self.opf_name)
+
             for item in self.opf_xpath('//opf:spine/opf:itemref[@idref]'):
                 idref = item.get('idref')
                 if idref in removed:
                     self.remove_from_xml(item)
                     self.dirty(self.opf_name)
 
+            for meta in self.opf_xpath('//opf:meta[@name="cover" and @content]'):
+                if meta.get('content') in removed:
+                    self.remove_from_xml(meta)
+                    self.dirty(self.opf_name)
+
         for item in self.opf_xpath('//opf:guide/opf:reference[@href]'):
             if self.href_to_name(item.get('href'), self.opf_name) == name:
                 self.remove_from_xml(item)
@@ -436,9 +524,19 @@ class Container(object):
         self.dirtied.discard(name)
         if not keep_parsed:
             self.parsed_cache.pop(name)
-        with open(self.name_path_map[name], 'wb') as f:
+        dest = self.name_path_map[name]
+        if self.cloned and nlinks_file(dest) > 1:
+            # Decouple this file from its links
+            os.unlink(dest)
+        with open(dest, 'wb') as f:
             f.write(data)
 
+    def filesize(self, name):
+        if name in self.dirtied:
+            self.commit_item(name, keep_parsed=True)
+        path = self.name_to_abspath(name)
+        return os.path.getsize(path)
+
     def open(self, name, mode='rb'):
         ''' Open the file pointed to by name for direct read/write. Note that
         this will commit the file if it is dirtied and remove it from the parse
@@ -451,11 +549,18 @@ class Container(object):
         base = os.path.dirname(path)
         if not os.path.exists(base):
             os.makedirs(base)
+        else:
+            if self.cloned and mode not in {'r', 'rb'} and os.path.exists(path) and nlinks_file(path) > 1:
+                # Decouple this file from its links
+                temp = path + 'xxx'
+                shutil.copyfile(path, temp)
+                os.unlink(path)
+                os.rename(temp, path)
         return open(path, mode)
 
-    def commit(self, outpath=None):
+    def commit(self, outpath=None, keep_parsed=False):
         for name in tuple(self.dirtied):
-            self.commit_item(name)
+            self.commit_item(name, keep_parsed=keep_parsed)
 
     def compare_to(self, other):
         if set(self.name_path_map) != set(other.name_path_map):
@@ -467,6 +572,7 @@ class Container(object):
                 if f1.read() != f2.read():
                     mismatches.append('The file %s is not the same'%name)
         return '\n'.join(mismatches)
+# }}}
 
 # EPUB {{{
 class InvalidEpub(InvalidBook):
@@ -487,9 +593,17 @@ class EpubContainer(Container):
             'rights.xml': False,
     }
 
-    def __init__(self, pathtoepub, log):
+    def __init__(self, pathtoepub, log, clone_data=None, tdir=None):
+        if clone_data is not None:
+            super(EpubContainer, self).__init__(None, None, log, clone_data=clone_data)
+            for x in ('pathtoepub', 'container', 'obfuscated_fonts'):
+                setattr(self, x, clone_data[x])
+            return
+
         self.pathtoepub = pathtoepub
-        tdir = self.root = os.path.abspath(os.path.realpath(PersistentTemporaryDirectory('_epub_container')))
+        if tdir is None:
+            tdir = os.path.abspath(os.path.realpath(PersistentTemporaryDirectory('_epub_container')))
+        self.root = tdir
         with open(self.pathtoepub, 'rb') as stream:
             try:
                 zf = ZipFile(stream)
@@ -527,6 +641,41 @@ class EpubContainer(Container):
         if 'META-INF/encryption.xml' in self.name_path_map:
             self.process_encryption()
 
+    def clone_data(self, dest_dir):
+        ans = super(EpubContainer, self).clone_data(dest_dir)
+        ans['pathtoepub'] = self.pathtoepub
+        ans['obfuscated_fonts'] = self.obfuscated_fonts.copy()
+        ans['container'] = copy.deepcopy(self.container)
+        return ans
+
+    @property
+    def names_that_need_not_be_manifested(self):
+        return super(EpubContainer, self).names_that_need_not_be_manifested | {'META-INF/' + x for x in self.META_INF}
+
+    @property
+    def names_that_must_not_be_removed(self):
+        return super(EpubContainer, self).names_that_must_not_be_removed | {'META-INF/container.xml'}
+
+    def remove_item(self, name):
+        # Handle removal of obfuscated fonts
+        if name == 'META-INF/encryption.xml':
+            self.obfuscated_fonts.clear()
+        if name in self.obfuscated_fonts:
+            self.obfuscated_fonts.pop(name, None)
+            enc = self.parsed('META-INF/encryption.xml')
+            for em in enc.xpath('//*[local-name()="EncryptionMethod" and @Algorithm]'):
+                alg = em.get('Algorithm')
+                if alg not in {ADOBE_OBFUSCATION, IDPF_OBFUSCATION}:
+                    continue
+                try:
+                    cr = em.getparent().xpath('descendant::*[local-name()="CipherReference" and @URI]')[0]
+                except (IndexError, ValueError, KeyError):
+                    continue
+                if name == self.href_to_name(cr.get('URI')):
+                    self.remove_from_xml(em.getparent())
+                    self.dirty('META-INF/encryption.xml')
+        super(EpubContainer, self).remove_item(name)
+
     def process_encryption(self):
         fonts = {}
         enc = self.parsed('META-INF/encryption.xml')
@@ -534,7 +683,10 @@ class EpubContainer(Container):
             alg = em.get('Algorithm')
             if alg not in {ADOBE_OBFUSCATION, IDPF_OBFUSCATION}:
                 raise DRMError()
-            cr = em.getparent().xpath('descendant::*[local-name()="CipherReference" and @URI]')[0]
+            try:
+                cr = em.getparent().xpath('descendant::*[local-name()="CipherReference" and @URI]')[0]
+            except (IndexError, ValueError, KeyError):
+                continue
             name = self.href_to_name(cr.get('URI'))
             path = self.name_path_map.get(name, None)
             if path is not None:
@@ -578,8 +730,8 @@ class EpubContainer(Container):
             decrypt_font(tkey, path, alg)
             self.obfuscated_fonts[font] = (alg, tkey)
 
-    def commit(self, outpath=None):
+    def commit(self, outpath=None, keep_parsed=False):
-        super(EpubContainer, self).commit()
+        super(EpubContainer, self).commit(keep_parsed=keep_parsed)
         for name in self.obfuscated_fonts:
             if name not in self.name_path_map:
                 continue
@@ -620,9 +772,17 @@ class AZW3Container(Container):
 
     book_type = 'azw3'
 
-    def __init__(self, pathtoazw3, log):
+    def __init__(self, pathtoazw3, log, clone_data=None, tdir=None):
+        if clone_data is not None:
+            super(AZW3Container, self).__init__(None, None, log, clone_data=clone_data)
+            for x in ('pathtoazw3', 'obfuscated_fonts'):
+                setattr(self, x, clone_data[x])
+            return
+
         self.pathtoazw3 = pathtoazw3
-        tdir = self.root = os.path.abspath(os.path.realpath(PersistentTemporaryDirectory('_azw3_container')))
+        if tdir is None:
+            tdir = os.path.abspath(os.path.realpath(PersistentTemporaryDirectory('_azw3_container')))
+        self.root = tdir
         with open(pathtoazw3, 'rb') as stream:
             raw = stream.read(3)
             if raw == b'TPZ':
@@ -659,8 +819,14 @@ class AZW3Container(Container):
         super(AZW3Container, self).__init__(tdir, opf_path, log)
         self.obfuscated_fonts = {x.replace(os.sep, '/') for x in obfuscated_fonts}
 
-    def commit(self, outpath=None):
+    def clone_data(self, dest_dir):
-        super(AZW3Container, self).commit()
+        ans = super(AZW3Container, self).clone_data(dest_dir)
+        ans['pathtoazw3'] = self.pathtoazw3
+        ans['obfuscated_fonts'] = self.obfuscated_fonts.copy()
+        return ans
+
+    def commit(self, outpath=None, keep_parsed=False):
+        super(AZW3Container, self).commit(keep_parsed=keep_parsed)
         if outpath is None:
             outpath = self.pathtoazw3
         from calibre.ebooks.conversion.plumber import Plumber, create_oebbook
@@ -675,11 +841,11 @@ class AZW3Container(Container):
         outp.convert(oeb, outpath, inp, plumber.opts, default_log)
 # }}}
 
-def get_container(path, log=None):
+def get_container(path, log=None, tdir=None):
     if log is None:
         log = default_log
     ebook = (AZW3Container if path.rpartition('.')[-1].lower() in {'azw3', 'mobi'}
-            else EpubContainer)(path, log)
+            else EpubContainer)(path, log, tdir=tdir)
     return ebook
 
 def test_roundtrip():

src/calibre/ebooks/oeb/polish/cover.py

@@ -33,6 +33,21 @@ def set_azw3_cover(container, cover_path, report):
     container.dirty(container.opf_name)
     report('Cover updated' if found else 'Cover inserted')
 
+def get_azw3_raster_cover_name(container):
+    items = container.opf_xpath('//opf:guide/opf:reference[@href and contains(@type, "cover")]')
+    if items:
+        return container.href_to_name(items[0].get('href'))
+
+def get_raster_cover_name(container):
+    if container.book_type == 'azw3':
+        return get_azw3_raster_cover_name(container)
+    return find_cover_image(container, strict=True)
+
+def get_cover_page_name(container):
+    if container.book_type == 'azw3':
+        return
+    return find_cover_page(container)
+
 def set_cover(container, cover_path, report):
     if container.book_type == 'azw3':
         set_azw3_cover(container, cover_path, report)
@@ -52,7 +67,7 @@ COVER_TYPES = {
     'other.ms-coverimage', 'other.ms-thumbimage-standard',
     'other.ms-thumbimage', 'thumbimagestandard', 'cover'}
 
-def find_cover_image(container):
+def find_cover_image(container, strict=False):
     'Find a raster image marked as a cover in the OPF'
     manifest_id_map = container.manifest_id_map
     mm = container.mime_map
@@ -69,6 +84,9 @@ def find_cover_image(container):
         if ref_type.lower() == 'cover' and is_raster_image(mm.get(name, None)):
             return name
 
+    if strict:
+        return
+
     # Find the largest image from all possible guide cover items
     largest_cover = (None, 0)
     for ref_type, name in guide_type_map.iteritems():

src/calibre/ebooks/oeb/polish/tests/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+
+

src/calibre/ebooks/oeb/polish/tests/base.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import os, unittest, shutil
+
+from calibre.ptempfile import PersistentTemporaryDirectory
+from calibre.utils.logging import DevNull
+import calibre.ebooks.oeb.polish.container as pc
+
+def get_cache():
+    from calibre.constants import cache_dir
+    cache = os.path.join(cache_dir(), 'polish-test')
+    if not os.path.exists(cache):
+        os.mkdir(cache)
+    return cache
+
+def needs_recompile(obj, srcs):
+    if isinstance(srcs, type('')):
+        srcs = [srcs]
+    try:
+        obj_mtime = os.stat(obj).st_mtime
+    except OSError:
+        return True
+    for src in srcs:
+        if os.stat(src).st_mtime > obj_mtime:
+            return True
+    return False
+
+def build_book(src, dest, args=()):
+    from calibre.ebooks.conversion.cli import main
+    main(['ebook-convert', src, dest] + list(args))
+
+def get_simple_book(fmt='epub'):
+    cache = get_cache()
+    ans = os.path.join(cache, 'simple.'+fmt)
+    src = os.path.join(os.path.dirname(__file__), 'simple.html')
+    if needs_recompile(ans, src):
+        x = src.replace('simple.html', 'index.html')
+        raw = open(src, 'rb').read().decode('utf-8')
+        raw = raw.replace('LMONOI', P('fonts/liberation/LiberationMono-Italic.ttf'))
+        raw = raw.replace('LMONO', P('fonts/liberation/LiberationMono-Regular.ttf'))
+        raw = raw.replace('IMAGE1', I('marked.png'))
+        try:
+            with open(x, 'wb') as f:
+                f.write(raw.encode('utf-8'))
+            build_book(x, ans, args=['--level1-toc=//h:h2', '--language=en', '--authors=Kovid Goyal',
+                                        '--cover=' + I('lt.png')])
+        finally:
+            try:
+                os.remove('index.html')
+            except:
+                pass
+    return ans
+
+devnull = DevNull()
+
+class BaseTest(unittest.TestCase):
+
+    longMessage = True
+    maxDiff = None
+
+    def setUp(self):
+        pc.default_log = devnull
+        self.tdir = PersistentTemporaryDirectory(suffix='-polish-test')
+
+    def tearDown(self):
+        shutil.rmtree(self.tdir, ignore_errors=True)
+        del self.tdir
+

src/calibre/ebooks/oeb/polish/tests/container.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import os
+
+from calibre.ebooks.oeb.polish.tests.base import BaseTest, get_simple_book
+
+from calibre.ebooks.oeb.polish.container import get_container, clone_container
+from calibre.utils.filenames import nlinks_file
+
+class ContainerTests(BaseTest):
+
+    def test_clone(self):
+        ' Test cloning of containers '
+        for fmt in ('epub', 'azw3'):
+            base = os.path.join(self.tdir, fmt + '-')
+            book = get_simple_book(fmt)
+            tdir = base + 'first'
+            os.mkdir(tdir)
+            c1 = get_container(book, tdir=tdir)
+            tdir = base + 'second'
+            os.mkdir(tdir)
+            c2 = clone_container(c1, tdir)
+
+            for c in (c1, c2):
+                for name, path in c.name_path_map.iteritems():
+                    self.assertEqual(2, nlinks_file(path), 'The file %s is not linked' % name)
+
+            for name in c1.name_path_map:
+                self.assertIn(name, c2.name_path_map)
+                self.assertEqual(c1.open(name).read(), c2.open(name).read(), 'The file %s differs' % name)
+
+            spine_names = tuple(x[0] for x in c1.spine_names)
+            text = spine_names[0]
+            root = c2.parsed(text)
+            root.xpath('//*[local-name()="body"]')[0].set('id', 'changed id for test')
+            c2.dirty(text)
+            c2.commit_item(text)
+            for c in (c1, c2):
+                self.assertEqual(1, nlinks_file(c.name_path_map[text]))
+            self.assertNotEqual(c1.open(text).read(), c2.open(text).read())
+
+            name = spine_names[1]
+            with c1.open(name, mode='r+b') as f:
+                f.seek(0, 2)
+                f.write(b'    ')
+            for c in (c1, c2):
+                self.assertEqual(1, nlinks_file(c.name_path_map[name]))
+            self.assertNotEqual(c1.open(name).read(), c2.open(name).read())
+
+            x = base + 'out.' + fmt
+            for c in (c1, c2):
+                c.commit(outpath=x)
+
+    def test_file_removal(self):
+        ' Test removal of files from the container '
+        book = get_simple_book()
+        c = get_container(book, tdir=self.tdir)
+        files = ('toc.ncx', 'cover.png', 'titlepage.xhtml')
+        self.assertIn('titlepage.xhtml', {x[0] for x in c.spine_names})
+        self.assertTrue(c.opf_xpath('//opf:meta[@name="cover"]'))
+        for x in files:
+            c.remove_item(x)
+        self.assertIn(c.opf_name, c.dirtied)
+        self.assertNotIn('titlepage.xhtml', {x[0] for x in c.spine_names})
+        self.assertFalse(c.opf_xpath('//opf:meta[@name="cover"]'))
+        raw = c.serialize_item(c.opf_name).decode('utf-8')
+        for x in files:
+            self.assertNotIn(x, raw)
+

src/calibre/ebooks/oeb/polish/tests/index.html

@@ -0,0 +1,64 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>A simple test page</title>
+		<meta http-equiv="content-type" content="text/html; charset=utf-8"/>
+        <style type="text/css">
+@font-face {
+    font-family: "Liberation Mono";
+    src: url(/home/kovid/work/calibre/resources/fonts/liberation/LiberationMono-Regular.ttf);
+}
+
+@font-face {
+    font-family: "Liberation Mono";
+    src: url(/home/kovid/work/calibre/resources/fonts/liberation/LiberationMono-Italic.ttf);
+    font-style: italic;
+}
+
+h1 {
+    color: DarkCyan;
+    text-align: center;
+}
+
+p { font-family: "Liberation Mono"; }
+        </style>
+	</head>
+	<body>
+    <h2>A simple test page</h2>
+<!--lorem-->
+<p>To pursue pleasure rationally encounter consequences that are extremely
+painful.</p>
+
+<p>Nor again is there anyone who loves or pursues or desires to obtain pain of
+itself, because it is pain, but because occasionally circumstances occur in
+which toil and pain can procure him some great pleasure. To take a trivial
+example, which of us ever undertakes laborious physical exercise, except to
+obtain some advantage from it? But who has any right to find fault with a man
+who chooses to enjoy a pleasure that has no <em>annoying</em> consequences, or one who
+avoids a pain that produces no resultant pleasure?</p>
+
+<div style="text-align:center"><img alt="test" src="/home/kovid/work/calibre/resources/images/marked.png"></div>
+
+<p>On the other hand.</p>
+
+<!--/lorem-->
+
+<h2>Another test page</h2>
+
+<!--lorem-->
+<p>The great explorer of the truth, the master-builder of human happiness. No
+one rejects, dislikes, or avoids pleasure itself, because it is pleasure, but
+because those who do not know how to pursue pleasure rationally encounter
+consequences that are extremely painful.</p>
+
+<p>Nor again is there anyone who loves or pursues or desires to obtain pain of
+itself, because it is pain, but because occasionally circumstances occur in
+which toil and pain can procure him some great pleasure. To take a trivial
+example, which of us ever undertakes laborious physical exercise, except to
+obtain some advantage from it? But who has any right.</p>
+<!--/lorem-->
+
+
+	</body>
+</html>
+

src/calibre/ebooks/oeb/polish/tests/main.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+
+try:
+    import init_calibre  # noqa
+except ImportError:
+    pass
+
+import os, unittest
+
+def find_tests():
+    return unittest.defaultTestLoader.discover(os.path.dirname(os.path.abspath(__file__)), pattern='*.py')
+
+if __name__ == '__main__':
+    from calibre.db.tests.main import run_tests
+    run_tests(find_tests=find_tests)
+
+

src/calibre/ebooks/oeb/polish/tests/simple.html

@@ -0,0 +1,64 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>A simple test page</title>
+		<meta http-equiv="content-type" content="text/html; charset=utf-8"/>
+        <style type="text/css">
+@font-face {
+    font-family: "Liberation Mono";
+    src: url(LMONO);
+}
+
+@font-face {
+    font-family: "Liberation Mono";
+    src: url(LMONOI);
+    font-style: italic;
+}
+
+h1 {
+    color: DarkCyan;
+    text-align: center;
+}
+
+p { font-family: "Liberation Mono"; }
+        </style>
+	</head>
+	<body>
+    <h2>A simple test page</h2>
+<!--lorem-->
+<p>To pursue pleasure rationally encounter consequences that are extremely
+painful.</p>
+
+<p>Nor again is there anyone who loves or pursues or desires to obtain pain of
+itself, because it is pain, but because occasionally circumstances occur in
+which toil and pain can procure him some great pleasure. To take a trivial
+example, which of us ever undertakes laborious physical exercise, except to
+obtain some advantage from it? But who has any right to find fault with a man
+who chooses to enjoy a pleasure that has no <em>annoying</em> consequences, or one who
+avoids a pain that produces no resultant pleasure?</p>
+
+<div style="text-align:center"><img alt="test" src="IMAGE1"></div>
+
+<p>On the other hand.</p>
+
+<!--/lorem-->
+
+<h2>Another test page</h2>
+
+<!--lorem-->
+<p>The great explorer of the truth, the master-builder of human happiness. No
+one rejects, dislikes, or avoids pleasure itself, because it is pleasure, but
+because those who do not know how to pursue pleasure rationally encounter
+consequences that are extremely painful.</p>
+
+<p>Nor again is there anyone who loves or pursues or desires to obtain pain of
+itself, because it is pain, but because occasionally circumstances occur in
+which toil and pain can procure him some great pleasure. To take a trivial
+example, which of us ever undertakes laborious physical exercise, except to
+obtain some advantage from it? But who has any right.</p>
+<!--/lorem-->
+
+
+	</body>
+</html>
+

src/calibre/gui2/__init__.py

@@ -873,6 +873,11 @@ class Application(QApplication):
                 'MessageBoxWarning': u'dialog_warning.png',
                 'MessageBoxCritical': u'dialog_error.png',
                 'MessageBoxQuestion': u'dialog_question.png',
+                # These two are used to calculate the sizes for the doc widget
+                # title bar buttons, therefore, they have to exist. The actual
+                # icon is not used.
+                'TitleBarCloseButton': u'window-close.png',
+                'TitleBarNormalButton': u'window-close.png',
                 }.iteritems():
             if v not in pcache:
                 p = I(v)

src/calibre/gui2/actions/mark_books.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from functools import partial
+
+from PyQt4.Qt import QTimer, QApplication, Qt
+
+from calibre.gui2 import error_dialog
+from calibre.gui2.actions import InterfaceAction
+
+class MarkBooksAction(InterfaceAction):
+
+    name = 'Mark Books'
+    action_spec = (_('Mark Books'), 'marked.png', _('Temporarily mark books'), 'Ctrl+M')
+    action_type = 'current'
+    action_add_menu = True
+    dont_add_to = frozenset([
+        'toolbar-device', 'context-menu-device', 'menubar-device', 'context-menu-cover-browser'])
+    action_menu_clone_qaction = _('Toggle mark for selected books')
+
+    accepts_drops = True
+
+    def accept_enter_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def accept_drag_move_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library"):
+            return True
+        return False
+
+    def drop_event(self, event, mime_data):
+        mime = 'application/calibre+from_library'
+        if mime_data.hasFormat(mime):
+            self.dropped_ids = tuple(map(int, str(mime_data.data(mime)).split()))
+            QTimer.singleShot(1, self.do_drop)
+            return True
+        return False
+
+    def do_drop(self):
+        book_ids = self.dropped_ids
+        del self.dropped_ids
+        if book_ids:
+            self.toggle_ids(book_ids)
+
+    def genesis(self):
+        self.qaction.triggered.connect(self.toggle_selected)
+        self.menu = m = self.qaction.menu()
+        m.aboutToShow.connect(self.about_to_show_menu)
+        ma = partial(self.create_menu_action, m)
+        self.show_marked_action = a = ma('show-marked', _('Show marked books'), icon='search.png', shortcut='Shift+Ctrl+M')
+        a.triggered.connect(self.show_marked)
+        self.clear_marked_action = a = ma('clear-all-marked', _('Clear all marked books'), icon='clear_left.png')
+        a.triggered.connect(self.clear_all_marked)
+        m.addSeparator()
+        self.mark_author_action = a = ma('mark-author', _('Mark all books by selected author(s)'), icon='plus.png')
+        a.triggered.connect(partial(self.mark_field, 'authors', True))
+        self.mark_series_action = a = ma('mark-series', _('Mark all books in the selected series'), icon='plus.png')
+        a.triggered.connect(partial(self.mark_field, 'series', True))
+        m.addSeparator()
+        self.unmark_author_action = a = ma('unmark-author', _('Clear all books by selected author(s)'), icon='minus.png')
+        a.triggered.connect(partial(self.mark_field, 'authors', False))
+        self.unmark_series_action = a = ma('unmark-series', _('Clear all books in the selected series'), icon='minus.png')
+        a.triggered.connect(partial(self.mark_field, 'series', False))
+
+    def gui_layout_complete(self):
+        for x in self.gui.bars_manager.main_bars + self.gui.bars_manager.child_bars:
+            try:
+                w = x.widgetForAction(self.qaction)
+                w.installEventFilter(self)
+            except:
+                continue
+
+    def eventFilter(self, obj, ev):
+        if ev.type() == ev.MouseButtonPress and ev.button() == Qt.LeftButton:
+            mods = QApplication.keyboardModifiers()
+            if mods & Qt.ControlModifier or mods & Qt.ShiftModifier:
+                self.show_marked()
+                return True
+        return False
+
+    def about_to_show_menu(self):
+        db = self.gui.current_db
+        num = len(db.data.marked_ids)
+        text = _('Show marked book') if num == 1 else (_('Show marked books') + (' (%d)' % num))
+        self.show_marked_action.setText(text)
+
+    def location_selected(self, loc):
+        enabled = loc == 'library'
+        self.qaction.setEnabled(enabled)
+
+    def toggle_selected(self):
+        book_ids = self._get_selected_ids()
+        if book_ids:
+            self.toggle_ids(book_ids)
+
+    def _get_selected_ids(self):
+        rows = self.gui.library_view.selectionModel().selectedRows()
+        if not rows or len(rows) == 0:
+            d = error_dialog(self.gui, _('Cannot mark'), _('No books selected'))
+            d.exec_()
+            return set([])
+        return set(map(self.gui.library_view.model().id, rows))
+
+    def toggle_ids(self, book_ids):
+        self.gui.current_db.data.toggle_marked_ids(book_ids)
+
+    def show_marked(self):
+        self.gui.search.set_search_string('marked:true')
+
+    def clear_all_marked(self):
+        self.gui.current_db.data.set_marked_ids(())
+        if unicode(self.gui.search.text()).startswith('marked:'):
+            self.gui.search.set_search_string('')
+
+    def mark_field(self, field, add):
+        book_ids = self._get_selected_ids()
+        if not book_ids:
+            return
+        db = self.gui.current_db
+        items = set()
+        for book_id in book_ids:
+            items |= set(db.new_api.field_ids_for(field, book_id))
+        book_ids = set()
+        for item_id in items:
+            book_ids |= db.new_api.books_for_field(field, item_id)
+        mids = db.data.marked_ids.copy()
+        for book_id in book_ids:
+            if add:
+                mids[book_id] = True
+            else:
+                mids.pop(book_id, None)
+        db.data.set_marked_ids(mids)
+

src/calibre/gui2/actions/tweak_epub.py

@@ -22,6 +22,7 @@ class TweakBook(QDialog):
 
     def __init__(self, parent, book_id, fmts, db):
         QDialog.__init__(self, parent)
+        self.setWindowIcon(QIcon(I('tweak.png')))
         self.book_id, self.fmts, self.db_ref = book_id, fmts, weakref.ref(db)
         self._exploded = None
         self._cleanup_dirs = []
@@ -54,7 +55,7 @@ class TweakBook(QDialog):
         self.rebuild_button.setEnabled(False)
         self.explode_button.setEnabled(True)
 
-    def setup_ui(self): # {{{
+    def setup_ui(self):  # {{{
         self._g = g = QHBoxLayout(self)
         self.setLayout(g)
         self._l = l = QVBoxLayout()
@@ -285,7 +286,7 @@ class TweakBook(QDialog):
 class TweakEpubAction(InterfaceAction):
 
     name = 'Tweak ePub'
-    action_spec = (_('Tweak Book'), 'trim.png',
+    action_spec = (_('Tweak Book'), 'tweak.png',
             _('Make small changes to ePub, HTMLZ or AZW3 format books'),
             _('T'))
     dont_add_to = frozenset(['context-menu-device'])

src/calibre/gui2/book_details.py

@@ -5,6 +5,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import os
+from functools import partial
+
 from PyQt4.Qt import (QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, QIcon,
     QPropertyAnimation, QEasingCurve, QApplication, QFontInfo, QAction,
     QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette, QMenu,
@@ -164,8 +167,16 @@ def render_data(mi, use_roman_numbers=True, all_fields=False):
         elif field == 'formats':
             if isdevice:
                 continue
-            fmts = [u'<a href="format:%s:%s">%s</a>' % (mi.id, x, x) for x
+            p = partial(prepare_string_for_xml, attribute=True)
-                        in mi.formats]
+            path = ''
+            if mi.path:
+                h, t = os.path.split(mi.path)
+                path = '/'.join((os.path.basename(h), t))
+            data = ({
+                'fmt':x, 'path':p(path or ''), 'fname':p(mi.format_files.get(x, '')),
+                'ext':x.lower(), 'id':mi.id
+            } for x in mi.formats)
+            fmts = [u'<a title="{path}/{fname}.{ext}" href="format:{id}:{fmt}">{fmt}</a>'.format(**x) for x in data]
             ans.append((field, row % (name, u', '.join(fmts))))
         elif field == 'identifiers':
             urls = urls_from_identifiers(mi.identifiers)

src/calibre/gui2/convert/look_and_feel.py

@@ -38,7 +38,7 @@ class LookAndFeelWidget(Widget, Ui_Form):
                     'remove_paragraph_spacing',
                     'remove_paragraph_spacing_indent_size',
                     'insert_blank_line_size',
-                    'input_encoding', 'filter_css',
+                    'input_encoding', 'filter_css', 'expand_css',
                     'asciiize', 'keep_ligatures',
                     'linearize_tables']
                 )

src/calibre/gui2/convert/look_and_feel.ui

@@ -21,13 +21,6 @@
      </property>
     </widget>
    </item>
-   <item row="12" column="3">
-    <widget class="QCheckBox" name="opt_linearize_tables">
-     <property name="text">
-      <string>&amp;Linearize tables</string>
-     </property>
-    </widget>
-   </item>
    <item row="1" column="0">
     <widget class="QLabel" name="label_18">
      <property name="text">
@@ -125,13 +118,6 @@
      </property>
     </widget>
    </item>
-   <item row="11" column="1" colspan="4">
-    <widget class="QCheckBox" name="opt_asciiize">
-     <property name="text">
-      <string>&amp;Transliterate unicode characters to ASCII</string>
-     </property>
-    </widget>
-   </item>
    <item row="12" column="0">
     <widget class="QCheckBox" name="opt_unsmarten_punctuation">
      <property name="text">
@@ -422,6 +408,27 @@
      </property>
     </widget>
    </item>
+   <item row="11" column="1" colspan="2">
+    <widget class="QCheckBox" name="opt_asciiize">
+     <property name="text">
+      <string>&amp;Transliterate unicode characters to ASCII</string>
+     </property>
+    </widget>
+   </item>
+   <item row="11" column="3" colspan="2">
+    <widget class="QCheckBox" name="opt_expand_css">
+     <property name="text">
+      <string>E&amp;xpand CSS</string>
+     </property>
+    </widget>
+   </item>
+   <item row="12" column="3" colspan="2">
+    <widget class="QCheckBox" name="opt_linearize_tables">
+     <property name="text">
+      <string>&amp;Linearize tables</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>

src/calibre/gui2/keyboard.py

@@ -28,7 +28,7 @@ ROOT = QModelIndex()
 class NameConflict(ValueError):
     pass
 
-def finalize(shortcuts, custom_keys_map={}): # {{{
+def finalize(shortcuts, custom_keys_map={}):  # {{{
     '''
     Resolve conflicts and assign keys to every action in shorcuts, which must
     be a OrderedDict. User specified mappings of unique names to keys (as a
@@ -69,12 +69,12 @@ def finalize(shortcuts, custom_keys_map={}): # {{{
 
 # }}}
 
-class Manager(QObject): # {{{
+class Manager(QObject):  # {{{
 
-    def __init__(self, parent=None):
+    def __init__(self, parent=None, config_name='shortcuts/main'):
         QObject.__init__(self, parent)
 
-        self.config = JSONConfig('shortcuts/main')
+        self.config = JSONConfig(config_name)
         self.shortcuts = OrderedDict()
         self.keys_map = {}
         self.groups = {}
@@ -127,7 +127,7 @@ class Manager(QObject): # {{{
             'map', {}).iteritems()}
         self.keys_map = finalize(self.shortcuts, custom_keys_map=custom_keys_map)
         #import pprint
-        #pprint.pprint(self.keys_map)
+        # pprint.pprint(self.keys_map)
 
     def replace_action(self, unique_name, new_action):
         '''
@@ -255,7 +255,8 @@ class ConfigModel(QAbstractItemModel, SearchQueryParser):
         kmap = {}
         for node in self.all_shortcuts:
             sc = node.data
-            if sc['set_to_default']: continue
+            if sc['set_to_default']:
+                continue
             keys = [unicode(k.toString(k.PortableText)) for k in sc['keys']]
             kmap[sc['unique_name']] = keys
         self.keyboard.config['map'] = kmap
@@ -343,7 +344,7 @@ class ConfigModel(QAbstractItemModel, SearchQueryParser):
 
 # }}}
 
-class Editor(QFrame): # {{{
+class Editor(QFrame):  # {{{
 
     editing_done = pyqtSignal(object)
 
@@ -403,10 +404,12 @@ class Editor(QFrame): # {{{
         self.current_keys = list(shortcut['keys'])
         default = ', '.join([unicode(k.toString(k.NativeText)) for k in
                     self.default_keys])
-        if not default: default = _('None')
+        if not default:
+            default = _('None')
         current = ', '.join([unicode(k.toString(k.NativeText)) for k in
                     self.current_keys])
-        if not current: current = _('None')
+        if not current:
+            current = _('None')
 
         self.use_default.setText(_('Default: %(deflt)s [Currently not conflicting: %(curr)s]')%
                 dict(deflt=default, curr=current))
@@ -461,7 +464,8 @@ class Editor(QFrame): # {{{
 
     def dup_check(self, sequence):
         for sc in self.all_shortcuts:
-            if sc is self.shortcut: continue
+            if sc is self.shortcut:
+                continue
             for k in sc['keys']:
                 if k == sequence:
                     return sc['name']
@@ -484,7 +488,7 @@ class Editor(QFrame): # {{{
 
 # }}}
 
-class Delegate(QStyledItemDelegate): # {{{
+class Delegate(QStyledItemDelegate):  # {{{
 
     changed_signal = pyqtSignal()
 
@@ -555,7 +559,8 @@ class Delegate(QStyledItemDelegate): # {{{
                 ckey = QKeySequence(ckey, QKeySequence.PortableText)
                 matched = False
                 for s in editor.all_shortcuts:
-                    if s is editor.shortcut: continue
+                    if s is editor.shortcut:
+                        continue
                     for k in s['keys']:
                         if k == ckey:
                             matched = True
@@ -581,7 +586,7 @@ class Delegate(QStyledItemDelegate): # {{{
 
 # }}}
 
-class ShortcutConfig(QWidget): # {{{
+class ShortcutConfig(QWidget):  # {{{
 
     changed_signal = pyqtSignal()
 

src/calibre/gui2/library/alternate_views.py

@@ -300,6 +300,10 @@ class AlternateViews(object):
             if self.current_book_state[0] is self.current_view:
                 self.current_view.restore_current_book_state(self.current_book_state[1])
             self.current_book_state = None
+
+    def marked_changed(self, old_marked, current_marked):
+        if self.current_view is not self.main_view:
+            self.current_view.marked_changed(old_marked, current_marked)
 # }}}
 
 # Rendering of covers {{{
@@ -422,7 +426,7 @@ class CoverDelegate(QStyledItemDelegate):
                 try:
                     p = self.marked_emblem
                 except AttributeError:
-                    p = self.marked_emblem = QPixmap(I('rating.png')).scaled(48, 48, transformMode=Qt.SmoothTransformation)
+                    p = self.marked_emblem = m.marked_icon.pixmap(48, 48)
                 drect = QRect(orect)
                 drect.setLeft(drect.left() + right_adjust)
                 drect.setRight(drect.left() + p.width())
@@ -731,6 +735,16 @@ class GridView(QListView):
             sel.merge(QItemSelection(m.index(min(group), 0), m.index(max(group), 0)), sm.Select)
         sm.select(sel, sm.ClearAndSelect)
 
+    def selectAll(self):
+        # We re-implement this to ensure that only indexes from column 0 are
+        # selected. The base class implementation selects all columns. This
+        # causes problems with selection syncing, see
+        # https://bugs.launchpad.net/bugs/1236348
+        m = self.model()
+        sm = self.selectionModel()
+        sel = QItemSelection(m.index(0, 0), m.index(m.rowCount(QModelIndex())-1, 0))
+        sm.select(sel, sm.ClearAndSelect)
+
     def set_current_row(self, row):
         sm = self.selectionModel()
         sm.setCurrentIndex(self.model().index(row, 0), sm.NoUpdate)
@@ -808,4 +822,14 @@ class GridView(QListView):
         self.set_current_row(row)
         self.select_rows((row,))
         self.scrollTo(self.model().index(row, 0), self.PositionAtCenter)
+
+    def marked_changed(self, old_marked, current_marked):
+        changed = old_marked | current_marked
+        m = self.model()
+        for book_id in changed:
+            try:
+                self.update(m.index(m.db.data.id_to_index(book_id), 0))
+            except ValueError:
+                pass
+
 # }}}

src/calibre/gui2/library/models.py

@@ -111,7 +111,7 @@ class ColumnIcon(object):  # {{{
                     d = os.path.join(config_dir, 'cc_icons', icon)
                     if (os.path.exists(d)):
                         bm = QPixmap(d)
-                        bm = bm.scaled(128, 128, aspectRatioMode= Qt.KeepAspectRatio,
+                        bm = bm.scaled(128, 128, aspectRatioMode=Qt.KeepAspectRatio,
                                        transformMode=Qt.SmoothTransformation)
                         icon_bitmaps.append(bm)
                         total_width += bm.width()
@@ -193,6 +193,8 @@ class BooksModel(QAbstractTableModel):  # {{{
         self.bool_yes_icon = QIcon(I('ok.png'))
         self.bool_no_icon = QIcon(I('list_remove.png'))
         self.bool_blank_icon = QIcon(I('blank.png'))
+        self.marked_icon = QIcon(I('marked.png'))
+        self.row_decoration = NONE
         self.device_connected = False
         self.ids_to_highlight = []
         self.ids_to_highlight_set = set()
@@ -210,6 +212,9 @@ class BooksModel(QAbstractTableModel):  # {{{
     def set_row_height(self, height):
         self.row_height = height
 
+    def set_row_decoration(self, current_marked):
+        self.row_decoration = self.bool_blank_icon if current_marked else None
+
     def change_alignment(self, colname, alignment):
         if colname in self.column_map and alignment in ('left', 'right', 'center'):
             old = self.alignment_map.get(colname, 'left')
@@ -488,6 +493,7 @@ class BooksModel(QAbstractTableModel):  # {{{
         mi.id = self.db.id(idx)
         mi.field_metadata = self.db.field_metadata
         mi.path = self.db.abspath(idx, create_dirs=False)
+        mi.format_files = self.db.new_api.format_files(self.db.data.index_to_id(idx))
         try:
             mi.marked = self.db.data.get_marked(idx, index_is_id=False)
         except:
@@ -921,6 +927,8 @@ class BooksModel(QAbstractTableModel):  # {{{
 
         if role == Qt.DisplayRole:  # orientation is vertical
             return QVariant(section+1)
+        if role == Qt.DecorationRole:
+            return self.marked_icon if self.db.data.get_marked(self.db.data.index_to_id(section)) else self.row_decoration
         return NONE
 
     def flags(self, index):

src/calibre/gui2/library/views.py

@@ -22,7 +22,7 @@ from calibre.gui2.library.delegates import (RatingDelegate, PubDateDelegate,
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
 from calibre.gui2.library.alternate_views import AlternateViews, setup_dnd_interface
 from calibre.utils.config import tweaks, prefs
-from calibre.gui2 import error_dialog, gprefs
+from calibre.gui2 import error_dialog, gprefs, FunctionDispatcher
 from calibre.gui2.library import DEFAULT_SORT
 from calibre.constants import filesystem_encoding
 from calibre import force_unicode
@@ -63,6 +63,11 @@ class HeaderView(QHeaderView):  # {{{
                     opt.state |= QStyle.State_MouseOver
         sm = self.selectionModel()
         if opt.orientation == Qt.Vertical:
+            try:
+                opt.icon = model.headerData(logical_index, opt.orientation, Qt.DecorationRole)
+                opt.iconAlignment = Qt.AlignVCenter
+            except TypeError:
+                pass
             if sm.isRowSelected(logical_index, QModelIndex()):
                 opt.state |= QStyle.State_Sunken
 
@@ -214,6 +219,7 @@ class BooksView(QTableView):  # {{{
         self.setSortingEnabled(True)
         self.selectionModel().currentRowChanged.connect(self._model.current_changed)
         self.preserve_state = partial(PreserveViewState, self)
+        self.marked_changed_listener = FunctionDispatcher(self.marked_changed)
 
         # {{{ Column Header setup
         self.can_add_columns = True
@@ -229,6 +235,7 @@ class BooksView(QTableView):  # {{{
         self.column_header.customContextMenuRequested.connect(self.show_column_header_context_menu)
         self.column_header.sectionResized.connect(self.column_resized, Qt.QueuedConnection)
         self.row_header = HeaderView(Qt.Vertical, self)
+        self.row_header.setResizeMode(self.row_header.Fixed)
         self.setVerticalHeader(self.row_header)
         # }}}
 
@@ -682,7 +689,20 @@ class BooksView(QTableView):  # {{{
         self.publisher_delegate.set_auto_complete_function(db.all_publishers)
         self.alternate_views.set_database(db, stage=1)
 
+    def marked_changed(self, old_marked, current_marked):
+        self.alternate_views.marked_changed(old_marked, current_marked)
+        if bool(old_marked) == bool(current_marked):
+            changed = old_marked | current_marked
+            sections = tuple(map(self.model().db.data.id_to_index, changed))
+            self.row_header.headerDataChanged(Qt.Vertical, min(sections), max(sections))
+        else:
+            # Marked items have either appeared or all been removed
+            self.model().set_row_decoration(current_marked)
+            self.row_header.headerDataChanged(Qt.Vertical, 0, self.row_header.count()-1)
+            self.row_header.geometriesChanged.emit()
+
     def database_changed(self, db):
+        db.data.add_marked_listener(self.marked_changed_listener)
         for i in range(self.model().columnCount(None)):
             if self.itemDelegateForColumn(i) in (self.rating_delegate,
                     self.timestamp_delegate, self.pubdate_delegate,

src/calibre/gui2/metadata/basic_widgets.py

@@ -892,8 +892,7 @@ class Cover(ImageView):  # {{{
     download_cover = pyqtSignal()
 
     def __init__(self, parent):
-        ImageView.__init__(self, parent)
+        ImageView.__init__(self, parent, show_size_pref_name='edit_metadata_cover_widget', default_show_size=True)
-        self.show_size = True
         self.dialog = parent
         self._cdata = None
         self.cdata_before_trim = None

src/calibre/gui2/preferences/adding.py

@@ -35,6 +35,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                 (_('Create new record for each duplicate format'), 'new record')]
         r('automerge', gprefs, choices=choices)
         r('new_book_tags', prefs, setting=CommaSeparatedList)
+        r('mark_new_books', prefs)
         r('auto_add_path', gprefs, restart_required=True)
         r('auto_add_check_for_duplicates', gprefs)
         r('auto_add_auto_convert', gprefs)

src/calibre/gui2/preferences/adding.ui

@@ -151,6 +151,13 @@ Author matching is exact.</string>
          </property>
         </widget>
        </item>
+       <item row="3" column="2">
+        <widget class="QCheckBox" name="opt_mark_new_books">
+         <property name="text">
+          <string>&amp;Mark newly added books</string>
+         </property>
+        </widget>
+       </item>
       </layout>
      </widget>
      <widget class="QWidget" name="tab_4">

src/calibre/gui2/store/stores/amazon_ca_plugin.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+store_version = 1  # Needed for dynamic plugin loading
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.search_result import SearchResult
+
+class AmazonCAKindleStore(StorePlugin):
+    '''
+    For comments on the implementation, please see amazon_plugin.py
+    '''
+
+    search_url = 'http://www.amazon.ca/s/url=search-alias%3Ddigital-text&field-keywords='
+    details_url = 'http://amazon.ca/dp/'
+    drm_search_text = u'Simultaneous Device Usage'
+    drm_free_text = u'Unlimited'
+
+    def open(self, parent=None, detail_item=None, external=False):
+        #aff_id = {'tag': ''}
+        # Use Kovid's affiliate id 30% of the time.
+        # if random.randint(1, 10) in (1, 2, 3):
+        #    aff_id['tag'] = 'calibrebs-20'
+        # store_link = 'http://www.amazon.ca/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id
+        store_link = 'http://www.amazon.ca/ebooks-kindle/b/ref=sa_menu_kbo?ie=UTF8&node=2980423011'
+        if detail_item:
+            # aff_id['asin'] = detail_item
+            # store_link = 'http://www.amazon.ca/dp/%(asin)s/?tag=%(tag)s' % aff_id
+            store_link = 'http://www.amazon.ca/dp/' + detail_item + '/'
+        open_url(QUrl(store_link))
+
+    def search(self, query, max_results=10, timeout=60):
+        url = self.search_url + query.encode('ascii', 'backslashreplace').replace('%', '%25').replace('\\x', '%').replace(' ', '+')
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+
+            if doc.xpath('//div[@id = "atfResults" and contains(@class, "grid")]'):
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "ilresults")]'):
+                data_xpath = '//li[(@class="ilo")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './div[@class = "ilf"]/a/img[contains(@class, "ilo")]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                # Results can be in a grid (table) or a column
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL") or contains(@class, "rsltGridList")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            elif doc.xpath('//div[@id = "atfResults" and contains(@class, "list")]'):
+                data_xpath = '//div[contains(@class, "prod")]'
+                format_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and not(contains(@class, "bld"))]/text()')
+                asin_xpath = '@name'
+                cover_xpath = './/img[@class="productImage"]/@src'
+                title_xpath = './/h3[@class="newaps"]/a//text()'
+                author_xpath = './/h3[@class="newaps"]//span[contains(@class, "reg")]//text()'
+                price_xpath = (
+                        './/ul[contains(@class, "rsltL")]'
+                        '//span[contains(@class, "lrg") and contains(@class, "bld")]/text()')
+            else:
+                return
+
+            for data in doc.xpath(data_xpath):
+                if counter <= 0:
+                    break
+
+                # Even though we are searching digital-text only Amazon will still
+                # put in results for non Kindle books (author pages). Se we need
+                # to explicitly check if the item is a Kindle book and ignore it
+                # if it isn't.
+                format = ''.join(data.xpath(format_xpath))
+                if 'kindle' not in format.lower():
+                    continue
+
+                # We must have an asin otherwise we can't easily reference the
+                # book later.
+                asin = data.xpath(asin_xpath)
+                if asin:
+                    asin = asin[0]
+                else:
+                    continue
+
+                cover_url = ''.join(data.xpath(cover_xpath))
+
+                title = ''.join(data.xpath(title_xpath))
+                author = ''.join(data.xpath(author_xpath))
+                try:
+                    author = author.split('by ', 1)[1].split(" (")[0]
+                except:
+                    pass
+
+                price = ''.join(data.xpath(price_xpath))
+
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url.strip()
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = asin.strip()
+                s.formats = 'Kindle'
+
+                yield s
+
+    def get_details(self, search_result, timeout):
+        url = self.details_url
+
+        br = browser()
+        with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' +
+                           self.drm_search_text + '")])'):
+                if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' +
+                               self.drm_free_text + '") and contains(b, "' +
+                               self.drm_search_text + '")])'):
+                    search_result.drm = SearchResult.DRM_UNLOCKED
+                else:
+                    search_result.drm = SearchResult.DRM_UNKNOWN
+            else:
+                search_result.drm = SearchResult.DRM_LOCKED
+        return True

src/calibre/gui2/store/stores/ebookpoint_plugin.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import (unicode_literals, division, absolute_import, print_function)
-store_version = 3 # Needed for dynamic plugin loading
+store_version = 4 # Needed for dynamic plugin loading
 
 __license__ = 'GPL 3'
 __copyright__ = '2011-2013, Tomasz Długosz <tomek3d@gmail.com>'
@@ -60,13 +60,15 @@ class EbookpointStore(BasicStoreConfig, StorePlugin):
                 if not id:
                     continue
 
+                formats = ', '.join(data.xpath('.//div[@class="ikony"]/span/text()'))
+                if formats == 'MP3':
+                    continue
                 cover_url = ''.join(data.xpath('.//a[@class="cover"]/img/@src'))
                 title = ''.join(data.xpath('.//h3/a/@title'))
                 title = re.sub('eBook.', '', title)
                 author = ''.join(data.xpath('.//p[@class="author"]//text()'))
                 price = ''.join(data.xpath('.//p[@class="price"]/ins/text()'))
 
-                formats = ', '.join(data.xpath('.//div[@class="ikony"]/span/text()'))
 
                 counter -= 1
 

src/calibre/gui2/store/stores/wolnelektury_plugin.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import (unicode_literals, division, absolute_import, print_function)
-store_version = 1 # Needed for dynamic plugin loading
+store_version = 1  # Needed for dynamic plugin loading
 
 __license__ = 'GPL 3'
-__copyright__ = '2011, Tomasz Długosz <tomek3d@gmail.com>'
+__copyright__ = '2012-2013, Tomasz Długosz <tomek3d@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import urllib
@@ -21,11 +21,11 @@ from calibre.gui2.store.basic_config import BasicStoreConfig
 from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
-class BookotekaStore(BasicStoreConfig, StorePlugin):
+class WolneLekturyStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
 
-        url = 'http://bookoteka.pl/ebooki'
+        url = 'http://wolnelektury.pl'
         detail_url = None
 
         if detail_item:
@@ -40,37 +40,39 @@ class BookotekaStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://bookoteka.pl/list?search=' + urllib.quote_plus(query) + '&cat=1&hp=1&type=1'
+        url = 'http://wolnelektury.pl/szukaj?q=' + urllib.quote_plus(query.encode('utf-8'))
 
         br = browser()
 
         counter = max_results
         with closing(br.open(url, timeout=timeout)) as f:
             doc = html.fromstring(f.read())
-            for data in doc.xpath('//li[@class="EBOOK"]'):
+            for data in doc.xpath('//li[@class="Book-item"]'):
                 if counter <= 0:
                     break
 
-                id = ''.join(data.xpath('.//a[@class="item_link"]/@href'))
+                id = ''.join(data.xpath('.//div[@class="title"]/a/@href'))
                 if not id:
                     continue
 
-                cover_url = ''.join(data.xpath('.//a[@class="item_link"]/img/@src'))
+                cover_url = ''.join(data.xpath('.//a[1]/img/@src'))
-                title = ''.join(data.xpath('.//div[@class="shelf_title"]/a/text()'))
+                title = ''.join(data.xpath('.//div[@class="title"]/a[1]/text()'))
-                author = ''.join(data.xpath('.//div[@class="shelf_authors"][1]/text()'))
+                author = ', '.join(data.xpath('.//div[@class="mono author"]/a/text()'))
-                price = ''.join(data.xpath('.//span[@class="EBOOK"]/text()'))
+                price = '0,00 zł'
-                price = price.replace('.', ',')
-                formats = ', '.join(data.xpath('.//a[@class="fancybox protected"]/text()'))
 
                 counter -= 1
 
                 s = SearchResult()
-                s.cover_url = 'http://bookoteka.pl' + cover_url
+                for link in data.xpath('.//div[@class="book-box-formats mono"]/span/a'):
+                    ext = ''.join(link.xpath('./text()'))
+                    href = 'http://wolnelektury.pl' + link.get('href')
+                    s.downloads[ext] = href
+                s.cover_url = 'http://wolnelektury.pl' + cover_url.strip()
                 s.title = title.strip()
-                s.author = author.strip()
+                s.author = author
                 s.price = price
-                s.detail_item = 'http://bookoteka.pl' + id.strip()
+                s.detail_item = 'http://wolnelektury.pl' + id
+                s.formats = ', '.join(s.downloads.keys())
                 s.drm = SearchResult.DRM_UNLOCKED
-                s.formats = formats.strip()
 
                 yield s

src/calibre/gui2/tweak_book/__init__.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+
+_current_container = None
+
+def current_container():
+    return _current_container
+
+def set_current_container(container):
+    global _current_container
+    _current_container = container

src/calibre/gui2/tweak_book/boss.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import tempfile, shutil
+
+from PyQt4.Qt import QObject
+
+from calibre.gui2 import error_dialog, choose_files
+from calibre.ptempfile import PersistentTemporaryDirectory
+from calibre.ebooks.oeb.polish.main import SUPPORTED
+from calibre.ebooks.oeb.polish.container import get_container, clone_container
+from calibre.gui2.tweak_book import set_current_container, current_container
+from calibre.gui2.tweak_book.undo import GlobalUndoHistory
+
+class Boss(QObject):
+
+    def __init__(self, parent=None):
+        QObject.__init__(self, parent)
+        self.global_undo = GlobalUndoHistory()
+        self.container_count = 0
+        self.tdir = None
+
+    def __call__(self, gui):
+        self.gui = gui
+        gui.file_list.delete_requested.connect(self.delete_requested)
+
+    def mkdtemp(self):
+        self.container_count += 1
+        return tempfile.mkdtemp(prefix='%05d-' % self.container_count, dir=self.tdir)
+
+    def check_dirtied(self):
+        # TODO: Implement this
+        return True
+
+    def open_book(self, path=None):
+        if not self.check_dirtied():
+            return
+
+        if not hasattr(path, 'rpartition'):
+            path = choose_files(self.gui, 'open-book-for-tweaking', _('Choose book'),
+                                [(_('Books'), [x.lower() for x in SUPPORTED])], all_files=False, select_only_single_file=True)
+            if not path:
+                return
+            path = path[0]
+
+        ext = path.rpartition('.')[-1].upper()
+        if ext not in SUPPORTED:
+            return error_dialog(self.gui, _('Unsupported format'),
+                _('Tweaking is only supported for books in the %s formats.'
+                  ' Convert your book to one of these formats first.') % _(' and ').join(sorted(SUPPORTED)),
+                show=True)
+
+        self.container_count = -1
+        if self.tdir:
+            shutil.rmtree(self.tdir, ignore_errors=True)
+        self.tdir = PersistentTemporaryDirectory()
+        self.gui.blocking_job('open_book', _('Opening book, please wait...'), self.book_opened, get_container, path, tdir=self.mkdtemp())
+
+    def book_opened(self, job):
+        if job.traceback is not None:
+            return error_dialog(self.gui, _('Failed to open book'),
+                    _('Failed to open book, click Show details for more information.'),
+                                det_msg=job.traceback, show=True)
+        container = job.result
+        set_current_container(container)
+        self.current_metadata = self.gui.current_metadata = container.mi
+        self.global_undo.open_book(container)
+        self.gui.update_window_title()
+        self.gui.file_list.build(container)
+
+    def add_savepoint(self, msg):
+        nc = clone_container(current_container(), self.mkdtemp())
+        self.global_undo.add_savepoint(nc, msg)
+        set_current_container(nc)
+
+    def delete_requested(self, spine_items, other_items):
+        if not self.check_dirtied():
+            return
+        self.add_savepoint(_('Delete files'))
+        c = current_container()
+        c.remove_from_spine(spine_items)
+        for name in other_items:
+            c.remove_item(name)
+        self.gui.file_list.delete_done(spine_items, other_items)
+
+

src/calibre/gui2/tweak_book/file_list.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from PyQt4.Qt import (
+    QWidget, QTreeWidget, QGridLayout, QSize, Qt, QTreeWidgetItem, QIcon,
+    QStyledItemDelegate, QStyle, QPixmap, QPainter, pyqtSignal)
+
+from calibre import human_readable
+from calibre.ebooks.oeb.base import OEB_STYLES, OEB_DOCS
+from calibre.ebooks.oeb.polish.container import guess_type
+from calibre.ebooks.oeb.polish.cover import get_cover_page_name, get_raster_cover_name
+from calibre.gui2 import error_dialog
+from calibre.gui2.tweak_book import current_container
+
+TOP_ICON_SIZE = 24
+NAME_ROLE = Qt.UserRole
+CATEGORY_ROLE = NAME_ROLE + 1
+NBSP = '\xa0'
+
+class ItemDelegate(QStyledItemDelegate):  # {{{
+
+    def sizeHint(self, option, index):
+        ans = QStyledItemDelegate.sizeHint(self, option, index)
+        top_level = not index.parent().isValid()
+        ans += QSize(0, 20 if top_level else 10)
+        return ans
+
+    def paint(self, painter, option, index):
+        top_level = not index.parent().isValid()
+        hover = option.state & QStyle.State_MouseOver
+        if hover:
+            if top_level:
+                suffix = '%s(%d)' % (NBSP, index.model().rowCount(index))
+            else:
+                suffix = NBSP + human_readable(current_container().filesize(unicode(index.data(NAME_ROLE).toString())))
+            br = painter.boundingRect(option.rect, Qt.AlignRight|Qt.AlignVCenter, suffix)
+        if top_level and index.row() > 0:
+            option.rect.adjust(0, 5, 0, 0)
+            painter.drawLine(option.rect.topLeft(), option.rect.topRight())
+            option.rect.adjust(0, 1, 0, 0)
+        if hover:
+            option.rect.adjust(0, 0, -br.width(), 0)
+        QStyledItemDelegate.paint(self, painter, option, index)
+        if hover:
+            option.rect.adjust(0, 0, br.width(), 0)
+            painter.drawText(option.rect, Qt.AlignRight|Qt.AlignVCenter, suffix)
+# }}}
+
+class FileList(QTreeWidget):
+
+    delete_requested = pyqtSignal(object, object)
+
+    def __init__(self, parent=None):
+        QTreeWidget.__init__(self, parent)
+        self.delegate = ItemDelegate(self)
+        self.setTextElideMode(Qt.ElideMiddle)
+        self.setItemDelegate(self.delegate)
+        self.setIconSize(QSize(16, 16))
+        self.header().close()
+        self.setDragEnabled(True)
+        self.setSelectionMode(self.ExtendedSelection)
+        self.viewport().setAcceptDrops(True)
+        self.setDropIndicatorShown(True)
+        self.setDragDropMode(self.InternalMove)
+        self.setAutoScroll(True)
+        self.setAutoScrollMargin(TOP_ICON_SIZE*2)
+        self.setDefaultDropAction(Qt.MoveAction)
+        self.setAutoExpandDelay(1000)
+        self.setAnimated(True)
+        self.setMouseTracking(True)
+        self.in_drop_event = False
+        self.setContextMenuPolicy(Qt.CustomContextMenu)
+        self.customContextMenuRequested.connect(self.show_context_menu)
+        self.root = self.invisibleRootItem()
+        self.emblem_cache = {}
+        self.rendered_emblem_cache = {}
+        self.top_level_pixmap_cache = {
+            name : QPixmap(I(icon)).scaled(TOP_ICON_SIZE, TOP_ICON_SIZE, transformMode=Qt.SmoothTransformation)
+            for name, icon in {
+                'text':'keyboard-prefs.png',
+                'styles':'lookfeel.png',
+                'fonts':'font.png',
+                'misc':'mimetypes/dir.png',
+                'images':'view-image.png',
+            }.iteritems()}
+
+    def build(self, container):
+        self.clear()
+        self.root = self.invisibleRootItem()
+        self.root.setFlags(Qt.ItemIsDragEnabled)
+        self.categories = {}
+        for category, text in (
+            ('text', _('Text')),
+            ('styles', _('Styles')),
+            ('images', _('Images')),
+            ('fonts', _('Fonts')),
+            ('misc', _('Miscellaneous')),
+        ):
+            self.categories[category] = i = QTreeWidgetItem(self.root, 0)
+            i.setText(0, text)
+            i.setData(0, Qt.DecorationRole, self.top_level_pixmap_cache[category])
+            f = i.font(0)
+            f.setBold(True)
+            i.setFont(0, f)
+            i.setData(0, NAME_ROLE, category)
+            flags = Qt.ItemIsEnabled
+            if category == 'text':
+                flags |= Qt.ItemIsDropEnabled
+            i.setFlags(flags)
+
+        processed, seen = {}, {}
+
+        cover_page_name = get_cover_page_name(container)
+        cover_image_name = get_raster_cover_name(container)
+        manifested_names = set()
+        for names in container.manifest_type_map.itervalues():
+            manifested_names |= set(names)
+
+        font_types = {guess_type('a.'+x) for x in ('ttf', 'otf', 'woff')}
+
+        def get_category(mt):
+            category = 'misc'
+            if mt.startswith('image/'):
+                category = 'images'
+            elif mt in font_types:
+                category = 'fonts'
+            elif mt in OEB_STYLES:
+                category = 'styles'
+            elif mt in OEB_DOCS:
+                category = 'text'
+            return category
+
+        def set_display_name(name, item):
+            if name in processed:
+                # We have an exact duplicate (can happen if there are
+                # duplicates in the spine)
+                item.setText(0, processed[name].text(0))
+                return
+
+            parts = name.split('/')
+            text = parts[-1]
+            while text in seen and parts:
+                text = parts.pop() + '/' + text
+            seen[text] = item
+            item.setText(0, text)
+
+        def render_emblems(item, emblems):
+            emblems = tuple(emblems)
+            if not emblems:
+                return
+            icon = self.rendered_emblem_cache.get(emblems, None)
+            if icon is None:
+                pixmaps = []
+                for emblem in emblems:
+                    pm = self.emblem_cache.get(emblem, None)
+                    if pm is None:
+                        pm = self.emblem_cache[emblem] = QPixmap(
+                            I(emblem)).scaled(self.iconSize(), transformMode=Qt.SmoothTransformation)
+                    pixmaps.append(pm)
+                num = len(pixmaps)
+                w, h = pixmaps[0].width(), pixmaps[0].height()
+                if num == 1:
+                    icon = self.rendered_emblem_cache[emblems] = QIcon(pixmaps[0])
+                else:
+                    canvas = QPixmap((num * w) + ((num-1)*2), h)
+                    canvas.fill(Qt.transparent)
+                    painter = QPainter(canvas)
+                    for i, pm in enumerate(pixmaps):
+                        painter.drawPixmap(i * (w + 2), 0, pm)
+                    painter.end()
+                    icon = self.rendered_emblem_cache[emblems] = canvas
+            item.setData(0, Qt.DecorationRole, icon)
+
+        ok_to_be_unmanifested = container.names_that_need_not_be_manifested
+
+        def create_item(name, linear=None):
+            imt = container.mime_map.get(name, guess_type(name))
+            icat = get_category(imt)
+            category = 'text' if linear is not None else ({'text':'misc'}.get(icat, icat))
+            item = QTreeWidgetItem(self.categories['text' if linear is not None else category], 1)
+            flags = Qt.ItemIsEnabled | Qt.ItemIsSelectable
+            if category == 'text':
+                flags |= Qt.ItemIsDragEnabled
+            item.setFlags(flags)
+            item.setStatusTip(0, _('Full path: ') + name)
+            item.setData(0, NAME_ROLE, name)
+            item.setData(0, CATEGORY_ROLE, category)
+            set_display_name(name, item)
+            # TODO: Add appropriate tooltips based on the emblems
+            emblems = []
+            if name in {cover_page_name, cover_image_name}:
+                emblems.append('default_cover.png')
+            if name not in manifested_names and name not in ok_to_be_unmanifested:
+                emblems.append('dialog_question.png')
+            if linear is False:
+                emblems.append('arrow-down.png')
+            if linear is None and icat == 'text':
+                # Text item outside spine
+                emblems.append('dialog_warning.png')
+            if category == 'text' and name in processed:
+                # Duplicate entry in spine
+                emblems.append('dialog_warning.png')
+
+            render_emblems(item, emblems)
+            return item
+
+        for name, linear in container.spine_names:
+            processed[name] = create_item(name, linear=linear)
+
+        all_files = list(container.manifest_type_map.iteritems())
+        all_files.append((guess_type('a.opf'), [container.opf_name]))
+
+        for name in container.name_path_map:
+            if name in processed:
+                continue
+            processed[name] = create_item(name)
+
+        for c in self.categories.itervalues():
+            self.expandItem(c)
+
+    def show_context_menu(self, point):
+        pass
+
+    def keyPressEvent(self, ev):
+        if ev.key() in (Qt.Key_Delete, Qt.Key_Backspace):
+            ev.accept()
+            self.request_delete()
+        else:
+            return QTreeWidget.keyPressEvent(self, ev)
+
+    def request_delete(self):
+        names = {unicode(item.data(0, NAME_ROLE).toString()) for item in self.selectedItems()}
+        bad = names & current_container().names_that_must_not_be_removed
+        if bad:
+            return error_dialog(self, _('Cannot delete'),
+                         _('The file(s) %s cannot be deleted.') % ('<b>%s</b>' % ', '.join(bad)), show=True)
+
+        text = self.categories['text']
+        children = (text.child(i) for i in xrange(text.childCount()))
+        spine_removals = [(unicode(item.data(0, NAME_ROLE).toString()), item.isSelected()) for item in children]
+        other_removals = {unicode(item.data(0, NAME_ROLE).toString()) for item in self.selectedItems()
+                          if unicode(item.data(0, CATEGORY_ROLE).toString()) != 'text'}
+        self.delete_requested.emit(spine_removals, other_removals)
+
+    def delete_done(self, spine_removals, other_removals):
+        removals = []
+        for i, (name, remove) in enumerate(spine_removals):
+            if remove:
+                removals.append(self.categories['text'].child(i))
+        for category, parent in self.categories.iteritems():
+            if category != 'text':
+                for i in xrange(parent.childCount()):
+                    child = parent.child(i)
+                    if unicode(child.data(0, NAME_ROLE).toString()) in other_removals:
+                        removals.append(child)
+
+        for c in removals:
+            c.parent().removeChild(c)
+
+class FileListWidget(QWidget):
+
+    delete_requested = pyqtSignal(object, object)
+
+    def __init__(self, parent=None):
+        QWidget.__init__(self, parent)
+        self.setLayout(QGridLayout(self))
+        self.file_list = FileList(self)
+        self.layout().addWidget(self.file_list)
+        self.layout().setContentsMargins(0, 0, 0, 0)
+        for x in ('delete_requested',):
+            getattr(self.file_list, x).connect(getattr(self, x))
+        for x in ('delete_done',):
+            setattr(self, x, getattr(self.file_list, x))
+
+    def build(self, container):
+        self.file_list.build(container)
+
+

src/calibre/gui2/tweak_book/job.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import time
+from threading import Thread
+from functools import partial
+
+from PyQt4.Qt import (QWidget, QVBoxLayout, QLabel, Qt, QPainter, QBrush, QColor)
+
+from calibre.gui2 import Dispatcher
+from calibre.gui2.progress_indicator import ProgressIndicator
+
+class LongJob(Thread):
+
+    daemon = True
+
+    def __init__(self, name, user_text, callback, function, *args, **kwargs):
+        Thread.__init__(self, name=name)
+        self.user_text = user_text
+        self.function = function
+        self.args, self.kwargs = args, kwargs
+        self.result = self.traceback = None
+        self.time_taken = None
+        self.callback = callback
+
+    def run(self):
+        st = time.time()
+        try:
+            self.result = self.function(*self.args, **self.kwargs)
+        except:
+            import traceback
+            self.traceback = traceback.format_exc()
+        self.time_taken = time.time() - st
+        try:
+            self.callback(self)
+        finally:
+            pass
+
+class BlockingJob(QWidget):
+
+    def __init__(self, parent):
+        QWidget.__init__(self, parent)
+        l = QVBoxLayout()
+        self.setLayout(l)
+        l.addStretch(10)
+        self.pi = ProgressIndicator(self, 128)
+        l.addWidget(self.pi, alignment=Qt.AlignHCenter)
+        self.msg = QLabel('')
+        l.addSpacing(10)
+        l.addWidget(self.msg, alignment=Qt.AlignHCenter)
+        l.addStretch(10)
+        self.setVisible(False)
+
+    def start(self):
+        self.setGeometry(0, 0, self.parent().width(), self.parent().height())
+        self.setVisible(True)
+        self.raise_()
+        self.pi.startAnimation()
+
+    def stop(self):
+        self.pi.stopAnimation()
+        self.setVisible(False)
+
+    def job_done(self, callback, job):
+        del job.callback
+        self.stop()
+        callback(job)
+
+    def paintEvent(self, ev):
+        p = QPainter(self)
+        p.fillRect(ev.region().boundingRect(), QBrush(QColor(200, 200, 200, 160), Qt.SolidPattern))
+        p.end()
+        QWidget.paintEvent(self, ev)
+
+    def __call__(self, name, user_text, callback, function, *args, **kwargs):
+        self.msg.setText('<h2>%s</h2>' % user_text)
+        job = LongJob(name, user_text, Dispatcher(partial(self.job_done, callback)), function, *args, **kwargs)
+        job.start()
+        self.start()

src/calibre/gui2/tweak_book/main.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import sys, os
+
+from PyQt4.Qt import QIcon
+
+from calibre.constants import islinux
+from calibre.gui2 import Application, ORG_NAME, APP_UID
+from calibre.ptempfile import reset_base_dir
+from calibre.utils.config import OptionParser
+from calibre.gui2.tweak_book.ui import Main
+
+def option_parser():
+    return OptionParser('''\
+%prog [opts] [path_to_ebook]
+
+Launch the calibre tweak book tool.
+''')
+
+def main(args=sys.argv):
+    # Ensure we can continue to function if GUI is closed
+    os.environ.pop('CALIBRE_WORKER_TEMP_DIR', None)
+    reset_base_dir()
+
+    parser = option_parser()
+    opts, args = parser.parse_args(args)
+    override = 'calibre-tweak-book' if islinux else None
+    app = Application(args, override_program_name=override)
+    app.load_builtin_fonts()
+    app.setWindowIcon(QIcon(I('tweak.png')))
+    Application.setOrganizationName(ORG_NAME)
+    Application.setApplicationName(APP_UID)
+    main = Main(opts)
+    sys.excepthook = main.unhandled_exception
+    main.show()
+    if len(args) > 1:
+        main.boss.open_book(args[1])
+    app.exec_()
+
+if __name__ == '__main__':
+    main()
+

src/calibre/gui2/tweak_book/ui.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from PyQt4.Qt import QDockWidget, Qt, QLabel, QIcon, QAction
+
+from calibre.gui2.main_window import MainWindow
+from calibre.gui2.tweak_book import current_container
+from calibre.gui2.tweak_book.file_list import FileListWidget
+from calibre.gui2.tweak_book.job import BlockingJob
+from calibre.gui2.tweak_book.boss import Boss
+from calibre.gui2.keyboard import Manager as KeyboardManager
+
+class Main(MainWindow):
+
+    APP_NAME = _('Tweak Book')
+
+    def __init__(self, opts):
+        MainWindow.__init__(self, opts, disable_automatic_gc=True)
+        self.boss = Boss(self)
+        self.setWindowTitle(self.APP_NAME)
+        self.setWindowIcon(QIcon(I('tweak.png')))
+        self.opts = opts
+        self.path_to_ebook = None
+        self.container = None
+        self.current_metadata = None
+        self.blocking_job = BlockingJob(self)
+        self.keyboard = KeyboardManager(parent=self, config_name='shortcuts/tweak')
+
+        self.create_actions()
+        self.create_menubar()
+        self.create_toolbar()
+        self.create_docks()
+
+        self.status_bar = self.statusBar()
+        self.l = QLabel('Placeholder')
+
+        self.setCentralWidget(self.l)
+        self.boss(self)
+
+        self.keyboard.finalize()
+
+    def create_actions(self):
+        group = _('Global Actions')
+
+        def reg(icon, text, target, sid, keys, description):
+            ac = QAction(QIcon(I(icon)), text, self)
+            ac.triggered.connect(target)
+            if isinstance(keys, type('')):
+                keys = (keys,)
+            self.keyboard.register_shortcut(
+                sid, unicode(ac.text()), default_keys=keys, description=description, action=ac, group=group)
+            self.addAction(ac)
+            return ac
+
+        self.action_open_book = reg('document_open.png', _('Open &book'), self.boss.open_book, 'open-book', 'Ctrl+O', _('Open a new book'))
+
+    def create_menubar(self):
+        b = self.menuBar()
+        f = b.addMenu(_('&File'))
+        f.addAction(self.action_open_book)
+
+    def create_toolbar(self):
+        self.global_bar = b = self.addToolBar(_('Global'))
+        b.addAction(self.action_open_book)
+
+    def create_docks(self):
+        self.file_list_dock = d = QDockWidget(_('&Files Browser'), self)
+        d.setAllowedAreas(Qt.LeftDockWidgetArea | Qt.RightDockWidgetArea)
+        self.file_list = FileListWidget(d)
+        d.setWidget(self.file_list)
+        self.addDockWidget(Qt.LeftDockWidgetArea, d)
+
+    def resizeEvent(self, ev):
+        self.blocking_job.resize(ev.size())
+        return super(Main, self).resizeEvent(ev)
+
+    def update_window_title(self):
+        self.setWindowTitle(self.current_metadata.title + ' [%s] - %s' %(current_container().book_type.upper(), self.APP_NAME))

src/calibre/gui2/tweak_book/undo.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import shutil
+
+def cleanup(containers):
+    for container in containers:
+        try:
+            shutil.rmtree(container.root, ignore_errors=True)
+        except:
+            pass
+
+class State(object):
+
+    def __init__(self, container):
+        self.container = container
+        self.message = None
+
+class GlobalUndoHistory(object):
+
+    def __init__(self):
+        self.states = []
+        self.pos = 0
+
+    @property
+    def current_container(self):
+        return self.states[self.pos].container
+
+    def open_book(self, container):
+        self.states = [State(container)]
+        self.pos = 0
+
+    def add_savepoint(self, new_container, message):
+        self.states[self.pos].message = message
+        extra = self.states[self.pos+1:]
+        cleanup(extra)
+        self.states = self.states[:self.pos+1]
+        self.states.append(State(new_container))
+        self.pos += 1
+
+    def undo(self):
+        if self.pos > 0:
+            self.pos -= 1
+            return self.current_container
+
+    def redo(self):
+        if self.pos < len(self.states) - 1:
+            self.pos += 1
+            return self.current_container
+
+

src/calibre/gui2/viewer/bookmarkmanager.py

@@ -100,7 +100,7 @@ class BookmarkManager(QDialog, Ui_BookmarkManager):
             bad = False
             try:
                 for bm in imported:
-                    if len(bm) != 2:
+                    if 'title' not in bm:
                         bad = True
                         break
             except:
@@ -109,9 +109,9 @@ class BookmarkManager(QDialog, Ui_BookmarkManager):
             if not bad:
                 bookmarks = self.get_bookmarks()
                 for bm in imported:
-                    if bm not in bookmarks and bm['title'] != 'calibre_current_page_bookmark':
+                    if bm not in bookmarks:
                         bookmarks.append(bm)
-                self.set_bookmarks(bookmarks)
+                self.set_bookmarks([bm for bm in bookmarks if bm['title'] != 'calibre_current_page_bookmark'])
 
 if __name__ == '__main__':
     from PyQt4.Qt import QApplication

src/calibre/gui2/viewer/documentview.py

@@ -498,12 +498,10 @@ class DocumentView(QWebView):  # {{{
                 d.OpenImageInNewWindow, d.OpenLink, d.Reload, d.InspectElement]))
 
         self.search_online_action = QAction(QIcon(I('search.png')), '', self)
-        self.search_online_action.setShortcut(Qt.CTRL+Qt.Key_E)
         self.search_online_action.triggered.connect(self.search_online)
         self.addAction(self.search_online_action)
         self.dictionary_action = QAction(QIcon(I('dictionary.png')),
                 _('&Lookup in dictionary'), self)
-        self.dictionary_action.setShortcut(Qt.CTRL+Qt.Key_L)
         self.dictionary_action.triggered.connect(self.lookup)
         self.addAction(self.dictionary_action)
         self.image_popup = ImagePopup(self)
@@ -514,7 +512,6 @@ class DocumentView(QWebView):  # {{{
         self.view_table_action.triggered.connect(self.popup_table)
         self.search_action = QAction(QIcon(I('dictionary.png')),
                 _('&Search for next occurrence'), self)
-        self.search_action.setShortcut(Qt.CTRL+Qt.Key_S)
         self.search_action.triggered.connect(self.search_next)
         self.addAction(self.search_action)
 
@@ -652,9 +649,9 @@ class DocumentView(QWebView):  # {{{
         text = self._selectedText()
         if text and img.isNull():
             self.search_online_action.setText(text)
-            menu.addAction(self.search_online_action)
+            for x, sc in (('search_online', 'Search online'), ('dictionary', 'Lookup word'), ('search', 'Next occurrence')):
-            menu.addAction(self.dictionary_action)
+                ac = getattr(self, '%s_action' % x)
-            menu.addAction(self.search_action)
+                menu.addAction(ac.icon(), '%s [%s]' % (unicode(ac.text()), ','.join(self.shortcuts.get_shortcuts(sc))), ac.trigger)
 
         if not text and img.isNull():
             menu.addSeparator()

src/calibre/gui2/viewer/keys.py

@@ -6,6 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+from calibre.constants import isosx
 
 SHORTCUTS = {
         'Next Page' : (['PgDown', 'Space'],
@@ -50,4 +51,37 @@ SHORTCUTS = {
         'Forward': (['Alt+Right'],
             _('Forward')),
 
+        'Quit': (['Ctrl+Q', 'Ctrl+W', 'Alt+F4'],
+            _('Quit')),
+
+        'Focus Search': (['/', 'Ctrl+F'],
+            _('Start search')),
+
+        'Show metadata': (['Ctrl+I'],
+            _('Show metadata')),
+
+        'Font larger': (['Ctrl+='],
+            _('Font size larger')),
+
+        'Font smaller': (['Ctrl+-'],
+            _('Font size smaller')),
+
+        'Fullscreen': ((['Ctrl+Meta+F'] if isosx else ['Ctrl+Shift+F', 'F11']),
+            _('Fullscreen')),
+
+        'Find next': (['F3'],
+            _('Find next')),
+
+        'Find previous': (['Shift+F3'],
+            _('Find previous')),
+
+        'Search online': (['Ctrl+E'],
+            _('Search online for word')),
+
+        'Lookup word': (['Ctrl+L'],
+            _('Lookup word in dictionary')),
+
+        'Next occurrence': (['Ctrl+S'],
+            _('Go to next occurrence of selected word')),
+
 }

src/calibre/gui2/viewer/main.py

@@ -8,7 +8,7 @@ from threading import Thread
 from PyQt4.Qt import (QApplication, Qt, QIcon, QTimer, QByteArray, QSize,
         QTime, QDoubleSpinBox, QLabel, QTextBrowser, QPropertyAnimation,
         QPainter, QBrush, QColor, pyqtSignal, QUrl, QRegExpValidator, QRegExp,
-        QLineEdit, QToolButton, QMenu, QInputDialog, QAction, QKeySequence,
+        QLineEdit, QToolButton, QMenu, QInputDialog, QAction,
         QModelIndex)
 
 from calibre.gui2.viewer.main_ui import Ui_EbookViewer
@@ -234,18 +234,9 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.view_resized_timer.timeout.connect(self.viewport_resize_finished)
         self.view_resized_timer.setSingleShot(True)
         self.resize_in_progress = False
-        qs = [Qt.CTRL+Qt.Key_Q,Qt.CTRL+Qt.Key_W]
-        self.action_quit.setShortcuts(qs)
         self.action_quit.triggered.connect(self.quit)
-        self.action_focus_search = QAction(self)
-        self.addAction(self.action_focus_search)
-        self.action_focus_search.setShortcuts([Qt.Key_Slash,
-            QKeySequence(QKeySequence.Find)])
-        self.action_focus_search.triggered.connect(lambda x:
-                self.search.setFocus(Qt.OtherFocusReason))
         self.action_copy.setDisabled(True)
         self.action_metadata.setCheckable(True)
-        self.action_metadata.setShortcut(Qt.CTRL+Qt.Key_I)
         self.action_table_of_contents.setCheckable(True)
         self.toc.setMinimumWidth(80)
         self.action_reference_mode.setCheckable(True)
@@ -255,18 +246,14 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.action_copy.triggered[bool].connect(self.copy)
         self.action_font_size_larger.triggered.connect(self.font_size_larger)
         self.action_font_size_smaller.triggered.connect(self.font_size_smaller)
-        self.action_font_size_larger.setShortcut(Qt.CTRL+Qt.Key_Equal)
-        self.action_font_size_smaller.setShortcut(Qt.CTRL+Qt.Key_Minus)
         self.action_open_ebook.triggered[bool].connect(self.open_ebook)
         self.action_next_page.triggered.connect(self.view.next_page)
         self.action_previous_page.triggered.connect(self.view.previous_page)
         self.action_find_next.triggered.connect(self.find_next)
         self.action_find_previous.triggered.connect(self.find_previous)
         self.action_full_screen.triggered[bool].connect(self.toggle_fullscreen)
-        self.action_full_screen.setShortcuts([Qt.Key_F11, Qt.CTRL+Qt.SHIFT+Qt.Key_F])
+        self.action_full_screen.setToolTip(_('Toggle full screen [%s]') %
-        self.action_full_screen.setToolTip(_('Toggle full screen (%s)') %
+                _(' or ').join([x for x in self.view.shortcuts.get_shortcuts('Fullscreen')]))
-                _(' or ').join([unicode(x.toString(x.NativeText)) for x in
-                    self.action_full_screen.shortcuts()]))
         self.action_back.triggered[bool].connect(self.back)
         self.action_forward.triggered[bool].connect(self.forward)
         self.action_preferences.triggered.connect(self.do_config)
@@ -348,11 +335,6 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.pos_label.setFocusPolicy(Qt.NoFocus)
         self.clock_timer = QTimer(self)
         self.clock_timer.timeout.connect(self.update_clock)
-        self.esc_full_screen_action = a = QAction(self)
-        self.addAction(a)
-        a.setShortcut(Qt.Key_Escape)
-        a.setEnabled(False)
-        a.triggered.connect(self.action_full_screen.trigger)
 
         self.print_menu = QMenu()
         self.print_menu.addAction(QIcon(I('print-preview.png')), _('Print Preview'))
@@ -360,9 +342,6 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.tool_bar.widgetForAction(self.action_print).setPopupMode(QToolButton.MenuButtonPopup)
         self.action_print.triggered.connect(self.print_book)
         self.print_menu.actions()[0].triggered.connect(self.print_preview)
-        ca = self.view.copy_action
-        ca.setShortcut(QKeySequence.Copy)
-        self.addAction(ca)
         self.open_history_menu = QMenu()
         self.clear_recent_history_action = QAction(
                 _('Clear list of recently opened books'), self)
@@ -487,6 +466,11 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                 at_start=True)
 
     def lookup(self, word):
+        from calibre.gui2.viewer.documentview import config
+        opts = config().parse()
+        settings = self.dictionary_view.page().settings()
+        settings.setFontSize(settings.DefaultFontSize, opts.default_font_size)
+        settings.setFontSize(settings.DefaultFixedFontSize, opts.mono_font_size)
         self.dictionary_view.setHtml('<html><body><p>'+
             _('Connecting to dict.org to lookup: <b>%s</b>&hellip;')%word +
             '</p></body></html>')
@@ -513,7 +497,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         p = Printing(self.iterator, self)
         p.start_preview()
 
-    def toggle_fullscreen(self, x):
+    def toggle_fullscreen(self):
         if self.isFullScreen():
             self.showNormal()
         else:
@@ -539,7 +523,6 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
 
     def show_full_screen_label(self):
         f = self.full_screen_label
-        self.esc_full_screen_action.setEnabled(True)
         height = 200
         width = int(0.7*self.view.width())
         f.resize(width, height)
@@ -604,7 +587,6 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.clock_timer.stop()
         self.vertical_scrollbar.setVisible(True)
         self.window_mode_changed = 'normal'
-        self.esc_full_screen_action.setEnabled(False)
         self.settings_changed()
         self.full_screen_label.setVisible(False)
         if hasattr(self, '_original_frame_margins'):
@@ -727,11 +709,11 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
 
     def magnification_changed(self, val):
         tt = '%(action)s [%(sc)s]\n'+_('Current magnification: %(mag).1f')
-        sc = unicode(self.action_font_size_larger.shortcut().toString())
+        sc = _(' or ').join(self.view.shortcuts.get_shortcuts('Font larger'))
         self.action_font_size_larger.setToolTip(
                 tt %dict(action=unicode(self.action_font_size_larger.text()),
                          mag=val, sc=sc))
-        sc = unicode(self.action_font_size_smaller.shortcut().toString())
+        sc = _(' or ').join(self.view.shortcuts.get_shortcuts('Font smaller'))
         self.action_font_size_smaller.setToolTip(
                 tt %dict(action=unicode(self.action_font_size_smaller.text()),
                          mag=val, sc=sc))
@@ -1116,10 +1098,40 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             self.load_path(self.iterator.spine[self.current_index-1], pos=1.0)
 
     def keyPressEvent(self, event):
-        MainWindow.keyPressEvent(self, event)
+        if event.key() == Qt.Key_Escape:
-        if not event.isAccepted():
+            if self.metadata.isVisible():
-            if not self.view.handle_key_press(event):
+                self.metadata.setVisible(False)
-                event.ignore()
+                event.accept()
+                return
+            if self.isFullScreen():
+                self.toggle_fullscreen()
+                event.accept()
+                return
+        try:
+            key = self.view.shortcuts.get_match(event)
+        except AttributeError:
+            return MainWindow.keyPressEvent(self, event)
+        action = {
+            'Quit':self.action_quit,
+            'Show metadata':self.action_metadata,
+            'Copy':self.view.copy_action,
+            'Font larger': self.action_font_size_larger,
+            'Font smaller': self.action_font_size_smaller,
+            'Fullscreen': self.action_full_screen,
+            'Find next': self.action_find_next,
+            'Find previous': self.action_find_previous,
+            'Search online': self.view.search_online_action,
+            'Lookup word': self.view.dictionary_action,
+            'Next occurrence': self.view.search_action,
+        }.get(key, None)
+        if action is not None:
+            event.accept()
+            action.trigger()
+            return
+        if key == 'Focus Search':
+            self.search.setFocus(Qt.OtherFocusReason)
+        if not self.view.handle_key_press(event):
+            event.ignore()
 
     def __enter__(self):
         return self

src/calibre/gui2/viewer/main.ui

@@ -248,9 +248,6 @@
    <property name="toolTip">
     <string>Find next occurrence</string>
    </property>
-   <property name="shortcut">
-    <string>F3</string>
-   </property>
   </action>
   <action name="action_copy">
    <property name="icon">
@@ -317,9 +314,6 @@
    <property name="toolTip">
     <string>Find previous occurrence</string>
    </property>
-   <property name="shortcut">
-    <string>Shift+F3</string>
-   </property>
   </action>
   <action name="action_toggle_paged_mode">
    <property name="checkable">

src/calibre/gui2/widgets.py

@@ -247,7 +247,7 @@ class ImageDropMixin(object):  # {{{
     def set_pixmap(self, pmap):
         self.setPixmap(pmap)
 
-    def contextMenuEvent(self, ev):
+    def build_context_menu(self):
         cm = QMenu(self)
         paste = cm.addAction(_('Paste Cover'))
         copy = cm.addAction(_('Copy Cover'))
@@ -255,7 +255,10 @@ class ImageDropMixin(object):  # {{{
             paste.setEnabled(False)
         copy.triggered.connect(self.copy_to_clipboard)
         paste.triggered.connect(self.paste_from_clipboard)
-        cm.exec_(ev.globalPos())
+        return cm
+
+    def contextMenuEvent(self, ev):
+        self.build_context_menu().exec_(ev.globalPos())
 
     def copy_to_clipboard(self):
         QApplication.instance().clipboard().setPixmap(self.get_pixmap())
@@ -276,13 +279,16 @@ class ImageView(QWidget, ImageDropMixin):  # {{{
     BORDER_WIDTH = 1
     cover_changed = pyqtSignal(object)
 
-    def __init__(self, parent=None):
+    def __init__(self, parent=None, show_size_pref_name=None, default_show_size=False):
         QWidget.__init__(self, parent)
+        self.show_size_pref_name = ('show_size_on_cover_' + show_size_pref_name) if show_size_pref_name else None
         self._pixmap = QPixmap(self)
         self.setMinimumSize(QSize(150, 200))
         ImageDropMixin.__init__(self)
         self.draw_border = True
         self.show_size = False
+        if self.show_size_pref_name:
+            self.show_size = gprefs.get(self.show_size_pref_name, default_show_size)
 
     def setPixmap(self, pixmap):
         if not isinstance(pixmap, QPixmap):
@@ -291,6 +297,19 @@ class ImageView(QWidget, ImageDropMixin):  # {{{
         self.updateGeometry()
         self.update()
 
+    def build_context_menu(self):
+        m = ImageDropMixin.build_context_menu(self)
+        if self.show_size_pref_name:
+            text = _('Hide size in corner') if self.show_size else _('Show size in corner')
+            m.addAction(text, self.toggle_show_size)
+        return m
+
+    def toggle_show_size(self):
+        self.show_size ^= True
+        if self.show_size_pref_name:
+            gprefs[self.show_size_pref_name] = self.show_size
+        self.update()
+
     def pixmap(self):
         return self._pixmap
 

src/calibre/library/cli.py

@@ -629,9 +629,12 @@ def command_set_metadata(args, dbpath):
 
     if opts.field:
         fields = {k:v for k, v in fields()}
+        fields['title_sort'] = fields['sort']
         vals = {}
         for x in opts.field:
             field, val = x.partition(':')[::2]
+            if field == 'sort':
+                field = 'title_sort'
             if field not in fields:
                 print >>sys.stderr, _('%s is not a known field'%field)
                 return 1

src/calibre/library/server/content.py

@@ -18,6 +18,7 @@ from calibre.utils.magick.draw import (save_cover_data_to, Image,
         thumbnail as generate_thumbnail)
 from calibre.utils.filenames import ascii_filename
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
+from calibre.utils.config import tweaks
 
 plugboard_content_server_value = 'content_server'
 plugboard_content_server_formats = ['epub', 'mobi', 'azw3']
@@ -175,8 +176,13 @@ class ContentServer(object):
             cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
 
             if thumbnail:
-                return generate_thumbnail(cover,
+                quality = tweaks['content_server_thumbnail_compression_quality']
-                        width=thumb_width, height=thumb_height)[-1]
+                if quality < 50:
+                    quality = 50
+                elif quality > 99:
+                    quality = 99
+                return generate_thumbnail(cover, width=thumb_width,
+                        height=thumb_height, compression_quality=quality)[-1]
 
             img = Image()
             img.load(cover)

src/calibre/library/server/opds.py

@@ -539,9 +539,14 @@ class OPDSServer(object):
             try:
                 p = which.index(':')
                 category = which[p+1:]
+                which = which[:p]
+                # This line will toss an exception for composite columns
                 which = int(which[:p])
             except:
-                raise cherrypy.HTTPError(404, 'Tag %r not found'%which)
+                # Might be a composite column, where we have the lookup key
+                if not (category in self.db.field_metadata and
+                        self.db.field_metadata[category]['datatype'] == 'composite'):
+                    raise cherrypy.HTTPError(404, 'Tag %r not found'%which)
 
         categories = self.categories_cache(
                 self.get_opds_allowed_ids_for_version(version))

src/calibre/utils/config.py

@@ -8,8 +8,7 @@ Manage application-wide preferences.
 '''
 import os, cPickle, base64, datetime, json, plistlib
 from copy import deepcopy
-from optparse import OptionParser as _OptionParser, OptionGroup
+import optparse
-from optparse import IndentedHelpFormatter
 
 from calibre.constants import (config_dir, CONFIG_DIR_MODE, __appname__,
         get_version, __author__)
@@ -18,6 +17,10 @@ from calibre.utils.config_base import (make_config_dir, Option, OptionValues,
         OptionSet, ConfigInterface, Config, prefs, StringConfig, ConfigProxy,
         read_raw_tweaks, read_tweaks, write_tweaks, tweaks, plugin_dir)
 
+# optparse uses gettext.gettext instead of _ from builtins, so we
+# monkey patch it.
+optparse._ = _
+
 if False:
     # Make pyflakes happy
     Config, ConfigProxy, Option, OptionValues, StringConfig
@@ -27,7 +30,7 @@ if False:
 def check_config_write_access():
     return os.access(config_dir, os.W_OK) and os.access(config_dir, os.X_OK)
 
-class CustomHelpFormatter(IndentedHelpFormatter):
+class CustomHelpFormatter(optparse.IndentedHelpFormatter):
 
     def format_usage(self, usage):
         from calibre.utils.terminal import colored
@@ -72,7 +75,7 @@ class CustomHelpFormatter(IndentedHelpFormatter):
         return "".join(result)+'\n'
 
 
-class OptionParser(_OptionParser):
+class OptionParser(optparse.OptionParser):
 
     def __init__(self,
                  usage='%prog [options] filename',
@@ -91,7 +94,7 @@ class OptionParser(_OptionParser):
                  '''enclose the arguments in quotation marks.''')+'\n'
         if version is None:
             version = '%%prog (%s %s)'%(__appname__, get_version())
-        _OptionParser.__init__(self, usage=usage, version=version, epilog=epilog,
+        optparse.OptionParser.__init__(self, usage=usage, version=version, epilog=epilog,
                                formatter=CustomHelpFormatter(),
                                conflict_handler=conflict_handler, **kwds)
         self.gui_mode = gui_mode
@@ -104,22 +107,22 @@ class OptionParser(_OptionParser):
     def print_usage(self, file=None):
         from calibre.utils.terminal import ANSIStream
         s = ANSIStream(file)
-        _OptionParser.print_usage(self, file=s)
+        optparse.OptionParser.print_usage(self, file=s)
 
     def print_help(self, file=None):
         from calibre.utils.terminal import ANSIStream
         s = ANSIStream(file)
-        _OptionParser.print_help(self, file=s)
+        optparse.OptionParser.print_help(self, file=s)
 
     def print_version(self, file=None):
         from calibre.utils.terminal import ANSIStream
         s = ANSIStream(file)
-        _OptionParser.print_version(self, file=s)
+        optparse.OptionParser.print_version(self, file=s)
 
     def error(self, msg):
         if self.gui_mode:
             raise Exception(msg)
-        _OptionParser.error(self, msg)
+        optparse.OptionParser.error(self, msg)
 
     def merge(self, parser):
         '''
@@ -182,8 +185,8 @@ class OptionParser(_OptionParser):
 
     def add_option_group(self, *args, **kwargs):
         if isinstance(args[0], type(u'')):
-            args = [OptionGroup(self, *args, **kwargs)] + list(args[1:])
+            args = [optparse.OptionGroup(self, *args, **kwargs)] + list(args[1:])
-        return _OptionParser.add_option_group(self, *args, **kwargs)
+        return optparse.OptionParser.add_option_group(self, *args, **kwargs)
 
 class DynamicConfig(dict):
     '''

src/calibre/utils/config_base.py

@@ -400,6 +400,8 @@ def _prefs():
             help=_('Add new formats to existing book records'))
     c.add_opt('installation_uuid', default=None, help='Installation UUID')
     c.add_opt('new_book_tags', default=[], help=_('Tags to apply to books added to the library'))
+    c.add_opt('mark_new_books', default=False, help=_(
+        'Mark newly added books. The mark is a temporary mark that is automatically removed when calibre is restarted.'))
 
     # these are here instead of the gui preferences because calibredb and
     # calibre server can execute searches

src/calibre/utils/filenames.py

@@ -295,6 +295,15 @@ def windows_hardlink(src, dest):
         msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest)
         raise Exception(msg % ('hardlink size: %d not the same as source size' % sz))
 
+def windows_nlinks(path):
+    import win32file
+    dwFlagsAndAttributes = win32file.FILE_FLAG_BACKUP_SEMANTICS if os.path.isdir(path) else 0
+    handle = win32file.CreateFile(path, win32file.GENERIC_READ, win32file.FILE_SHARE_READ, None, win32file.OPEN_EXISTING, dwFlagsAndAttributes, None)
+    try:
+        return win32file.GetFileInformationByHandle(handle)[7]
+    finally:
+        handle.Close()
+
 class WindowsAtomicFolderMove(object):
 
     '''
@@ -400,6 +409,12 @@ def hardlink_file(src, dest):
         return
     os.link(src, dest)
 
+def nlinks_file(path):
+    ' Return number of hardlinks to the file '
+    if iswindows:
+        return windows_nlinks(path)
+    return os.stat(path).st_nlink
+
 def atomic_rename(oldpath, newpath):
     '''Replace the file newpath with the file oldpath. Can fail if the files
     are on different volumes. If succeeds, guaranteed to be atomic. newpath may