Sync to trunk.

Changelog.yaml

@@ -19,6 +19,149 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.8.63
+  date: 2012-08-02
+
+  new features:
+    - title: "E-book viewer: Allow quick saving and loading of viewer settings as 'themes'." 
+      tickets: [1024611]
+
+    - title: "Ebook-viewer: Add a restore defaults button to the viewer preferences dialog"
+
+    - title: "E-book viewer: Add simple settings for text and background colors"
+
+    - title: "Add an entry to save to disk when right clicking a format in the book details panel"
+
+    - title: "ODT metadata: Read first image as the metadata cover from ODT files. Also allow ODT authors to set custom properties for extended metadata."
+
+    - title: "E-book viewer and PDF Output: Resize images that are longer than the page to fit onto a single page"
+ 
+  bug fixes:
+    - title: "KF8 Output: Fix bug where some calibre generated KF8 files would cause the Amazon KF8 viewer on the Touch to go to into an infinite loop when using the next page function"
+      tickets: [1026421]
+
+    - title: "News download: Add support for <img> tags that link to SVG images."
+      tickets: [1031553]
+
+    - title: "Update podofo to 0.9.1 in all binary builds, to fix corruption of some PDFs when updating metadata."
+      tickets: [1031086]
+
+    - title: "Catalog generation: Handle authors whose last name is a number."
+
+    - title: "KF8 Input: Handle html entities in the NCX toc entries correctly"
+
+    - title: "Fix a calibre crash that affected some windows installs"
+      tickets: [1030234]
+
+    - title: "MOBI Output: Normalize unicode strings before writing to file, to workaround lack of support for non-normal unicode in Amazon's MOBI renderer."
+      tickets: [1029825]
+
+    - title: "EPUB Input: Handle files that have duplicate entries in the spine"
+
+    - title: "Fix regression in Kobo driver that caused the on device column to not be updated after deleting books"
+
+  new recipes:
+    - title: Dziennik Polski 
+      author: Gregorz Maj 
+      
+    - title: High Country Blogs 
+      author: Armin Geller
+
+    - title: Philosophy Now 
+      author: Rick Shang
+
+- version: 0.8.62
+  date: 2012-07-27
+
+  new features:
+    - title: "Book details panel: Allow right clicking on a format to delete it."
+
+    - title: "When errors occur in lots of background jobs, add an option to the error message to temporarily suppress subsequent error messages."
+      tickets: [886904]
+
+    - title: "E-book viewer full screen mode: Allow clicking in the left and right page margins to turn pages."
+      tickets: [1024819]
+
+    - title: "Drivers for various Android devices"
+      tickets: [1028690,1027431]
+
+    - title: "Advanced search dialog: When starting on the title/author/etc. tab, restore the previously used search kind as well."
+      tickets: [1029745]
+
+    - title: "When presenting the calibre must be restarted warning after installing a new plugin, add a restart now button so that the user can conveniently restart calibre. Currently only works when going vie Preferences->Plugins->Get new plugins"
+ 
+  bug fixes:
+    - title: "Fix main window layout state being saved incorrectly if calibre is killed without a proper shutdown"
+
+    - title: "Fix boolean and date searching in non english calibre installs."
+
+    - title: "Conversion: Ignore invalid chapter detection and level n ToC expressions instead of erroring out"
+
+  improved recipes:
+    - Psychology Today
+    - The Smithsonian
+    - The New Republic
+    - Various updated Polish news sources
+    - The Sun
+    - San Francisco Bay Guardian
+    - AnandTech
+    - Smashing Magazine
+
+  new recipes:
+    - title: Linux Journal and Conowego.pl 
+      author: fenuks
+
+    - title: A list apart and .net magazine 
+      author: Marc Busque
+
+- version: 0.8.61
+  date: 2012-07-20
+
+  new features:
+    - title: "E-book viewer: Add a paged mode that splits up the text into pages, like in a paper book instead of presenting it as a single column. To activate click the button with the yellow scroll icon in the top right corner."
+      type: major
+      description: "In paged mode, the ebook viewer no longer cuts off the last line of text at the bottom of the screen, and it respects CSS page-break directives. You can also set page margins and control the number of pages displayed on screen by clicking the Preferences button in the viewer and going to 'Text layout in paged mode'."
+
+    - title: "Digitally sign the calibre OS X and windows builds"
+
+    - title: "Get Books: Add Mills and Boon UK"
+
+    - title: "Various minor improvements to the Bulk metadata edit dialog"
+      tickets: [1025825, 1025838, 1025628]
+
+    - title: "Fix various regression in the auto-complete functionality for authors/series/tags etc introduced in 0.8.60"
+
+    - title: "Drivers for various new Android devices"
+      tickets: [1024934]
+
+    - title: "MOBI: Add support for the new language EXTH header field in MOBI files generated by kindlegen 2.5"
+ 
+  bug fixes:
+    - title: "KF8 Output: Fix calibre produced KF8 files not showing the 'Use publisher font' option on the Kindle Touch when they have embedded fonts"
+
+    - title: "Txt/fb2/rtf/pml/rb output: Fix non-visibile element's tail text (which should be visible) is being ignored when it shouldn't." 
+      tickets: [1026541]
+
+    - title: "Book details panel: When displaying a link to amazon, use a country specific name like amazon.fr instead of using amazon.com for all countries"
+
+    - title: "Conversion: When splitting on page breaks, ignore page-breaks with values of auto and inherit. "
+      tickets: [1018875]
+
+    - title: "Metadata jacket: Specify foreground in addition to the  background color for the title banner so that it remain readable if the user tries to monkey with the CSS in the viewer."
+
+    - title: "PDF Output: Fix rendering of cover as first age of PDF (ignore margins so that the image covers the entire page)"
+
+    - title: "Linux binaries: Bundle libglib to avoid incompatibilities with glib on various distros."
+      tickets: [1022019]
+
+    - title: "Fix find_identical_books() choking on books with too many authors"
+
+
+  improved recipes:
+    - Toronto Star
+    - American Prospect
+    - faz.net
+
 - version: 0.8.60
   date: 2012-07-13
 

manual/conversion.rst

@@ -710,3 +710,35 @@ EPUB from the ZIP file are::
 
 Note that because this file explores the potential of EPUB, most of the advanced formatting is not going to work on readers less capable than |app|'s built-in EPUB viewer. 
 
+
+Convert ODT documents
+~~~~~~~~~~~~~~~~~~~~~
+
+|app| can directly convert ODT (OpenDocument Text) files. You should use styles to format your document and minimize the use of direct formatting.
+When inserting images into your document you need to anchor them to the paragraph, images anchored to a page will all end up in the front of the conversion.
+
+To enable automatic detection of chapters, you need to mark them with the build-in styles called 'Heading 1', 'Heading 2', ..., 'Heading 6' ('Heading 1' equates to the HTML tag <h1>, 'Heading 2' to <h2> etc). When you convert in |app| you can enter which style you used into the 'Detect chapters at' box. Example:
+
+  * If you mark Chapters with style 'Heading 2', you have to set the 'Detect chapters at' box to ``//h:h2``
+  * For a nested TOC with Sections marked with 'Heading 2' and the Chapters marked with 'Heading 3' you need to enter ``//h:h2|//h:h3``. On the Convert - TOC page set the 'Level 1 TOC' box to ``//h:h2`` and the 'Level 2 TOC' box to ``//h:h3``.
+
+Well-known document properties (Title, Keywords, Description, Creator) are recognized and |app| will use the first image (not to small, and with good aspect-ratio) as the cover image.
+
+There is also an advanced property conversion mode, which is activated by setting the custom property ``opf.metadata`` ('Yes or No' type) to Yes in your ODT document (File->Properties->Custom Properties).
+If this property is detected by |app|, the following custom properties are recognized (``opf.authors`` overrides document creator)::
+
+    opf.titlesort
+    opf.authors
+    opf.authorsort
+    opf.publisher
+    opf.pubdate
+    opf.isbn
+    opf.language
+    opf.series
+    opf.seriesindex
+
+In addition to this, you can specify the picture to use as the cover by naming it ``opf.cover`` (right click, Picture->Options->Name) in the ODT. If no picture with this name is found, the 'smart' method is used.
+As the cover detection might result in double covers in certain output formats, the process will remove the paragraph (only if the only content is the cover!) from the document. But this works only with the named picture!
+
+To disable cover detection you can set the custom property ``opf.nocover`` ('Yes or No' type) to Yes in advanced mode.
+

manual/develop.rst

@@ -149,24 +149,25 @@ the previously checked out |app| code directory, for example::
 
 calibre is the directory that contains the src and resources sub-directories. Ensure you have installed the |app| commandline tools via :guilabel:`Preferences->Advanced->Miscellaneous` in the |app| GUI.
 
-The next step is to set the environment variable ``CALIBRE_DEVELOP_FROM`` to the absolute path of the src directory.
-So, following the example above, it would be ``/Users/kovid/work/calibre/src``. Apple 
-`documentation <http://developer.apple.com/mac/library/documentation/MacOSX/Conceptual/BPRuntimeConfig/Articles/EnvironmentVars.html#//apple_ref/doc/uid/20002093-BCIJIJBH>`_
-on how to set environment variables. 
+The next step is to create a bash script that will set the environment variable ``CALIBRE_DEVELOP_FROM`` to the absolute path of the src directory when running calibre in debug mode.
 
-Once you have set the environment variable, open a new Terminal and check that it was correctly set by using
-the command::
+Create a plain text file::
 
-    echo $CALIBRE_DEVELOP_FROM
+    #!/bin/sh
+    export CALIBRE_DEVELOP_FROM="/Users/kovid/work/calibre/src"
+    calibre-debug -g
 
-Setting this environment variable means that |app| will now load all its Python code from the specified location.
+Save this file as ``/usr/bin/calibre-develop``, then set its permissions so that it can be executed::
 
-That's it! You are now ready to start hacking on the |app| code. For example, open the file :file:`src/calibre/__init__.py`
-in your favorite editor and add the line::
+    chmod +x /usr/bin/calibre-develop
 
-    print ("Hello, world!")
+Once you have done this, run::
 
-near the top of the file. Now run the command :command:`calibredb`. The very first line of output should be ``Hello, world!``.
+    calibre-develop
+
+You should see some diagnostic information in the Terminal window as calibre
+starts up, and you should see an asterisk after the version number in the GUI
+window, indicating that you are running from source.
 
 Linux development environment
 ------------------------------

manual/news.rst

@@ -30,7 +30,7 @@ Lets pick a couple of feeds that look interesting:
     #. Business Travel: http://feeds.portfolio.com/portfolio/businesstravel
     #. Tech Observer: http://feeds.portfolio.com/portfolio/thetechobserver
 
-I got the URLs by clicking the little orange RSS icon next to each feed name. To make |app| download the feeds and convert them into an ebook, you should click the :guilabel:`Fetch news` button and then the :guilabel:`Add a custom news source` menu item. A dialog similar to that shown below should open up.
+I got the URLs by clicking the little orange RSS icon next to each feed name. To make |app| download the feeds and convert them into an ebook, you should right click the :guilabel:`Fetch news` button and then the :guilabel:`Add a custom news source` menu item. A dialog similar to that shown below should open up.
 
 .. image:: images/custom_news.png
     :align: center

recipes/anandtech.recipe

@@ -21,8 +21,12 @@ class anan(BasicNewsRecipe):
     remove_javascript = True
     encoding = 'utf-8'
 
-    remove_tags=[dict(name='a', attrs={'style':'width:110px; margin-top:0px;text-align:center;'}),
-		 dict(name='a', attrs={'style':'width:110px; margin-top:0px; margin-right:20px;text-align:center;'})]
+    remove_tags=[
+        dict(name='a', attrs={'style':'width:110px; margin-top:0px;text-align:center;'}),
+        dict(name='a', attrs={'style':'width:110px; margin-top:0px; margin-right:20px;text-align:center;'}),
+        {'attrs':{'class':['article_links', 'header', 'body_right']}},
+        {'id':['crumbs']},
+        ]
 
     feeds =  [ ('Anandtech', 'http://www.anandtech.com/rss/')]
 

recipes/benchmark_pl.recipe

@@ -1,6 +1,6 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
-class Benchmark_pl(BasicNewsRecipe):
+class BenchmarkPl(BasicNewsRecipe):
     title          = u'Benchmark.pl'
     __author__        = 'fenuks'
     description   = u'benchmark.pl -IT site'
@@ -14,7 +14,7 @@ class Benchmark_pl(BasicNewsRecipe):
     preprocess_regexps = [(re.compile(ur'<h3><span style="font-size: small;">&nbsp;Zobacz poprzednie <a href="http://www.benchmark.pl/news/zestawienie/grupa_id/135">Opinie dnia:</a></span>.*</body>', re.DOTALL|re.IGNORECASE), lambda match: '</body>'), (re.compile(ur'Więcej o .*?</ul>', re.DOTALL|re.IGNORECASE), lambda match: '')]
     keep_only_tags=[dict(name='div', attrs={'class':['m_zwykly', 'gallery']})]
     remove_tags_after=dict(name='div', attrs={'class':'body'})
-    remove_tags=[dict(name='div', attrs={'class':['kategoria', 'socialize', 'thumb', 'panelOcenaObserwowane', 'categoryNextToSocializeGallery']}), dict(name='table', attrs={'background':'http://www.benchmark.pl/uploads/backend_img/a/fotki_newsy/opinie_dnia/bg.png'}), dict(name='table', attrs={'width':'210', 'cellspacing':'1', 'cellpadding':'4', 'border':'0', 'align':'right'})]
+    remove_tags=[dict(name='div', attrs={'class':['kategoria', 'socialize', 'thumb', 'panelOcenaObserwowane', 'categoryNextToSocializeGallery', 'breadcrumb']}), dict(name='table', attrs={'background':'http://www.benchmark.pl/uploads/backend_img/a/fotki_newsy/opinie_dnia/bg.png'}), dict(name='table', attrs={'width':'210', 'cellspacing':'1', 'cellpadding':'4', 'border':'0', 'align':'right'})]
     INDEX= 'http://www.benchmark.pl'
     feeds          = [(u'Aktualności', u'http://www.benchmark.pl/rss/aktualnosci-pliki.xml'), 
                           (u'Testy i recenzje', u'http://www.benchmark.pl/rss/testy-recenzje-minirecenzje.xml')]

recipes/conowego_pl.recipe

@@ -0,0 +1,38 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
+class CoNowegoPl(BasicNewsRecipe):
+    title          = u'conowego.pl'
+    __author__        = 'fenuks'
+    description   = u'Nowy wortal technologiczny oraz gazeta internetowa. Testy najnowszych produktów, fachowe porady i recenzje. U nas znajdziesz wszystko o elektronice użytkowej !'
+    cover_url      = 'http://www.conowego.pl/fileadmin/templates/main/images/logo_top.png'
+    category       = 'IT, news'
+    language       = 'pl'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    remove_empty_feeds = True
+    use_embedded_content = False
+    keep_only_tags = [dict(name='div', attrs={'class':'news_list single_view'})]
+    remove_tags = [dict(name='div', attrs={'class':['ni_bottom', 'ni_rank', 'ni_date']})]
+    feeds          = [(u'Aktualno\u015bci', u'http://www.conowego.pl/rss/aktualnosci-5/?type=100'), (u'Gaming', u'http://www.conowego.pl/rss/gaming-6/?type=100'), (u'Porady', u'http://www.conowego.pl/rss/porady-3/?type=100'), (u'Testy', u'http://www.conowego.pl/rss/testy-2/?type=100')]
+
+    def preprocess_html(self, soup):
+        for i in soup.findAll('img'):
+            i.parent.insert(0, BeautifulSoup('<br />'))
+            i.insert(len(i), BeautifulSoup('<br />'))
+        self.append_page(soup, soup.body)
+        return soup
+
+
+    def append_page(self, soup, appendtag):
+        tag = appendtag.find('div', attrs={'class':'pages'})
+        if tag:
+            nexturls=tag.findAll('a')
+            for nexturl in nexturls[:-1]:
+                soup2 = self.index_to_soup('http://www.conowego.pl/' + nexturl['href'])
+                pagetext = soup2.find(attrs={'class':'ni_content'})
+                pos = len(appendtag.contents)
+                appendtag.insert(pos, pagetext)
+
+            for r in appendtag.findAll(attrs={'class':['pages', 'paginationWrap']}):
+                r.extract()

recipes/dot_net.recipe

@@ -0,0 +1,32 @@
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+class NetMagazineRecipe (BasicNewsRecipe):
+   __author__ = u'Marc Busqué <marc@lamarciana.com>'
+   __url__ = 'http://www.lamarciana.com'
+   __version__ = '1.0'
+   __license__   = 'GPL v3'
+   __copyright__ = u'2012, Marc Busqué <marc@lamarciana.com>'
+   title = u'.net magazine'
+   description = u'net is the world’s best-selling magazine for web designers and developers, featuring tutorials from leading agencies, interviews with the web’s biggest names, and agenda-setting features on the hottest issues affecting the internet today.'
+   language = 'en'
+   tags = 'web development, software'
+   oldest_article = 7
+   remove_empty_feeds = True
+   no_stylesheets = True
+   cover_url = u'http://media.netmagazine.futurecdn.net/sites/all/themes/netmag/logo.png'
+   keep_only_tags = [
+         dict(name='article', attrs={'class': re.compile('^node.*$', re.IGNORECASE)})
+         ]
+   remove_tags = [
+         dict(name='span', attrs={'class': 'comment-count'}),
+         dict(name='div', attrs={'class': 'item-list share-links'}),
+         dict(name='footer'),
+         ]
+   remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan', 'valign', 'vspace', 'hspace', 'alt', 'width', 'height', 'style']
+   extra_css = 'img {max-width: 100%; display: block; margin: auto;} .captioned-image div {text-align: center; font-style: italic;}'
+
+   feeds = [
+         (u'.net', u'http://feeds.feedburner.com/net/topstories'),
+         ]

recipes/dziennik_polski.recipe

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+__license__='GPL v3'
+__author__='grzegorz.maj@dziennik.krakow.pl>'
+
+'''
+http://dziennikpolski24.pl
+Author: grzegorz.maj@dziennik.krakow.pl
+'''
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class DziennikPolski24(BasicNewsRecipe):
+
+    title=u'Dziennik Polski'
+    publisher=u'Grupa Polskapresse'
+
+    __author__='grzegorz.maj'
+    description=u'Wiadomości z wydania Dziennika Polskiego'
+    oldest_article=1
+    max_articles_per_feed=50
+    needs_subscription=True
+
+    remove_javascript=True
+    no_stylesheets=True
+    use_embedded_content=False
+    remove_empty_feeds=True
+    extra_css='.date{margin-top: 4em;} .logo_author{margin-left:0.5em;}'
+
+    publication_type='newspaper'
+    cover_url='http://www.dziennikpolski24.pl/_p/images/logoDP24-b.gif'
+    INDEX='http://dziennikpolski24.pl/'
+
+    encoding='utf-8'
+    language='pl'
+
+    keep_only_tags=[
+
+                                   dict(name = 'div', attrs = {'class':['toolbar']})
+                                   , dict(name = 'h1')
+                                   , dict(name = 'h2', attrs = {'class':['teaser']})
+                                   , dict(name = 'div', attrs = {'class':['picture']})
+                                   , dict(name = 'div', attrs = {'id':['showContent']})
+                                    , dict(name = 'div', attrs = {'class':['paging']})
+                                    , dict(name = 'div', attrs = {'class':['wykupTresc']})
+                                ]
+
+    remove_tags=[
+
+                          ]
+
+    feeds=[
+                      (u'Kraj', u'http://www.dziennikpolski24.pl/rss/feed/1151')
+                    , (u'Świat', u'http://www.dziennikpolski24.pl/rss/feed/1153')
+                    , (u'Gospodarka', u'http://www.dziennikpolski24.pl/rss/feed/1154')
+                    , (u'Małopolska', u'http://www.dziennikpolski24.pl/rss/feed/1155')
+                    , (u'Kultura', u'http://www.dziennikpolski24.pl/rss/feed/1156')
+                    , (u'Opinie', u'http://www.dziennikpolski24.pl/rss/feed/1158')
+                    , (u'Kronika Nowohucka', u'http://www.dziennikpolski24.pl/rss/feed/1656')
+                    , (u'Na bieżąco', u'http://www.dziennikpolski24.pl/rss/feed/1543')
+                    , (u'Londyn 2012', u'http://www.dziennikpolski24.pl/rss/feed/2545')
+                    , (u'Piłka nożna', u'http://www.dziennikpolski24.pl/rss/feed/2196')
+                    , (u'Siatkówka', u'http://www.dziennikpolski24.pl/rss/feed/2197')
+                    , (u'Koszykówka', u'http://www.dziennikpolski24.pl/rss/feed/2198')
+                    , (u'Tenis', u'http://www.dziennikpolski24.pl/rss/feed/2199')
+                    , (u'Formuła 1', u'http://www.dziennikpolski24.pl/rss/feed/2203')
+                    , (u'Lekkoatletyka', u'http://www.dziennikpolski24.pl/rss/feed/2204')
+                    , (u'Żużel', u'http://www.dziennikpolski24.pl/rss/feed/2200')
+                    , (u'Sporty motorowe', u'http://www.dziennikpolski24.pl/rss/feed/2206')
+                    , (u'Publicystyka sportowa', u'http://www.dziennikpolski24.pl/rss/feed/2201')
+                    , (u'Kolarstwo', u'http://www.dziennikpolski24.pl/rss/feed/2205')
+                    , (u'Inne', u'http://www.dziennikpolski24.pl/rss/feed/2202')
+                    , (u'Miasto Kraków', u'http://www.dziennikpolski24.pl/rss/feed/1784')
+                    , (u'Region nowosądecki', u'http://www.dziennikpolski24.pl/rss/feed/1795')
+                    , (u'Region Małopolski Zachodniej', u'http://www.dziennikpolski24.pl/rss/feed/1793')
+                    , (u'Region tarnowski', u'http://www.dziennikpolski24.pl/rss/feed/1797')
+                    , (u'Region podhalański', u'http://www.dziennikpolski24.pl/rss/feed/1789')
+                    , (u'Region olkuski', u'http://www.dziennikpolski24.pl/rss/feed/1670')
+                    , (u'Region miechowski', u'http://www.dziennikpolski24.pl/rss/feed/1806')
+                    , (u'Region podkrakowski', u'http://www.dziennikpolski24.pl/rss/feed/1787')
+                    , (u'Region proszowicki', u'http://www.dziennikpolski24.pl/rss/feed/1804')
+                    , (u'Region wielicki', u'http://www.dziennikpolski24.pl/rss/feed/1802')
+                    , (u'Region podbeskidzki', u'http://www.dziennikpolski24.pl/rss/feed/1791')
+                    , (u'Region myślenicki', u'http://www.dziennikpolski24.pl/rss/feed/1800')
+                    , (u'Autosalon', u'http://www.dziennikpolski24.pl/rss/feed/1294')
+                    , (u'Kariera', u'http://www.dziennikpolski24.pl/rss/feed/1289')
+                    , (u'Przegląd nieruchomości', u'http://www.dziennikpolski24.pl/rss/feed/1281')
+                    , (u'Magnes', u'http://www.dziennikpolski24.pl/rss/feed/1283')
+                    , (u'Magazyn Piątek', u'http://www.dziennikpolski24.pl/rss/feed/1293')
+                    , (u'Pejzaż rodzinny', u'http://www.dziennikpolski24.pl/rss/feed/1274')
+                    , (u'Podróże', u'http://www.dziennikpolski24.pl/rss/feed/1275')
+                    , (u'Konsument', u'http://www.dziennikpolski24.pl/rss/feed/1288')
+    ]
+
+    def append_page(self, soup, appendtag):
+        loop=False
+        tag=soup.find('div', attrs = {'class':'paging'})
+        if tag:
+            loop=True
+            li_nks=tag.findAll('li')
+            appendtag.find('div', attrs = {'class':'paging'}).extract()
+        if appendtag.find('ul', attrs = {'class':'menuf'}):
+            appendtag.find('ul', attrs = {'class':'menuf'}).extract()
+        while loop:
+            loop=False
+            for li_nk in li_nks:
+                link_tag=li_nk.contents[0].contents[0].string
+                if u'następna' in link_tag:
+                    soup2=self.index_to_soup(self.INDEX+li_nk.contents[0]['href'])
+                    if soup2.find('div', attrs = {'id':'showContent'}):
+                        pagetext=soup2.find('div', attrs = {'id':'showContent'})
+                        pos=len(appendtag.contents)
+                        appendtag.insert(pos, pagetext)
+                        if soup2.find('div', attrs = {'class':'rightbar'}):
+                            pagecont=soup2.find('div', attrs = {'class':'rightbar'})
+                            tag=pagecont.find('div', attrs = {'class':'paging'})
+                            li_nks=tag.findAll('li')
+                            loop=True
+
+    def get_browser(self):
+        br=BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open('http://www.dziennikpolski24.pl/pl/moje-konto/950606-loguj.html')
+            br.select_form(nr = 1)
+            br["user_login[login]"]=self.username
+            br['user_login[pass]']=self.password
+            br.submit()
+        return br
+
+
+    def preprocess_html(self, soup):
+        self.append_page(soup, soup.body)
+        return soup
+

recipes/ekundelek_pl.recipe

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+__license__ = 'GPL v3'
+__copyright__ = u'2012, Artur Stachecki <artur.stachecki@gmail.com>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class swiatczytnikow(BasicNewsRecipe):
+    title = u'eKundelek'
+    description = u'Najsympatyczniejszy blog o e-czytnikach Kindle'
+    language = 'pl'
+    __author__ = u'Artur Stachecki'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    remove_tags = [dict(name = 'div', attrs = {'class' : 'feedflare'})]
+
+    feeds = [(u'Wpisy', u'http://feeds.feedburner.com/Ekundelekpl?format=xml')]

recipes/fhm_uk.recipe

@@ -18,15 +18,15 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
     keep_only_tags = [
                dict(name='h1'),
                dict(name='img',attrs={'id' : 'ctl00_Body_imgMainImage'}),
-               dict(name='div',attrs={'id' : ['articleLeft']}),
-                               dict(name='div',attrs={'class' : ['imagesCenterArticle','containerCenterArticle','articleBody']}),
+               dict(name='div',attrs={'id' : ['profileLeft','articleLeft','profileRight','profileBody']}),
+               dict(name='div',attrs={'class' : ['imagesCenterArticle','containerCenterArticle','articleBody',]}),
 
         ]
 
-    #remove_tags    = [
-                              #dict(attrs={'class' : ['player']}),
+    remove_tags    = [
+                              dict(attrs={'id' : ['ctl00_Body_divSlideShow' ]}),
 
-    #]
+    ]
     feeds          = [
         (u'Homepage 1',u'http://feed43.com/6655867614547036.xml'),
         (u'Homepage 2',u'http://feed43.com/4167731873103110.xml'),
@@ -34,7 +34,7 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
         (u'Homepage 4',u'http://feed43.com/6550421522527341.xml'),
         (u'Funny - The Very Best Of The Internet',u'http://feed43.com/4538510106331565.xml'),
         (u'Gaming',u'http://feed43.com/6537162612465672.xml'),
-        (u'Girls',u'http://feed43.com/3674777224513254.xml'),
+        (u'Girls',u'http://feed43.com/4574262733341068.xml'),# edit link http://feed43.com/feed.html?name=4574262733341068
                            ]
 
     extra_css = '''

recipes/film_web.recipe

@@ -1,6 +1,7 @@
 from calibre.web.feeds.news import BasicNewsRecipe
-
-class Filmweb_pl(BasicNewsRecipe):
+import re
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
+class FilmWebPl(BasicNewsRecipe):
     title          = u'FilmWeb'
     __author__        = 'fenuks'
     description   = 'FilmWeb - biggest polish movie site'
@@ -12,8 +13,9 @@ class Filmweb_pl(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets= True
     remove_empty_feeds=True
+    preprocess_regexps = [(re.compile(u'\(kliknij\,\ aby powiększyć\)', re.IGNORECASE), lambda m: ''), ]#(re.compile(ur' | ', re.IGNORECASE), lambda m: '')]
     extra_css      = '.hdrBig {font-size:22px;} ul {list-style-type:none; padding: 0; margin: 0;}'
-    remove_tags= [dict(name='div', attrs={'class':['recommendOthers']}), dict(name='ul', attrs={'class':'fontSizeSet'})]
+    remove_tags= [dict(name='div', attrs={'class':['recommendOthers']}), dict(name='ul', attrs={'class':'fontSizeSet'}), dict(attrs={'class':'userSurname anno'})]
     keep_only_tags= [dict(name='h1', attrs={'class':['hdrBig', 'hdrEntity']}), dict(name='div', attrs={'class':['newsInfo', 'newsInfoSmall', 'reviewContent description']})]
     feeds          = [(u'Wszystkie newsy', u'http://www.filmweb.pl/feed/news/latest'),
                          (u'News / Filmy w produkcji', 'http://www.filmweb.pl/feed/news/category/filminproduction'),
@@ -31,13 +33,12 @@ class Filmweb_pl(BasicNewsRecipe):
                          (u'News / Kino polskie', u'http://www.filmweb.pl/feed/news/category/polish.cinema'),
                          (u'News / Telewizja', u'http://www.filmweb.pl/feed/news/category/tv'),
                          (u'Recenzje redakcji', u'http://www.filmweb.pl/feed/reviews/latest'),
-                         (u'Recenzje użytkowników', u'http://www.filmweb.pl/feed/user-reviews/latest')]
+                         (u'Recenzje użytkowników', u'http://www.filmweb.pl/feed/user-reviews/latest')
+                          ]
 
     def skip_ad_pages(self, soup):
         skip_tag = soup.find('a', attrs={'class':'welcomeScreenButton'})
         if skip_tag is not None:
-            self.log.warn('skip_tag')
-            self.log.warn(skip_tag)
             return self.index_to_soup(skip_tag['href'], raw=True)
 
 
@@ -45,4 +46,9 @@ class Filmweb_pl(BasicNewsRecipe):
         for a in soup('a'):
             if a.has_key('href') and 'http://' not in a['href'] and 'https://' not in a['href']:
                 a['href']=self.index + a['href']
+        for i in soup.findAll('a', attrs={'class':'fn'}):
+            i.insert(len(i), BeautifulSoup('<br />'))
+        for i in soup.findAll('sup'):
+            if not i.string or i.string.startswith('(kliknij'):
+                i.extract()
         return soup

recipes/gry_online_pl.recipe

@@ -1,6 +1,6 @@
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
-class Gry_online_pl(BasicNewsRecipe):
+class GryOnlinePl(BasicNewsRecipe):
     title          = u'Gry-Online.pl'
     __author__        = 'fenuks'
     description   = 'Gry-Online.pl - computer games'
@@ -21,17 +21,18 @@ class Gry_online_pl(BasicNewsRecipe):
         tag = appendtag.find('div', attrs={'class':'n5p'})
         if tag:
             nexturls=tag.findAll('a')
-            for nexturl in nexturls[1:]:
-                try:
-                    soup2 = self.index_to_soup('http://www.gry-online.pl/S020.asp'+ nexturl['href'])
-                except:
-                    soup2 = self.index_to_soup('http://www.gry-online.pl/S022.asp'+ nexturl['href'])
+            url_part = soup.find('link', attrs={'rel':'canonical'})['href']
+            url_part = url_part[25:].rpartition('?')[0]
+            for nexturl in nexturls[1:-1]:
+                soup2 = self.index_to_soup('http://www.gry-online.pl/' + url_part + nexturl['href'])
                 pagetext = soup2.find(attrs={'class':'gc660'})
                 for r in pagetext.findAll(name='header'):
                     r.extract()
+                for r in pagetext.findAll(attrs={'itemprop':'description'}):
+                    r.extract()
                 pos = len(appendtag.contents)
                 appendtag.insert(pos, pagetext)
-            for r in appendtag.findAll(attrs={'class':['n5p', 'add-info', 'twitter-share-button']}):
+            for r in appendtag.findAll(attrs={'class':['n5p', 'add-info', 'twitter-share-button', 'lista lista3 lista-gry']}):
                 r.extract()
 
 

recipes/high_country_blogs.recipe

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>, Armin Geller'
+
+'''
+Fetch High Country News - Blogs
+'''
+from calibre.web.feeds.news import BasicNewsRecipe
+class HighCountryNewsBlogs(BasicNewsRecipe):
+
+    title                 = u'High Country News - Blogs'
+    description           = u'High Country News - Blogs (RSS Version)'
+    __author__            = 'Armin Geller' # 2012-08-01
+    publisher             = 'High Country News'
+    category              = 'news, politics, Germany'
+    timefmt               = ' [%a, %d %b %Y]'
+    language              = 'en'
+    encoding              = 'UTF-8'
+    publication_type      = 'newspaper'
+    oldest_article        = 7
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    auto_cleanup          = True
+    remove_javascript     = True
+    use_embedded_content  = False
+    masthead_url          = 'http://www.hcn.org/logo.jpg'
+    cover_source          = 'http://www.hcn.org'
+
+    def get_cover_url(self):
+       cover_source_soup = self.index_to_soup(self.cover_source)
+       preview_image_div = cover_source_soup.find(attrs={'class':' portaltype-Plone Site content--hcn template-homepage_view'})
+       return preview_image_div.div.img['src']
+
+    feeds = [
+              (u'From the Blogs', u'http://feeds.feedburner.com/hcn/FromTheBlogs?format=xml'),
+
+              (u'Heard around the West', u'http://feeds.feedburner.com/hcn/heard?format=xml'),
+              (u'The GOAT Blog', u'http://feeds.feedburner.com/hcn/goat?format=xml'),
+              (u'The Range', u'http://feeds.feedburner.com/hcn/range?format=xml'),
+             ]
+
+    def print_version(self, url):
+          return url
+

recipes/linux_journal.recipe

@@ -0,0 +1,36 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LinuxJournal(BasicNewsRecipe):
+    title          = u'Linux Journal'
+    __author__        = 'fenuks'
+    description   = u'The monthly magazine of the Linux community, promoting the use of Linux worldwide.'
+    cover_url      = 'http://www.linuxjournal.com/files/linuxjournal.com/ufiles/logo-lj.jpg'
+    category       = 'IT, Linux'
+    language       = 'en'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    use_embedded_content = False
+    remove_empty_feeds = True
+    keep_only_tags=[dict(id='content-inner')]
+    remove_tags_after= dict(attrs={'class':'user-signature clear-block'})
+    remove_tags=[dict(attrs={'class':['user-signature clear-block', 'breadcrumb',  'terms terms-inline']})]
+    feeds          = [(u'Front Page', u'http://feeds.feedburner.com/linuxjournalcom'), (u'News', u'http://feeds.feedburner.com/LinuxJournal-BreakingNews'), (u'Blogs', u'http://www.linuxjournal.com/blog/feed'), (u'Audio/Video', u'http://www.linuxjournal.com/taxonomy/term/28/0/feed'), (u'Community', u'http://www.linuxjournal.com/taxonomy/term/18/0/feed'), (u'Education', u'http://www.linuxjournal.com/taxonomy/term/25/0/feed'), (u'Embedded', u'http://www.linuxjournal.com/taxonomy/term/27/0/feed'), (u'Hardware', u'http://www.linuxjournal.com/taxonomy/term/23/0/feed'), (u'HOWTOs', u'http://www.linuxjournal.com/taxonomy/term/19/0/feed'), (u'International', u'http://www.linuxjournal.com/taxonomy/term/30/0/feed'), (u'Security', u'http://www.linuxjournal.com/taxonomy/term/31/0/feed'), (u'Software', u'http://www.linuxjournal.com/taxonomy/term/17/0/feed'), (u'Sysadmin', u'http://www.linuxjournal.com/taxonomy/term/21/0/feed'), (u'Webmaster', u'http://www.linuxjournal.com/taxonomy/term/24/0/feed')]
+
+    def append_page(self, soup, appendtag):
+        next = appendtag.find('li', attrs={'class':'pager-next'})
+        while next:    
+            nexturl = next.a['href']
+            appendtag.find('div', attrs={'class':'links'}).extract()
+            soup2 = self.index_to_soup('http://www.linuxjournal.com'+ nexturl)
+            pagetext = soup2.find(attrs={'class':'node-inner'}).find(attrs={'class':'content'})
+            next = appendtag.find('li', attrs={'class':'pager-next'})
+            pos = len(appendtag.contents)
+            appendtag.insert(pos, pagetext)
+        tag = appendtag.find('div', attrs={'class':'links'})
+        if tag:
+            tag.extract()
+
+    def preprocess_html(self, soup):
+        self.append_page(soup, soup.body)
+        return soup

recipes/list_apart.recipe

@@ -0,0 +1,33 @@
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AListApart (BasicNewsRecipe):
+   __author__ = u'Marc Busqué <marc@lamarciana.com>'
+   __url__ = 'http://www.lamarciana.com'
+   __version__ = '1.0'
+   __license__   = 'GPL v3'
+   __copyright__ = u'2012, Marc Busqué <marc@lamarciana.com>'
+   title = u'A List Apart'
+   description = u'A List Apart Magazine (ISSN: 1534-0295) explores the design, development, and meaning of web content, with a special focus on web standards and best practices.'
+   language = 'en'
+   tags = 'web development, software'
+   oldest_article = 120
+   remove_empty_feeds = True
+   no_stylesheets = True
+   encoding = 'utf8'
+   cover_url = u'http://alistapart.com/pix/alalogo.gif'
+   keep_only_tags = [
+         dict(name='div', attrs={'id': 'content'})
+         ]
+   remove_tags = [
+         dict(name='ul', attrs={'id': 'metastuff'}),
+         dict(name='div', attrs={'class': 'discuss'}),
+         dict(name='div', attrs={'class': 'discuss'}),
+         dict(name='div', attrs={'id': 'learnmore'}),
+         ]
+   remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan', 'valign', 'vspace', 'hspace', 'alt', 'width', 'height']
+   extra_css = u'img {max-width: 100%; display: block; margin: auto;} #authorbio img {float: left; margin-right: 2%;}'
+
+   feeds = [
+         (u'A List Apart', u'http://www.alistapart.com/site/rss'),
+         ]

recipes/metro_uk.recipe

@@ -1,31 +1,42 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1306097511(BasicNewsRecipe):
     title          = u'Metro UK'
-    description = 'News as provide by The Metro -UK'
+    description = 'Author Dave Asbury : News as provide by The Metro -UK'
     #timefmt = ''
     __author__ = 'Dave Asbury'
-    #last update 9/6/12
+    #last update 4/8/12
     cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg'
-    #no_stylesheets = True
+    no_stylesheets = True
     oldest_article = 1
-    max_articles_per_feed = 10
+    max_articles_per_feed = 12
     remove_empty_feeds = True
     remove_javascript     = True
-    auto_cleanup = True
+    #auto_cleanup = True
     encoding = 'UTF-8'
-
+    cover_url ='http://profile.ak.fbcdn.net/hprofile-ak-snc4/157897_117118184990145_840702264_n.jpg'
     language = 'en_GB'
     masthead_url        = 'http://e-edition.metro.co.uk/images/metro_logo.gif'
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:1.6em;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:1.2em;}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:1.0em;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:1.0em;}
+		'''
     keep_only_tags = [
-
+		#dict(name='h1'),
+		#dict(name='h2'),
+		#dict(name='div', attrs={'class' : ['row','article','img-cnt figure','clrd']})
+	#dict(name='h3'),
+	#dict(attrs={'class' :  'BText'}),
 		]
     remove_tags    = [
-
+                                dict(name='span',attrs={'class' : 'share'}),
+		dict(name='li'),
+		dict(attrs={'class' : ['twitter-share-button','header-forms','hdr-lnks','close','art-rgt','fd-gr1-b clrd google-article','news m12 clrd clr-b p5t shareBtm','item-ds csl-3-img news','c-1of3 c-last','c-1of1','pd','item-ds csl-3-img sport']}),
+           		dict(attrs={'id' : ['','sky-left','sky-right','ftr-nav','and-ftr','notificationList','logo','miniLogo','comments-news','metro_extras']})
                                ]
-
+    remove_tags_before = dict(name='h1')
+    #remove_tags_after = dict(attrs={'id':['topic-buttons']})
 
     feeds          = [
         (u'News', u'http://www.metro.co.uk/rss/news/'), (u'Money', u'http://www.metro.co.uk/rss/money/'), (u'Sport', u'http://www.metro.co.uk/rss/sport/'), (u'Film', u'http://www.metro.co.uk/rss/metrolife/film/'), (u'Music', u'http://www.metro.co.uk/rss/metrolife/music/'), (u'TV', u'http://www.metro.co.uk/rss/tv/'), (u'Showbiz', u'http://www.metro.co.uk/rss/showbiz/'), (u'Weird News', u'http://www.metro.co.uk/rss/weird/'), (u'Travel', u'http://www.metro.co.uk/rss/travel/'), (u'Lifestyle', u'http://www.metro.co.uk/rss/lifestyle/'), (u'Books', u'http://www.metro.co.uk/rss/lifestyle/books/'), (u'Food', u'http://www.metro.co.uk/rss/lifestyle/restaurants/')]
-    extra_css  = '''
-	body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
-                	 '''

recipes/natemat_pl.recipe

@@ -1,3 +1,4 @@
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class NaTemat(BasicNewsRecipe):
@@ -8,8 +9,9 @@ class NaTemat(BasicNewsRecipe):
     description   = u'informacje, komentarze, opinie'
     category       = 'news'
     language       = 'pl'
+    preprocess_regexps = [(re.compile(ur'Czytaj też\:.*?</a>', re.IGNORECASE), lambda m: ''), (re.compile(ur'Zobacz też\:.*?</a>', re.IGNORECASE), lambda m: ''), (re.compile(ur'Czytaj więcej\:.*?</a>', re.IGNORECASE), lambda m: ''), (re.compile(ur'Czytaj również\:.*?</a>', re.IGNORECASE), lambda m: '')]
     cover_url= 'http://blog.plona.pl/wp-content/uploads/2012/05/natemat.png'
     no_stylesheets = True
     keep_only_tags= [dict(id='main')]
-    remove_tags= [dict(attrs={'class':['button', 'block-inside style_default', 'article-related']})]
+    remove_tags= [dict(attrs={'class':['button', 'block-inside style_default', 'article-related', 'user-header', 'links']}), dict(name='img', attrs={'class':'indent'})]
     feeds          = [(u'Artyku\u0142y', u'http://natemat.pl/rss/wszystkie')]

recipes/phillosophy_now.recipe

@@ -0,0 +1,75 @@
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from collections import OrderedDict
+
+class PhilosophyNow(BasicNewsRecipe):
+
+    title       = 'Philosophy Now'
+    __author__  = 'Rick Shang'
+
+    description = '''Philosophy Now is a lively magazine for everyone
+    interested in ideas. It isn't afraid to tackle all the major questions of
+    life, the universe and everything. Published every two months, it tries to
+    corrupt innocent citizens by convincing them that philosophy can be
+    exciting, worthwhile and comprehensible, and also to provide some enjoyable
+    reading matter for those already ensnared by the muse, such as philosophy
+    students and academics.'''
+    language = 'en'
+    category = 'news'
+    encoding = 'UTF-8'
+
+    keep_only_tags = [dict(attrs={'id':'fullMainColumn'})]
+    remove_tags = [dict(attrs={'class':'articleTools'})]
+    no_javascript = True
+    no_stylesheets = True
+    needs_subscription = True
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        br.open('https://philosophynow.org/auth/login')
+        br.select_form(nr = 1)
+        br['username'] = self.username
+        br['password'] = self.password
+        br.submit()
+        return br
+
+    def parse_index(self):
+        #Go to the issue
+        soup0 = self.index_to_soup('http://philosophynow.org/')
+        issue = soup0.find('div',attrs={'id':'navColumn'})
+
+        #Find date & cover
+        cover = issue.find('div', attrs={'id':'cover'})
+        date = self.tag_to_string(cover.find('h3')).strip()
+        self.timefmt = u' [%s]'%date
+        img=cover.find('img',src=True)['src']
+        self.cover_url = 'http://philosophynow.org' + re.sub('medium','large',img)
+        issuenum = re.sub('/media/images/covers/medium/issue','',img)
+        issuenum = re.sub('.jpg','',issuenum)
+
+        #Go to the main body
+        current_issue_url = 'http://philosophynow.org/issues/' + issuenum
+        soup = self.index_to_soup(current_issue_url)
+        div = soup.find ('div', attrs={'class':'articlesColumn'})
+
+        feeds = OrderedDict()
+
+        for post in div.findAll('h3'):
+            articles = []
+            a=post.find('a',href=True)
+            if a is not None:
+                url="http://philosophynow.org" + a['href']
+                title=self.tag_to_string(a).strip()
+                s=post.findPrevious('h4')
+                section_title = self.tag_to_string(s).strip()
+                d=post.findNext('p')
+                desc = self.tag_to_string(d).strip()
+                articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
+
+                if articles:
+                    if section_title not in feeds:
+                        feeds[section_title] = []
+                    feeds[section_title] += articles
+        ans = [(key, val) for key, val in feeds.iteritems()]
+        return ans
+

recipes/psych.recipe

@@ -1,44 +1,79 @@
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
 
-from calibre.ptempfile import PersistentTemporaryFile
-from calibre.web.feeds.news import BasicNewsRecipe
 
-class AdvancedUserRecipe1275708473(BasicNewsRecipe):
-    title          = u'Psychology Today'
-    _author__ = 'rty'
-    publisher = u'www.psychologytoday.com'
-    category = u'Psychology'
-    max_articles_per_feed = 100
-    remove_javascript = True
-    use_embedded_content   = False
-    no_stylesheets = True
+class PsychologyToday(BasicNewsRecipe):
+
+    title       = 'Psychology Today'
+    __author__  = 'Rick Shang'
+
+    description = 'This magazine takes information from the latest research in the field of psychology and makes it useful to people in their everyday lives. Its coverage encompasses self-improvement, relationships, the mind-body connection, health, family, the workplace and culture.'
     language = 'en'
-    temp_files = []
-    articles_are_obfuscated = True
-    remove_tags = [
-                    dict(name='div', attrs={'class':['print-source_url','field-items','print-footer']}),
-                    dict(name='span', attrs={'class':'print-footnote'}),
-                  ]
-    remove_tags_before  = dict(name='h1', attrs={'class':'print-title'})
-    remove_tags_after     = dict(name='div', attrs={'class':['field-items','print-footer']})
+    category = 'news'
+    encoding = 'UTF-8'
+    keep_only_tags = [dict(attrs={'class':['print-title', 'print-submitted', 'print-content', 'print-footer', 'print-source_url', 'print-links']})]
+    no_javascript = True
+    no_stylesheets = True
 
-    feeds          = [(u'Contents', u'http://www.psychologytoday.com/articles/index.rss')]
 
-    def get_article_url(self, article):
-       return article.get('link',  None)
+    def parse_index(self):
+        articles = []
+        soup = self.index_to_soup('http://www.psychologytoday.com/magazine')
+
+
+        #Go to the main body
+        div = soup.find('div',attrs={'id':'content-content'})
+        #Find cover & date
+        cover_item = div.find('div', attrs={'class':'collections-header-image'})
+        cover = cover_item.find('img',src=True)
+        self.cover_url = cover['src']
+        date = self.tag_to_string(cover['title'])
+        self.timefmt = u' [%s]'%date
+
+        articles = []
+        for post in div.findAll('div', attrs={'class':'collections-node-feature-info'}):
+            title = self.tag_to_string(post.find('h2'))
+            author_item=post.find('div', attrs={'class':'collection-node-byline'})
+            author = re.sub(r'.*by\s',"",self.tag_to_string(author_item).strip())
+            title = title + u' (%s)'%author
+            article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
+            print_page=article_page.find('li', attrs={'class':'print_html first'})
+            url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
+            desc = self.tag_to_string(post.find('div', attrs={'class':'collection-node-description'})).strip()
+            self.log('Found article:', title)
+            self.log('\t', url)
+            self.log('\t', desc)
+            articles.append({'title':title, 'url':url, 'date':'','description':desc})
+
+        for post in div.findAll('div', attrs={'class':'collections-node-thumbnail-info'}):
+            title = self.tag_to_string(post.find('h2'))
+            author_item=post.find('div', attrs={'class':'collection-node-byline'})
+            article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
+            print_page=article_page.find('li', attrs={'class':'print_html first'})
+            description = post.find('div', attrs={'class':'collection-node-description'})
+            author = re.sub(r'.*by\s',"",self.tag_to_string(description.nextSibling).strip())
+            desc = self.tag_to_string(description).strip()
+            url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
+            title = title + u' (%s)'%author
+            self.log('Found article:', title)
+            self.log('\t', url)
+            self.log('\t', desc)
+            articles.append({'title':title, 'url':url, 'date':'','description':desc})
+
+        for post in div.findAll('li', attrs={'class':['collection-item-list-odd','collection-item-list-even']}):
+            title = self.tag_to_string(post.find('h2'))
+            author_item=post.find('div', attrs={'class':'collection-node-byline'})
+            author = re.sub(r'.*by\s',"",self.tag_to_string(author_item).strip())
+            title = title + u' (%s)'%author
+            article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
+            print_page=article_page.find('li', attrs={'class':'print_html first'})
+            url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
+            desc = self.tag_to_string(post.find('div', attrs={'class':'collection-node-description'})).strip()
+            self.log('Found article:', title)
+            self.log('\t', url)
+            self.log('\t', desc)
+            articles.append({'title':title, 'url':url, 'date':'','description':desc})
+
+        return [('Current Issue', articles)]
 
-    def get_obfuscated_article(self, url):
-        br = self.get_browser()
-        br.open(url)
-        response = br.follow_link(url_regex = r'/print/[0-9]+', nr = 0)
-        html = response.read()
-        self.temp_files.append(PersistentTemporaryFile('_fa.html'))
-        self.temp_files[-1].write(html)
-        self.temp_files[-1].close()
-        return self.temp_files[-1].name
 
-    def get_cover_url(self):
-        index = 'http://www.psychologytoday.com/magazine/'
-        soup = self.index_to_soup(index)
-        for image in soup.findAll('img',{ "class" : "imagefield imagefield-field_magazine_cover" }):
-              return image['src'] + '.jpg'
-        return None

recipes/sfbg.recipe

@@ -6,20 +6,30 @@ class SanFranciscoBayGuardian(BasicNewsRecipe):
     __author__     = 'Krittika Goyal'
     oldest_article = 31 #days
     max_articles_per_feed = 25
+    #encoding = 'latin1'
 
     no_stylesheets = True
+    #remove_tags_before = dict(name='div', attrs={'id':'story_header'})
+    #remove_tags_after  = dict(name='div', attrs={'id':'shirttail'})
     remove_tags = [
        dict(name='iframe'),
+       #dict(name='div', attrs={'class':'related-articles'}),
+        #dict(name='div', attrs={'id':['story_tools', 'toolbox', 'shirttail', 'comment_widget']}),
+       #dict(name='ul', attrs={'class':'article-tools'}),
+       #dict(name='ul', attrs={'id':'story_tabs'}),
     ]
 
 
     feeds = [
         ('sfbg', 'http://www.sfbg.com/rss.xml'),
-        ('politics', 'http://www.sfbg.com/politics/rss.xml'),
-        ('blogs', 'http://www.sfbg.com/blog/rss.xml'),
-        ('pixel_vision', 'http://www.sfbg.com/pixel_vision/rss.xml'),
-        ('bruce', 'http://www.sfbg.com/bruce/rss.xml'),
     ]
 
 
-
+    #def preprocess_html(self, soup):
+        #story = soup.find(name='div', attrs={'id':'story_body'})
+        #td = heading.findParent(name='td')
+        #td.extract()
+        #soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
+        #body = soup.find(name='body')
+        #body.insert(0, story)
+        #return soup

recipes/smashing.recipe

@@ -1,50 +1,24 @@
-#!/usr/bin/env  python
-
-__license__   = 'GPL v3'
-__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
-'''
-www.smashingmagazine.com
-'''
-
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class SmashingMagazine(BasicNewsRecipe):
-    title                 = 'Smashing Magazine'
-    __author__            = 'Darko Miletic'
-    description           = 'We smash you with the information that will make your life easier, really'
-    oldest_article        = 20
+class SmashingMagazine (BasicNewsRecipe):
+   __author__ = u'Marc Busqué <marc@lamarciana.com>'
+   __url__ = 'http://www.lamarciana.com'
+   __version__ = '1.0.1'
+   __license__   = 'GPL v3'
+   __copyright__ = u'2012, Marc Busqué <marc@lamarciana.com>'
+   title = u'Smashing Magazine'
+   description = u'Founded in September 2006, Smashing Magazine delivers useful and innovative information to Web designers and developers. Our aim is to inform our readers about the latest trends and techniques in Web development. We try to persuade you not with the quantity but with the quality of the information we present. Smashing Magazine is and always has been independent.'
    language = 'en'
-    max_articles_per_feed = 100
+   tags = 'web development, software'
+   oldest_article = 7
+   remove_empty_feeds = True
    no_stylesheets = True
-    use_embedded_content  = False
-    publisher             = 'Smashing Magazine'
-    category              = 'news, web, IT, css, javascript, html'
-    encoding              = 'utf-8'
+   encoding = 'utf8'
+   cover_url = u'http://media.smashingmagazine.com/themes/smashingv4/images/logo.png'
+   remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan', 'valign', 'vspace', 'hspace', 'alt', 'width', 'height', 'style']
+   extra_css = u'body div table:first-child {display: none;} img {max-width: 100%; display: block; margin: auto;}'
 
-    conversion_options = {
-                             'comments'    : description
-                            ,'tags'        : category
-                            ,'publisher'   : publisher
-                         }
-
-    keep_only_tags = [dict(name='div', attrs={'id':'leftcolumn'})]
-    remove_tags_after = dict(name='ul',attrs={'class':'social'})
-    remove_tags = [
-                    dict(name=['link','object'])
-                   ,dict(name='h1',attrs={'class':'logo'})
-                   ,dict(name='div',attrs={'id':'booklogosec'})
-                   ,dict(attrs={'src':'http://media2.smashingmagazine.com/wp-content/uploads/images/the-smashing-book/smbook6.gif'})
+   feeds = [
+         (u'Smashing Magazine', u'http://rss1.smashingmagazine.com/feed/'),
          ]
-
-    feeds          = [(u'Articles', u'http://rss1.smashingmagazine.com/feed/')]
-
-    def preprocess_html(self, soup):
-        for iter in soup.findAll('div',attrs={'class':'leftframe'}):
-            it = iter.find('h1')
-            if it == None:
-               iter.extract()
-        for item in soup.findAll('img'):
-            oldParent = item.parent
-            if oldParent.name == 'a':
-               oldParent.name = 'div'
-        return soup

recipes/smith.recipe

@@ -1,61 +1,67 @@
 import re
-from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from collections import OrderedDict
 
-class SmithsonianMagazine(BasicNewsRecipe):
-    title          = u'Smithsonian Magazine'
+class Smithsonian(BasicNewsRecipe):
+
+    title       = 'Smithsonian Magazine'
+    __author__  = 'Rick Shang'
+
+    description = 'This magazine chronicles the arts, environment, sciences and popular culture of the times. It is edited for modern, well-rounded individuals with diverse, general interests. With your order, you become a National Associate Member of the Smithsonian. Membership benefits include your subscription to Smithsonian magazine, a personalized membership card, discounts from the Smithsonian catalog, and more.'
     language = 'en'
-    __author__     = 'Krittika Goyal and TerminalVeracity'
-    oldest_article = 31#days
-    max_articles_per_feed = 50
-    use_embedded_content = False
-    recursions = 1
-    cover_url = 'http://sphotos.xx.fbcdn.net/hphotos-snc7/431147_10150602715983253_764313347_n.jpg'
-    match_regexps = ['&page=[2-9]$']
-    preprocess_regexps = [
-        (re.compile(r'for more of Smithsonian\'s coverage on history, science and nature.', re.DOTALL), lambda m: '')
-        ]
-    extra_css             = """
-                               h1{font-size: large; margin: .2em 0}
-                               h2{font-size: medium; margin: .2em 0}
-                               h3{font-size: medium; margin: .2em 0}
-                               #byLine{margin: .2em 0}
-                               .articleImageCaptionwide{font-style: italic}
-                               .wp-caption-text{font-style: italic}
-                               img{display: block}
-                            """
+    category = 'news'
+    encoding = 'UTF-8'
+    keep_only_tags = [dict(attrs={'id':['articleTitle', 'subHead', 'byLine', 'articleImage', 'article-text']})]
+    remove_tags = [dict(attrs={'class':['related-articles-inpage', 'viewMorePhotos']})]
+    no_javascript = True
+    no_stylesheets = True
 
+    def parse_index(self):
+        #Go to the issue
+        soup0 = self.index_to_soup('http://www.smithsonianmag.com/issue/archive/')
+        div = soup0.find('div',attrs={'id':'archives'})
+        issue = div.find('ul',attrs={'class':'clear-both'})
+        current_issue_url = issue.find('a', href=True)['href']
+        soup = self.index_to_soup(current_issue_url)
 
-    remove_stylesheets = True
-    remove_tags_after  = dict(name='div', attrs={'class':['post','articlePaginationWrapper']})
-    remove_tags = [
-       dict(name='iframe'),
-       dict(name='div', attrs={'class':['article_sidebar_border','viewMorePhotos','addtoany_share_save_container','meta','social','OUTBRAIN','related-articles-inpage']}),
-       dict(name='div', attrs={'id':['article_sidebar_border', 'most-popular_large', 'most-popular-body_large','comment_section','article-related']}),
-       dict(name='ul', attrs={'class':'cat-breadcrumb col three last'}),
-       dict(name='h4', attrs={'id':'related-topics'}),
-       dict(name='table'),
-       dict(name='a', attrs={'href':['/subArticleBottomWeb','/subArticleTopWeb','/subArticleTopMag','/subArticleBottomMag']}),
-       dict(name='a', attrs={'name':'comments_shaded'}),
-    ]
+        #Go to the main body
+        div = soup.find ('div', attrs={'id':'content-inset'})
 
+        #Find date
+        date = re.sub('.*\:\W*', "", self.tag_to_string(div.find('h2')).strip())
+        self.timefmt = u' [%s]'%date
 
-    feeds          = [
-('History and Archeology',
- 'http://feeds.feedburner.com/smithsonianmag/history-archaeology'),
-('People and Places',
- 'http://feeds.feedburner.com/smithsonianmag/people-places'),
-('Science and Nature',
- 'http://feeds.feedburner.com/smithsonianmag/science-nature'),
-('Arts and Culture',
- 'http://feeds.feedburner.com/smithsonianmag/arts-culture'),
-('Travel',
- 'http://feeds.feedburner.com/smithsonianmag/travel'),
-]
+        #Find cover
+        self.cover_url = div.find('img',src=True)['src']
+
+        feeds = OrderedDict()
+        section_title = ''
+        subsection_title = ''
+        for post in div.findAll('div', attrs={'class':['plainModule', 'departments plainModule']}):
+            articles = []
+            prefix = ''
+            h3=post.find('h3')
+            if h3 is not None:
+                section_title = self.tag_to_string(h3)
+            else:
+                subsection=post.find('p',attrs={'class':'article-cat'})
+                link=post.find('a',href=True)
+                url=link['href']+'?c=y&story=fullstory'
+                if subsection is not None:
+                    subsection_title = self.tag_to_string(subsection)
+                    prefix = (subsection_title+': ')
+                    description=self.tag_to_string(post('p', limit=2)[1]).strip()
+                else:
+                    description=self.tag_to_string(post.find('p')).strip()
+                desc=re.sub('\sBy\s.*', '', description, re.DOTALL)
+                author=re.sub('.*By\s', '', description, re.DOTALL)
+                title=prefix + self.tag_to_string(link).strip()+ u' (%s)'%author
+                articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
+
+            if articles:
+                if section_title not in feeds:
+                    feeds[section_title] = []
+                feeds[section_title] += articles
+        ans = [(key, val) for key, val in feeds.iteritems()]
+        return ans
 
-    def preprocess_html(self, soup):
-        story = soup.find(name='div', attrs={'id':'article-body'})
-        soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
-        body = soup.find(name='body')
-        body.insert(0, story)
-        return soup

recipes/sueddeutsche_mobil.recipe

@@ -0,0 +1,117 @@
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+__license__   = 'GPL v3'
+__copyright__ = '2012, Andreas Zeiser <andreas.zeiser@web.de>'
+'''
+szmobil.sueddeutsche.de/
+'''
+
+from calibre import strftime
+from calibre.web.feeds.recipes import BasicNewsRecipe
+import re
+
+class SZmobil(BasicNewsRecipe):
+    title       = u'Süddeutsche Zeitung mobil'
+    __author__  = u'Andreas Zeiser'
+    description = u'Nachrichten aus Deutschland. Zugriff auf kostenpflichtiges Abo SZ mobil.'
+    publisher              = u'Sueddeutsche Zeitung'
+    language               = u'de'
+    publication_type       = u'newspaper'
+    category               = u'news, politics, Germany'
+
+    no_stylesheets         = True
+    oldest_article         = 2
+    encoding               = 'iso-8859-1'
+    needs_subscription     = True
+    remove_empty_feeds     = True
+    delay                  = 1
+    cover_source           = 'http://www.sueddeutsche.de/verlag'
+
+    timefmt = ' [%a, %d %b, %Y]'
+
+    root_url ='http://szmobil.sueddeutsche.de/'
+    keep_only_tags = [dict(name='div', attrs={'class':'article'})]
+
+    def get_cover_url(self):
+        src = self.index_to_soup(self.cover_source)
+        image_url = src.find(attrs={'class':'preview-image'})
+        return image_url.div.img['src']
+
+    def get_browser(self):
+        browser = BasicNewsRecipe.get_browser(self)
+
+        # Login via fetching of Streiflicht -> Fill out login request
+        url = self.root_url + 'show.php?id=streif'
+        browser.open(url)
+
+        browser.select_form(nr=0) # to select the first form
+        browser['username'] = self.username
+        browser['password'] = self.password
+        browser.submit()
+
+        return browser
+
+	def parse_index(self):
+        # find all sections
+        src = self.index_to_soup('http://szmobil.sueddeutsche.de')
+        feeds = []
+        for itt in src.findAll('a',href=True):
+            if itt['href'].startswith('show.php?section'):
+                feeds.append( (itt.string[0:-2],itt['href']) )
+
+        all_articles = []
+        for feed in feeds:
+            feed_url = self.root_url + feed[1]
+            feed_title = feed[0]
+
+            self.report_progress(0, ('Fetching feed')+' %s...'%(feed_title if feed_title else feed_url))
+
+            src = self.index_to_soup(feed_url)
+            articles = []
+            shorttitles = dict()
+            for itt in src.findAll('a', href=True):
+                if itt['href'].startswith('show.php?id='):
+                    article_url = itt['href']
+                    article_id = int(re.search("id=(\d*)&etag=", itt['href']).group(1))
+
+                    # first check if link is a special article in section "Meinungsseite"
+                    if itt.find('strong')!= None:
+                        article_name = itt.strong.string
+                        article_shorttitle = itt.contents[1]
+
+                        articles.append( (article_name, article_url, article_id) )
+                        shorttitles[article_id] = article_shorttitle
+                        continue
+
+
+                    # candidate for a general article
+                    if itt.string == None:
+                        article_name = ''
+                    else:
+                        article_name = itt.string
+
+                    if (article_name[0:10] == "&nbsp;mehr"):
+                        # just another link ("mehr") to an article
+                        continue
+
+                    if itt.has_key('id'):
+                        shorttitles[article_id] = article_name
+                    else:
+                        articles.append( (article_name, article_url, article_id) )
+
+            feed_articles = []
+            for article_name, article_url, article_id in articles:
+                url = self.root_url + article_url
+                title = article_name
+                pubdate = strftime('%a, %d %b')
+                description = ''
+                if shorttitles.has_key(article_id):
+                    description = shorttitles[article_id]
+                # we do not want the flag ("Impressum")
+                if "HERAUSGEGEBEN VOM" in description:
+                    continue
+                d = dict(title=title, url=url, date=pubdate, description=description, content='')
+                feed_articles.append(d)
+            all_articles.append( (feed_title, feed_articles) )
+
+        return all_articles
+

recipes/the_new_republic.recipe

@@ -1,45 +1,64 @@
-from calibre.web.feeds.news import BasicNewsRecipe
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from collections import OrderedDict
+
+class TNR(BasicNewsRecipe):
 
-class The_New_Republic(BasicNewsRecipe):
     title       = 'The New Republic'
-    __author__ = 'cix3'
-    language = 'en'
-    description = 'Intelligent, stimulating and rigorous examination of American politics, foreign policy and culture'
-    timefmt = ' [%b %d, %Y]'
+    __author__  = 'Rick Shang'
 
-    oldest_article = 7
-    max_articles_per_feed = 100
+    description = 'The New Republic is a journal of opinion with an emphasis on politics and domestic and international affairs. It carries feature articles by staff and contributing editors. The second half of each issue is devoted to book and the arts, theater, motion pictures, music and art.'
+    language = 'en'
+    category = 'news'
+    encoding = 'UTF-8'
+    remove_tags = [dict(attrs={'class':['print-logo','print-site_name','print-hr']})]
+    no_javascript = True
     no_stylesheets = True
 
-    remove_tags = [
-            dict(name='div', attrs={'class':['print-logo', 'print-site_name', 'img-left', 'print-source_url']}),
-            dict(name='hr', attrs={'class':'print-hr'}), dict(name='img')
-            ]
 
-    feeds = [
-        ('Politics', 'http://www.tnr.com/rss/articles/Politics'),
-        ('Books and Arts', 'http://www.tnr.com/rss/articles/Books-and-Arts'),
-        ('Economy', 'http://www.tnr.com/rss/articles/Economy'),
-        ('Environment and Energy', 'http://www.tnr.com/rss/articles/Environment-%2526-Energy'),
-        ('Health Care', 'http://www.tnr.com/rss/articles/Health-Care'),
-        ('Metro Policy', 'http://www.tnr.com/rss/articles/Metro-Policy'),
-        ('World', 'http://www.tnr.com/rss/articles/World'),
-        ('Film', 'http://www.tnr.com/rss/articles/Film'),
-        ('Books', 'http://www.tnr.com/rss/articles/books'),
-        ('The Book', 'http://www.tnr.com/rss/book'),
-        ('Jonathan Chait', 'http://www.tnr.com/rss/blogs/Jonathan-Chait'),
-        ('The Plank', 'http://www.tnr.com/rss/blogs/The-Plank'),
-        ('The Treatment', 'http://www.tnr.com/rss/blogs/The-Treatment'),
-        ('The Spine', 'http://www.tnr.com/rss/blogs/The-Spine'),
-        ('The Vine', 'http://www.tnr.com/rss/blogs/The-Vine'),
-        ('The Avenue', 'http://www.tnr.com/rss/blogs/The-Avenue'),
-        ('William Galston', 'http://www.tnr.com/rss/blogs/William-Galston'),
-        ('Simon Johnson', 'http://www.tnr.com/rss/blogs/Simon-Johnson'),
-        ('Ed Kilgore', 'http://www.tnr.com/rss/blogs/Ed-Kilgore'),
-        ('Damon Linker', 'http://www.tnr.com/rss/blogs/Damon-Linker'),
-        ('John McWhorter', 'http://www.tnr.com/rss/blogs/John-McWhorter')
-            ]
+    def parse_index(self):
 
-    def print_version(self, url):
-        return url.replace('http://www.tnr.com/', 'http://www.tnr.com/print/')
+        #Go to the issue
+        soup0 = self.index_to_soup('http://www.tnr.com/magazine-issues')
+        issue = soup0.find('div',attrs={'id':'current_issue'})
 
+        #Find date
+        date = self.tag_to_string(issue.find('div',attrs={'class':'date'})).strip()
+        self.timefmt = u' [%s]'%date
+
+        #Go to the main body
+        current_issue_url = 'http://www.tnr.com' + issue.find('a', href=True)['href']
+        soup = self.index_to_soup(current_issue_url)
+        div = soup.find ('div', attrs={'class':'article_detail_body'})
+
+
+
+        #Find cover
+        self.cover_url = div.find('img',src=True)['src']
+
+        feeds = OrderedDict()
+        section_title = ''
+        subsection_title = ''
+        for post in div.findAll('p'):
+            articles = []
+            em=post.find('em')
+            b=post.find('b')
+            a=post.find('a',href=True)
+            if em is not None:
+                section_title = self.tag_to_string(em).strip()
+                subsection_title = ''
+            elif b is not None:
+                subsection_title=self.tag_to_string(b).strip()
+            elif a is not None:
+                prefix = (subsection_title+': ') if subsection_title else ''
+                url=re.sub('www.tnr.com','www.tnr.com/print', a['href'])
+                author=re.sub('.*by\s', '', self.tag_to_string(post), re.DOTALL)
+                title=prefix + self.tag_to_string(a).strip()+ u' (%s)'%author
+                articles.append({'title':title, 'url':url, 'description':'', 'date':''})
+
+            if articles:
+                if section_title not in feeds:
+                    feeds[section_title] = []
+                feeds[section_title] += articles
+        ans = [(key, val) for key, val in feeds.iteritems()]
+        return ans

recipes/the_sun.recipe

@@ -1,4 +1,4 @@
-import re, random
+import random
 
 from calibre import browser
 from calibre.web.feeds.recipes import BasicNewsRecipe
@@ -8,46 +8,44 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
     title          = u'The Sun UK'
     description = 'Articles from The Sun tabloid UK'
     __author__ = 'Dave Asbury'
-    # last updated 15/7/12
+    # last updated 25/7/12
     language = 'en_GB'
     oldest_article = 1
-    max_articles_per_feed = 15
+    max_articles_per_feed = 12
     remove_empty_feeds = True
     no_stylesheets = True
 
 
     masthead_url = 'http://www.thesun.co.uk/sol/img/global/Sun-logo.gif'
     encoding = 'UTF-8'
-
-    remove_empty_feeds = True
     remove_javascript     = True
     no_stylesheets = True
 
+
+
+    #preprocess_regexps = [
+     #   (re.compile(r'<div class="foot-copyright".*?</div>', re.IGNORECASE | re.DOTALL), lambda match: '')]
+
+
     extra_css  = '''
     body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
                 '''
-
-    preprocess_regexps = [
-        (re.compile(r'<div class="foot-copyright".*?</div>', re.IGNORECASE | re.DOTALL), lambda match: '')]
-
-
-
     keep_only_tags = [
-                               dict(name='h1'),dict(name='h2',attrs={'class' : ['large','large centered','medium centered','medium']}),dict(name='h3'),
-               dict(name='div',attrs={'class' : 'text-center'}),
-               dict(name='div',attrs={'id' : 'bodyText'})
-               # dict(name='p')
+        dict(name='div',attrs={'class' : 'intro'}),
+                                dict(name='h3'),
+        dict(name='div',attrs={'id' : 'articlebody'}),
+           #dict(attrs={'class' : ['right_col_branding','related-stories','mystery-meat-link','ltbx-container','ltbx-var ltbx-hbxpn','ltbx-var ltbx-nav-loop','ltbx-var ltbx-url']}),
+           #                dict(name='div',attrs={'class' : 'cf'}),
+          # dict(attrs={'title' : 'download flash'}),
+          #                 dict(attrs={'style' : 'padding: 5px'})
+
            ]
+    remove_tags_after = [dict(id='bodyText')]
     remove_tags=[
-           #dict(name='head'),
-           dict(attrs={'class' : ['mystery-meat-link','ltbx-container','ltbx-var ltbx-hbxpn','ltbx-var ltbx-nav-loop','ltbx-var ltbx-url']}),
-                           dict(name='div',attrs={'class' : 'cf'}),
-           dict(attrs={'title' : 'download flash'}),
-                           dict(attrs={'style' : 'padding: 5px'})
-
+                  dict(name='li'),
+                              dict(attrs={'class' : 'grid-4 right-hand-column'}),
         ]
 
-
     feeds          = [
     (u'News', u'http://www.thesun.co.uk/sol/homepage/news/rss'),
     (u'Sport', u'http://www.thesun.co.uk/sol/homepage/sport/rss'),

recipes/wnp.recipe

@@ -1,7 +1,7 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
 
-class AdvancedUserRecipe1312886443(BasicNewsRecipe):
+class WNP(BasicNewsRecipe):
     title          = u'WNP'
     cover_url= 'http://k.wnp.pl/images/wnpLogo.gif'
     __author__        = 'fenuks'
@@ -12,7 +12,7 @@ class AdvancedUserRecipe1312886443(BasicNewsRecipe):
     oldest_article = 8
     max_articles_per_feed = 100
     no_stylesheets= True
-    remove_tags=[dict(attrs={'class':'printF'})]
+    remove_tags=[dict(attrs={'class':['printF', 'border3B2 clearfix', 'articleMenu clearfix']})]
     feeds          = [(u'Wiadomości gospodarcze', u'http://www.wnp.pl/rss/serwis_rss.xml'),
                           (u'Serwis Energetyka - Gaz', u'http://www.wnp.pl/rss/serwis_rss_1.xml'),
           (u'Serwis Nafta - Chemia', u'http://www.wnp.pl/rss/serwis_rss_2.xml'),

resources/default_tweaks.py

@@ -506,16 +506,6 @@ compile_gpm_templates = True
 #   default_tweak_format = 'remember'
 default_tweak_format = None
 
-#: Enable multi-character first-letters in the tag browser
-# Some languages have letters that can be represented by multiple characters.
-# For example, Czech has a 'character' "ch" that sorts between "h" and "i".
-# If this tweak is True, then the tag browser will take these characters into
-# consideration when partitioning by first letter.
-# Examples:
-#    enable_multicharacters_in_tag_browser = True
-#    enable_multicharacters_in_tag_browser = False
-enable_multicharacters_in_tag_browser = True
-
 #: Do not preselect a completion when editing authors/tags/series/etc.
 # This means that you can make changes and press Enter and your changes will
 # not be overwritten by a matching completion. However, if you wish to use the

setup/extensions.py

@@ -140,7 +140,7 @@ extensions = [
                     ['calibre/utils/podofo/podofo.cpp'],
                     libraries=['podofo'],
                     lib_dirs=[podofo_lib],
-                    inc_dirs=[podofo_inc],
+                    inc_dirs=[podofo_inc, os.path.dirname(podofo_inc)],
                     optional=True,
                     error=podofo_error),
 
@@ -174,6 +174,20 @@ if isosx:
                 ldflags=['-framework', 'IOKit'])
             )
 
+if islinux:
+    extensions.append(Extension('libmtp',
+        [
+        'calibre/devices/mtp/unix/devices.c',
+        'calibre/devices/mtp/unix/libmtp.c'
+        ],
+        headers=[
+        'calibre/devices/mtp/unix/devices.h',
+        'calibre/devices/mtp/unix/upstream/music-players.h',
+        'calibre/devices/mtp/unix/upstream/device-flags.h',
+        ],
+        libraries=['mtp']
+    ))
+
 if isunix:
     cc = os.environ.get('CC', 'gcc')
     cxx = os.environ.get('CXX', 'g++')

setup/installer/__init__.py

@@ -28,7 +28,10 @@ def is_vm_running(name):
     pat = '/%s/'%name
     pids= [pid for pid in os.listdir('/proc') if pid.isdigit()]
     for pid in pids:
+        try:
             cmdline = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read()
+        except IOError:
+            continue # file went away
         if 'vmware-vmx' in cmdline and pat in cmdline:
             return True
     return False

setup/installer/linux/freeze2.py

@@ -32,7 +32,7 @@ binary_includes = [
                 '/usr/lib/libunrar.so',
                 '/usr/lib/libsqlite3.so.0',
                 '/usr/lib/libmng.so.1',
-                '/usr/lib/libpodofo.so.0.8.4',
+                '/usr/lib/libpodofo.so.0.9.1',
                 '/lib/libz.so.1',
                 '/usr/lib/libtiff.so.5',
                 '/lib/libbz2.so.1',

setup/installer/osx/app/main.py

@@ -243,9 +243,6 @@ class Py2App(object):
     @flush
     def get_local_dependencies(self, path_to_lib):
         for x in self.get_dependencies(path_to_lib):
-            if x.startswith('libpodofo'):
-                yield x, x
-                continue
             for y in (SW+'/lib/', '/usr/local/lib/', SW+'/qt/lib/',
                     '/opt/local/lib/',
                     SW+'/python/Python.framework/', SW+'/freetype/lib/'):
@@ -330,10 +327,6 @@ class Py2App(object):
         for f in glob.glob('src/calibre/plugins/*.so'):
             shutil.copy2(f, dest)
             self.fix_dependencies_in_lib(join(dest, basename(f)))
-            if 'podofo' in f:
-                self.change_dep('libpodofo.0.8.4.dylib',
-                self.FID+'/'+'libpodofo.0.8.4.dylib', join(dest, basename(f)))
-
 
     @flush
     def create_plist(self):
@@ -380,7 +373,7 @@ class Py2App(object):
     @flush
     def add_podofo(self):
         info('\nAdding PoDoFo')
-        pdf = join(SW, 'lib', 'libpodofo.0.8.4.dylib')
+        pdf = join(SW, 'lib', 'libpodofo.0.9.1.dylib')
         self.install_dylib(pdf)
 
     @flush

setup/installer/windows/__init__.py

@@ -37,6 +37,7 @@ class Win32(VMInstaller):
     SHUTDOWN_CMD = ['shutdown.exe', '-s', '-f', '-t', '0']
 
     def sign_msi(self):
+        print ('Signing .msi ...')
         raw = open(self.VM).read()
         vmx = re.search(r'''launch_vmware\(['"](.+?)['"]''', raw).group(1)
         subprocess.check_call(['vmrun', '-T', 'ws', '-gu', 'kovid', '-gp',

setup/installer/windows/notes.rst

@@ -322,24 +322,7 @@ cp build/podofo-*/build/src/Release/podofo.exp lib/
 cp build/podofo-*/build/podofo_config.h include/podofo/
 cp -r build/podofo-*/src/* include/podofo/
 
-You have to use >=0.8.2
-
-The following patch (against -r1269) was required to get it to compile:
-
-
-Index: src/PdfFiltersPrivate.cpp
-===================================================================
---- src/PdfFiltersPrivate.cpp   (revision 1261)
-+++ src/PdfFiltersPrivate.cpp   (working copy)
-@@ -1019,7 +1019,7 @@
- /*
-  * Prepare for input from a memory buffer.
-  */
--GLOBAL(void)
-+void
- jpeg_memory_src (j_decompress_ptr cinfo, const JOCTET * buffer, size_t bufsize)
- {
-     my_src_ptr src;
+You have to use >=0.9.1
 
 
 ImageMagick

setup/iso_639/ca.po

@@ -12,14 +12,14 @@ msgstr ""
 "Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
 "devel@lists.alioth.debian.org>\n"
 "POT-Creation-Date: 2011-11-25 14:01+0000\n"
-"PO-Revision-Date: 2012-05-03 16:09+0000\n"
-"Last-Translator: Dídac Rios <didac@niorcs.com>\n"
+"PO-Revision-Date: 2012-07-23 10:54+0000\n"
+"Last-Translator: jmontane <Unknown>\n"
 "Language-Team: Catalan <linux@softcatala.org>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2012-05-04 04:47+0000\n"
-"X-Generator: Launchpad (build 15195)\n"
+"X-Launchpad-Export-Date: 2012-07-24 04:52+0000\n"
+"X-Generator: Launchpad (build 15668)\n"
 "Language: ca\n"
 
 #. name for aaa
@@ -5612,7 +5612,7 @@ msgstr "Caixubi"
 
 #. name for csc
 msgid "Catalan Sign Language"
-msgstr "Llenguatge de signes català"
+msgstr "Llengua de signes catalana"
 
 #. name for csd
 msgid "Chiangmai Sign Language"
@@ -27348,7 +27348,7 @@ msgstr "Llenguatge de signes veneçolà"
 
 #. name for vsv
 msgid "Valencian Sign Language"
-msgstr "Llenguatge de signes valencià"
+msgstr "Llengua de signes valenciana"
 
 #. name for vto
 msgid "Vitou"

setup/iso_639/de.po

@@ -18,14 +18,14 @@ msgstr ""
 "Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
 "devel@lists.alioth.debian.org>\n"
 "POT-Creation-Date: 2011-11-25 14:01+0000\n"
-"PO-Revision-Date: 2012-06-10 11:16+0000\n"
+"PO-Revision-Date: 2012-07-29 15:29+0000\n"
 "Last-Translator: SimonFS <simonschuette@arcor.de>\n"
 "Language-Team: German <debian-l10n-german@lists.debian.org>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2012-06-11 04:46+0000\n"
-"X-Generator: Launchpad (build 15376)\n"
+"X-Launchpad-Export-Date: 2012-07-30 04:54+0000\n"
+"X-Generator: Launchpad (build 15702)\n"
 "Language: de\n"
 
 #. name for aaa
@@ -114,11 +114,11 @@ msgstr "Solong"
 
 #. name for aax
 msgid "Mandobo Atas"
-msgstr ""
+msgstr "Mandobo Atas"
 
 #. name for aaz
 msgid "Amarasi"
-msgstr ""
+msgstr "Amarasi"
 
 # auch: Abbé, Abbey oder Abi
 #. name for aba
@@ -127,7 +127,7 @@ msgstr "Abé"
 
 #. name for abb
 msgid "Bankon"
-msgstr ""
+msgstr "Bankon"
 
 #. name for abc
 msgid "Ayta; Ambala"
@@ -135,7 +135,7 @@ msgstr ""
 
 #. name for abd
 msgid "Manide"
-msgstr ""
+msgstr "Manide"
 
 #. name for abe
 msgid "Abnaki; Western"
@@ -143,11 +143,11 @@ msgstr "Abnaki; Westlich"
 
 #. name for abf
 msgid "Abai Sungai"
-msgstr ""
+msgstr "Abai Sungai"
 
 #. name for abg
 msgid "Abaga"
-msgstr ""
+msgstr "Abaga"
 
 #. name for abh
 msgid "Arabic; Tajiki"
@@ -171,7 +171,7 @@ msgstr ""
 
 #. name for abm
 msgid "Abanyom"
-msgstr ""
+msgstr "Abanyom"
 
 #. name for abn
 msgid "Abua"
@@ -219,23 +219,23 @@ msgstr ""
 
 #. name for aby
 msgid "Aneme Wake"
-msgstr ""
+msgstr "Aneme Wake"
 
 #. name for abz
 msgid "Abui"
-msgstr ""
+msgstr "Abui"
 
 #. name for aca
 msgid "Achagua"
-msgstr ""
+msgstr "Achagua"
 
 #. name for acb
 msgid "Áncá"
-msgstr ""
+msgstr "Áncá"
 
 #. name for acd
 msgid "Gikyode"
-msgstr ""
+msgstr "Gikyode"
 
 #. name for ace
 msgid "Achinese"
@@ -267,7 +267,7 @@ msgstr ""
 
 #. name for acn
 msgid "Achang"
-msgstr ""
+msgstr "Achang"
 
 #. name for acp
 msgid "Acipa; Eastern"
@@ -7064,7 +7064,7 @@ msgstr ""
 
 #. name for egy
 msgid "Egyptian (Ancient)"
-msgstr "Ägyptisch"
+msgstr "Altägyptisch"
 
 #. name for ehu
 msgid "Ehueun"
@@ -9241,7 +9241,7 @@ msgstr ""
 
 #. name for hbo
 msgid "Hebrew; Ancient"
-msgstr ""
+msgstr "Althebräisch"
 
 #. name for hbs
 msgid "Serbo-Croatian"
@@ -28694,7 +28694,7 @@ msgstr ""
 
 #. name for xlg
 msgid "Ligurian (Ancient)"
-msgstr ""
+msgstr "Ligurisch"
 
 #. name for xli
 msgid "Liburnian"
@@ -28762,7 +28762,7 @@ msgstr ""
 
 #. name for xmk
 msgid "Macedonian; Ancient"
-msgstr ""
+msgstr "Altmazedonisch"
 
 #. name for xml
 msgid "Malaysian Sign Language"
@@ -28826,7 +28826,7 @@ msgstr ""
 
 #. name for xna
 msgid "North Arabian; Ancient"
-msgstr ""
+msgstr "Alt-Nordarabisch"
 
 #. name for xnb
 msgid "Kanakanabu"

setup/translations.py

@@ -152,7 +152,7 @@ class Translations(POT): # {{{
                     subprocess.check_call(['msgfmt', '-o', dest, iso639])
             elif locale not in ('en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc',
                     'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku',
-                    'fr_CA'):
+                    'fr_CA', 'him'):
                 self.warn('No ISO 639 translations for locale:', locale)
 
         self.write_stats()

src/calibre/__init__.py

@@ -201,7 +201,8 @@ def prints(*args, **kwargs):
         try:
             file.write(arg)
         except:
-            file.write(repr(arg))
+            import repr as reprlib
+            file.write(reprlib.repr(arg))
         if i != len(args)-1:
             file.write(bytes(sep))
     file.write(bytes(end))

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 60)
+numeric_version = (0, 8, 63)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 
@@ -93,6 +93,8 @@ class Plugins(collections.Mapping):
             plugins.append('winutil')
         if isosx:
             plugins.append('usbobserver')
+        if islinux:
+            plugins.append('libmtp')
         self.plugins = frozenset(plugins)
 
     def load_plugin(self, name):

src/calibre/customize/builtins.py

@@ -673,7 +673,7 @@ from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 from calibre.devices.kobo.driver import KOBO
 from calibre.devices.bambook.driver import BAMBOOK
 from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX
-
+from calibre.devices.smart_device_app.driver import SMART_DEVICE_APP
 
 
 # Order here matters. The first matched device is the one used.
@@ -746,6 +746,7 @@ plugins += [
     ITUNES,
     BOEYE_BEX,
     BOEYE_BDX,
+    SMART_DEVICE_APP,
     USER_DEFINED,
 ]
 # }}}

src/calibre/customize/conversion.py

@@ -91,6 +91,37 @@ class DummyReporter(object):
     def __call__(self, percent, msg=''):
         pass
 
+def gui_configuration_widget(name, parent, get_option_by_name,
+        get_option_help, db, book_id, for_output=True):
+    import importlib
+
+    def widget_factory(cls):
+        return cls(parent, get_option_by_name,
+            get_option_help, db, book_id)
+
+    if for_output:
+        try:
+            output_widget = importlib.import_module(
+                    'calibre.gui2.convert.'+name)
+            pw = output_widget.PluginWidget
+            pw.ICON = I('back.png')
+            pw.HELP = _('Options specific to the output format.')
+            return widget_factory(pw)
+        except ImportError:
+            pass
+    else:
+        try:
+            input_widget = importlib.import_module(
+                    'calibre.gui2.convert.'+name)
+            pw = input_widget.PluginWidget
+            pw.ICON = I('forward.png')
+            pw.HELP = _('Options specific to the input format.')
+            return widget_factory(pw)
+        except ImportError:
+            pass
+    return None
+
+
 class InputFormatPlugin(Plugin):
     '''
     InputFormatPlugins are responsible for converting a document into
@@ -225,6 +256,17 @@ class InputFormatPlugin(Plugin):
         '''
         pass
 
+    def gui_configuration_widget(self, parent, get_option_by_name,
+            get_option_help, db, book_id=None):
+        '''
+        Called to create the widget used for configuring this plugin in the
+        calibre GUI. The widget must be an instance of the PluginWidget class.
+        See the builting input plugins for examples.
+        '''
+        name = self.name.lower().replace(' ', '_')
+        return gui_configuration_widget(name, parent, get_option_by_name,
+                get_option_help, db, book_id, for_output=False)
+
 
 class OutputFormatPlugin(Plugin):
     '''
@@ -308,4 +350,16 @@ class OutputFormatPlugin(Plugin):
         '''
         pass
 
+    def gui_configuration_widget(self, parent, get_option_by_name,
+            get_option_help, db, book_id=None):
+        '''
+        Called to create the widget used for configuring this plugin in the
+        calibre GUI. The widget must be an instance of the PluginWidget class.
+        See the builtin output plugins for examples.
+        '''
+        name = self.name.lower().replace(' ', '_')
+        return gui_configuration_widget(name, parent, get_option_by_name,
+                get_option_help, db, book_id, for_output=True)
+
+
 

src/calibre/devices/android/driver.py

@@ -10,7 +10,7 @@ import cStringIO
 
 from calibre.devices.usbms.driver import USBMS
 
-HTC_BCDS = [0x100, 0x0222, 0x0226, 0x227, 0x228, 0x229]
+HTC_BCDS = [0x100, 0x0222, 0x0226, 0x227, 0x228, 0x229, 0x9999]
 
 class ANDROID(USBMS):
 
@@ -41,9 +41,10 @@ class ANDROID(USBMS):
                        0xca9  : HTC_BCDS,
                        0xcac  : HTC_BCDS,
                        0xccf  : HTC_BCDS,
+                       0xcd6  : HTC_BCDS,
                        0xce5  : HTC_BCDS,
                        0x2910 : HTC_BCDS,
-                       0xff9  : HTC_BCDS + [0x9999],
+                       0xff9  : HTC_BCDS,
             },
 
             # Eken
@@ -194,7 +195,7 @@ class ANDROID(USBMS):
             'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON',
             'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP',
             'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD',
-            'PMP5097C', 'MASS', 'NOVO7']
+            'PMP5097C', 'MASS', 'NOVO7', 'ZEKI']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -212,7 +213,7 @@ class ANDROID(USBMS):
             'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0_DRIVER',
             'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX',
             'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE',
-            'ADVANCED']
+            'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
@@ -221,7 +222,8 @@ class ANDROID(USBMS):
             'A1-07___C0541A4F', 'XT912', 'MB855', 'XT910', 'BOOK_A10_CARD',
             'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC',
             'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0_DRIVER', 'XT875',
-            'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD']
+            'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727',
+            'USB_FLASH_DRIVER', 'ANDROID']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/devices/apple/driver.py

@@ -5,7 +5,7 @@ __copyright__ = '2010, Gregory Riker'
 __docformat__ = 'restructuredtext en'
 
 
-import cStringIO, ctypes, datetime, os, re, shutil, sys, tempfile, time
+import cStringIO, ctypes, datetime, os, platform, re, shutil, sys, tempfile, time
 
 from calibre.constants import __appname__, __version__, DEBUG
 from calibre import fit_image, confirm_config_name, strftime as _strftime
@@ -2427,8 +2427,9 @@ class ITUNES(DriverBase):
 
             if DEBUG:
                 logger().info("  %s %s" % (__appname__, __version__))
-                logger().info("  [OSX %s - %s (%s), driver version %d.%d.%d]" %
-                 (self.iTunes.name(), self.iTunes.version(), self.initial_status,
+                logger().info("  [OSX %s, %s %s (%s), driver version %d.%d.%d]" %
+                 (platform.mac_ver()[0],
+                  self.iTunes.name(), self.iTunes.version(), self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
                 logger().info("  communicating with iTunes via %s %s using %s binding" % (as_name, as_version, as_binding))
                 logger().info("  calibre_library_path: %s" % self.calibre_library_path)

src/calibre/devices/eb600/driver.py

@@ -101,7 +101,7 @@ class POCKETBOOK360(EB600):
 
     VENDOR_NAME = ['PHILIPS', '__POCKET', 'POCKETBO']
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['MASS_STORGE', 'BOOK_USB_STORAGE',
-            'OK_POCKET_611_61']
+            'OK_POCKET_611_61', 'OK_POCKET_360+61']
 
     OSX_MAIN_MEM = OSX_CARD_A_MEM = 'Philips Mass Storge Media'
     OSX_MAIN_MEM_VOL_PAT = re.compile(r'/Pocket')

src/calibre/devices/errors.py

@@ -48,6 +48,19 @@ class OpenFeedback(DeviceError):
         '''
         raise NotImplementedError
 
+class InitialConnectionError(OpenFeedback):
+    """ Errors detected during connection after detection but before open, for
+    e.g. in the is_connected() method. """
+
+class OpenFailed(ProtocolError):
+    """ Raised when device cannot be opened this time. No retry is to be done.
+        The device should continue to be polled for future opens. If the
+        message is empty, no exception trace is produced. """
+
+    def __init__(self, msg):
+        ProtocolError.__init__(self, msg)
+        self.show_me = bool(msg and msg.strip())
+
 class DeviceBusy(ProtocolError):
     """ Raised when device is busy """
     def __init__(self, uerr=""):

src/calibre/devices/interface.py

@@ -15,6 +15,8 @@ class DevicePlugin(Plugin):
 
     #: Ordered list of supported formats
     FORMATS     = ["lrf", "rtf", "pdf", "txt"]
+    # If True, the config dialog will not show the formats box
+    HIDE_FORMATS_CONFIG_BOX = False
 
     #: VENDOR_ID can be either an integer, a list of integers or a dictionary
     #: If it is a dictionary, it must be a dictionary of dictionaries,
@@ -197,7 +199,7 @@ class DevicePlugin(Plugin):
     # }}}
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
-            detected_device=None) :
+            detected_device=None):
         """
         :param key: The key to unlock the device
         :param log_packets: If true the packet stream to/from the device is logged
@@ -295,7 +297,7 @@ class DevicePlugin(Plugin):
 
         :return: (device name, device version, software version on device, mime type)
                  The tuple can optionally have a fifth element, which is a
-                 drive information diction. See usbms.driver for an example.
+                 drive information dictionary. See usbms.driver for an example.
 
         """
         raise NotImplementedError()
@@ -496,6 +498,92 @@ class DevicePlugin(Plugin):
         '''
         return paths
 
+    def startup(self):
+        '''
+        Called when calibre is is starting the device. Do any initialization
+        required. Note that multiple instances of the class can be instantiated,
+        and thus __init__ can be called multiple times, but only one instance
+        will have this method called.
+        '''
+        pass
+
+    def shutdown(self):
+        '''
+        Called when calibre is shutting down, either for good or in preparation
+        to restart. Do any cleanup required.
+        '''
+        pass
+
+    # Dynamic control interface.
+    # The following methods are probably called on the GUI thread. Any driver
+    # that implements these methods must take pains to be thread safe, because
+    # the device_manager might be using the driver at the same time that one of
+    # these methods is called.
+
+    def is_dynamically_controllable(self):
+        '''
+        Called by the device manager when starting plugins. If this method returns
+        a string, then a) it supports the device manager's dynamic control
+        interface, and b) that name is to be used when talking to the plugin.
+
+        This method can be called on the GUI thread. A driver that implements
+        this method must be thread safe.
+        '''
+        return None
+
+    def start_plugin(self):
+        '''
+        This method is called to start the plugin. The plugin should begin
+        to accept device connections however it does that. If the plugin is
+        already accepting connections, then do nothing.
+
+        This method can be called on the GUI thread. A driver that implements
+        this method must be thread safe.
+        '''
+        pass
+
+    def stop_plugin(self):
+        '''
+        This method is called to stop the plugin. The plugin should no longer
+        accept connections, and should cleanup behind itself. It is likely that
+        this method should call shutdown. If the plugin is already not accepting
+        connections, then do nothing.
+
+        This method can be called on the GUI thread. A driver that implements
+        this method must be thread safe.
+        '''
+        pass
+
+    def get_option(self, opt_string, default=None):
+        '''
+        Return the value of the option indicated by opt_string. This method can
+        be called when the plugin is not started. Return None if the option does
+        not exist.
+
+        This method can be called on the GUI thread. A driver that implements
+        this method must be thread safe.
+        '''
+        return default
+
+    def set_option(self, opt_string, opt_value):
+        '''
+        Set the value of the option indicated by opt_string. This method can
+        be called when the plugin is not started.
+
+        This method can be called on the GUI thread. A driver that implements
+        this method must be thread safe.
+        '''
+        pass
+
+    def is_running(self):
+        '''
+        Return True if the plugin is started, otherwise false
+
+        This method can be called on the GUI thread. A driver that implements
+        this method must be thread safe.
+        '''
+        return False
+
 class BookList(list):
     '''
     A list of books. Each Book object must have the fields
@@ -519,7 +607,7 @@ class BookList(list):
         pass
 
     def supports_collections(self):
-        ''' Return True if the the device supports collections for this book list. '''
+        ''' Return True if the device supports collections for this book list. '''
         raise NotImplementedError()
 
     def add_book(self, book, replace_metadata):

src/calibre/devices/kindle/driver.py

@@ -13,7 +13,6 @@ import datetime, os, re, sys, json, hashlib
 from calibre.devices.kindle.bookmark import Bookmark
 from calibre.devices.usbms.driver import USBMS
 from calibre import strftime
-from calibre.utils.logging import default_log
 
 '''
 Notes on collections:
@@ -389,6 +388,7 @@ class KINDLE2(KINDLE):
         self.upload_apnx(path, filename, metadata, filepath)
 
     def upload_kindle_thumbnail(self, metadata, filepath):
+        from calibre.utils.logging import default_log
         coverdata = getattr(metadata, 'thumbnail', None)
         if not coverdata or not coverdata[2]:
             return

src/calibre/devices/kobo/driver.py

@@ -461,7 +461,7 @@ class KOBO(USBMS):
         self.report_progress(1.0, _('Removing books from device...'))
 
     def remove_books_from_metadata(self, paths, booklists):
-        if self.modify_datbase_check("remove_books_from_metatata") == False:
+        if self.modify_database_check("remove_books_from_metatata") == False:
             return
 
         for i, path in enumerate(paths):

src/calibre/devices/mtp/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/devices/mtp/base.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.devices.interface import DevicePlugin
+
+class MTPDeviceBase(DevicePlugin):
+    name = 'SmartDevice App Interface'
+    gui_name = _('MTP Device')
+    icon = I('devices/galaxy_s3.png')
+    description = _('Communicate with MTP devices')
+    author = 'Kovid Goyal'
+    version = (1, 0, 0)
+
+    # Invalid USB vendor information so the scanner will never match
+    VENDOR_ID                   = [0xffff]
+    PRODUCT_ID                  = [0xffff]
+    BCD                         = [0xffff]
+
+    THUMBNAIL_HEIGHT = 128
+    CAN_SET_METADATA = []
+
+    BACKLOADING_ERROR_MESSAGE = None
+
+    def __init__(self, *args, **kwargs):
+        DevicePlugin.__init__(self, *args, **kwargs)
+        self.progress_reporter = None
+
+    def reset(self, key='-1', log_packets=False, report_progress=None,
+            detected_device=None):
+        pass
+
+    def set_progress_reporter(self, report_progress):
+        self.progress_reporter = report_progress
+

src/calibre/devices/mtp/unix/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+'''
+libmtp based drivers for MTP devices on Unix like platforms.
+'''
+

src/calibre/devices/mtp/unix/detect.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.constants import plugins
+
+class MTPDetect(object):
+
+    def __init__(self):
+        p = plugins['libmtp']
+        self.libmtp = p[0]
+        if self.libmtp is None:
+            print ('Failed to load libmtp, MTP device detection disabled')
+            print (p[1])
+        self.cache = {}
+
+    def __call__(self, devices):
+        '''
+        Given a list of devices as returned by LinuxScanner, return the set of
+        devices that are likely to be MTP devices. This class maintains a cache
+        to minimize USB polling. Note that detection is partially based on a
+        list of known vendor and product ids. This is because polling some
+        older devices causes problems. Therefore, if this method identifies a
+        device as MTP, it is not actually guaranteed that it will be a working
+        MTP device.
+        '''
+        # First drop devices that have been disconnected from the cache
+        connected_devices = {(d.busnum, d.devnum, d.vendor_id, d.product_id,
+            d.bcd, d.serial) for d in devices}
+        for d in tuple(self.cache.iterkeys()):
+            if d not in connected_devices:
+                del self.cache[d]
+
+        # Since is_mtp_device() can cause USB traffic by probing the device, we
+        # cache its result
+        mtp_devices = set()
+        if self.libmtp is None:
+            return mtp_devices
+
+        for d in devices:
+            ans = self.cache.get((d.busnum, d.devnum, d.vendor_id, d.product_id,
+                    d.bcd, d.serial), None)
+            if ans is None:
+                ans = self.libmtp.is_mtp_device(d.busnum, d.devnum,
+                        d.vendor_id, d.product_id)
+                self.cache[(d.busnum, d.devnum, d.vendor_id, d.product_id,
+                    d.bcd, d.serial)] = ans
+            if ans:
+                mtp_devices.add(d)
+        return mtp_devices
+
+    def create_device(self, connected_device):
+        d = connected_device
+        return self.libmtp.Device(d.busnum, d.devnum, d.vendor_id,
+                d.product_id, d.manufacturer, d.product, d.serial)
+
+if __name__ == '__main__':
+    from calibre.devices.scanner import linux_scanner
+    mtp_detect = MTPDetect()
+    devs = mtp_detect(linux_scanner())
+    print ('Found %d MTP devices:'%len(devs))
+    for dev in devs:
+        print (dev, 'at busnum=%d and devnum=%d'%(dev.busnum, dev.devnum))
+        print()
+
+

src/calibre/devices/mtp/unix/devices.c

@@ -0,0 +1,16 @@
+/*
+ * devices.c
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the MIT license.
+ */
+
+#include "upstream/device-flags.h"
+#include "devices.h"
+
+const calibre_device_entry_t calibre_mtp_device_table[] = {
+#include "upstream/music-players.h"
+
+  , { NULL, 0xffff, NULL, 0xffff, DEVICE_FLAG_NONE }
+};
+

src/calibre/devices/mtp/unix/devices.h

@@ -0,0 +1,22 @@
+#pragma once
+/*
+ * devices.h
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the MIT license.
+ */
+#include <stdint.h>
+#include <stddef.h>
+
+struct calibre_device_entry_struct {
+  char *vendor; /**< The vendor of this device */
+  uint16_t vendor_id; /**< Vendor ID for this device */
+  char *product; /**< The product name of this device */
+  uint16_t product_id; /**< Product ID for this device */
+  uint32_t device_flags; /**< Bugs, device specifics etc */
+};
+
+typedef struct calibre_device_entry_struct calibre_device_entry_t;
+
+extern const calibre_device_entry_t calibre_mtp_device_table[];
+

src/calibre/devices/mtp/unix/driver.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import time, operator
+from threading import RLock
+from functools import wraps
+
+from calibre.devices.errors import OpenFailed
+from calibre.devices.mtp.base import MTPDeviceBase
+from calibre.devices.mtp.unix.detect import MTPDetect
+
+def synchronous(func):
+    @wraps(func)
+    def synchronizer(self, *args, **kwargs):
+        with self.lock:
+            return func(self, *args, **kwargs)
+    return synchronizer
+
+class MTP_DEVICE(MTPDeviceBase):
+
+    supported_platforms = ['linux']
+
+    def __init__(self, *args, **kwargs):
+        MTPDeviceBase.__init__(self, *args, **kwargs)
+        self.detect = MTPDetect()
+        self.dev = None
+        self.lock = RLock()
+        self.blacklisted_devices = set()
+
+    def report_progress(self, sent, total):
+        try:
+            p = int(sent/total * 100)
+        except ZeroDivisionError:
+            p = 100
+        if self.progress_reporter is not None:
+            self.progress_reporter(p)
+
+    @synchronous
+    def get_gui_name(self):
+        if self.dev is None or not self.dev.friendly_name: return self.name
+        return self.dev.friendly_name
+
+    @synchronous
+    def is_usb_connected(self, devices_on_system, debug=False,
+            only_presence=False):
+
+        # First remove blacklisted devices.
+        devs = []
+        for d in devices_on_system:
+            if (d.busnum, d.devnum, d.vendor_id,
+                d.product_id, d.bcd, d.serial) not in self.blacklisted_devices:
+                devs.append(d)
+
+        devs = self.detect(devs)
+        if self.dev is not None:
+            # Check if the currently opened device is still connected
+            ids = self.dev.ids
+            found = False
+            for d in devs:
+                if ( (d.busnum, d.devnum, d.vendor_id, d.product_id, d.serial)
+                        == ids ):
+                    found = True
+                    break
+            return found
+        # Check if any MTP capable device is present
+        return len(devs) > 0
+
+    @synchronous
+    def post_yank_cleanup(self):
+        self.dev = None
+
+    @synchronous
+    def shutdown(self):
+        self.dev = None
+
+    @synchronous
+    def open(self, connected_device, library_uuid):
+        def blacklist_device():
+            d = connected_device
+            self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
+                    d.product_id, d.bcd, d.serial))
+        try:
+            self.dev = self.detect.create_device(connected_device)
+        except ValueError:
+            # Give the device some time to settle
+            time.sleep(2)
+            try:
+                self.dev = self.detect.create_device(connected_device)
+            except ValueError:
+                # Black list this device so that it is ignored for the
+                # remainder of this session.
+                blacklist_device()
+                raise OpenFailed('%s is not a MTP device'%(connected_device,))
+        except TypeError:
+            blacklist_device()
+            raise OpenFailed('')
+
+        storage = sorted(self.dev.storage_info, key=operator.itemgetter('id'))
+        if not storage:
+            blacklist_device()
+            raise OpenFailed('No storage found for device %s'%(connected_device,))
+        self._main_id = storage[0]['id']
+        self._carda_id = self._cardb_id = None
+        if len(storage) > 1:
+            self._carda_id = storage[1]['id']
+        if len(storage) > 2:
+            self._cardb_id = storage[2]['id']
+
+    @synchronous
+    def get_device_information(self, end_session=True):
+        d = self.dev
+        return (d.friendly_name, d.device_version, d.device_version, '')
+
+    @synchronous
+    def card_prefix(self, end_session=True):
+        ans = [None, None]
+        if self._carda_id is not None:
+            ans[0] = 'mtp:%d:'%self._carda_id
+        if self._cardb_id is not None:
+            ans[1] = 'mtp:%d:'%self._cardb_id
+        return tuple(ans)
+
+    @synchronous
+    def total_space(self, end_session=True):
+        ans = [0, 0, 0]
+        for s in self.dev.storage_info:
+            i = {self._main_id:0, self._carda_id:1,
+                    self._cardb_id:2}.get(s['id'], None)
+            if i is not None:
+                ans[i] = s['capacity']
+        return tuple(ans)
+
+    @synchronous
+    def free_space(self, end_session=True):
+        self.dev.update_storage_info()
+        ans = [0, 0, 0]
+        for s in self.dev.storage_info:
+            i = {self._main_id:0, self._carda_id:1,
+                    self._cardb_id:2}.get(s['id'], None)
+            if i is not None:
+                ans[i] = s['freespace_bytes']
+        return tuple(ans)
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+    dev = MTP_DEVICE(None)
+    from calibre.devices.scanner import linux_scanner
+    devs = linux_scanner()
+    mtp_devs = dev.detect(devs)
+    dev.open(list(mtp_devs)[0], 'xxx')
+    d = dev.dev
+    print ("Opened device:", dev.get_gui_name())
+    print ("Storage info:")
+    pprint(d.storage_info)
+    print("Free space:", dev.free_space())
+    files, errs = d.get_filelist(dev)
+    pprint((len(files), errs))
+    folders, errs = d.get_folderlist()
+    pprint((len(folders), errs))
+

src/calibre/devices/mtp/unix/libmtp.c

@@ -0,0 +1,559 @@
+#define UNICODE
+#include <Python.h>
+
+#include <stdlib.h>
+#include <libmtp.h>
+
+#include "devices.h"
+
+// Macros and utilities
+#define ENSURE_DEV(rval) \
+    if (self->device == NULL) { \
+        PyErr_SetString(PyExc_ValueError, "This device has not been initialized."); \
+        return rval; \
+    }
+
+#define ENSURE_STORAGE(rval) \
+    if (self->device->storage == NULL) { \
+        PyErr_SetString(PyExc_RuntimeError, "The device has no storage information."); \
+        return rval; \
+    }
+
+// Storage types
+#define ST_Undefined            0x0000
+#define ST_FixedROM             0x0001
+#define ST_RemovableROM         0x0002
+#define ST_FixedRAM             0x0003
+#define ST_RemovableRAM         0x0004
+
+// Storage Access capability
+#define AC_ReadWrite            0x0000
+#define AC_ReadOnly             0x0001
+#define AC_ReadOnly_with_Object_Deletion    0x0002
+
+typedef struct {
+    PyObject *obj;
+    PyThreadState *state;
+} ProgressCallback;
+
+static int report_progress(uint64_t const sent, uint64_t const total, void const *const data) {
+    PyObject *res;
+    ProgressCallback *cb;
+
+    cb = (ProgressCallback *)data;
+    if (cb->obj != NULL) {
+        PyEval_RestoreThread(cb->state);
+        res = PyObject_CallMethod(cb->obj, "report_progress", "KK", sent, total);
+        Py_XDECREF(res);
+        cb->state = PyEval_SaveThread();
+    }
+    return 0;
+}
+
+static void dump_errorstack(LIBMTP_mtpdevice_t *dev, PyObject *list) {
+    LIBMTP_error_t *stack;
+    PyObject *err;
+
+    for(stack = LIBMTP_Get_Errorstack(dev); stack != NULL; stack=stack->next) {
+        err = Py_BuildValue("Is", stack->errornumber, stack->error_text);
+        if (err == NULL) break;
+        PyList_Append(list, err);
+        Py_DECREF(err);
+    }
+
+    LIBMTP_Clear_Errorstack(dev);
+}
+
+// }}}
+
+// Device object definition {{{
+typedef struct {
+    PyObject_HEAD
+    // Type-specific fields go here.
+    LIBMTP_mtpdevice_t *device;
+    PyObject *ids;
+    PyObject *friendly_name;
+    PyObject *manufacturer_name;
+    PyObject *model_name;
+    PyObject *serial_number;
+    PyObject *device_version;
+
+} libmtp_Device;
+
+// Device.__init__() {{{
+static void
+libmtp_Device_dealloc(libmtp_Device* self)
+{
+    if (self->device != NULL) LIBMTP_Release_Device(self->device);
+    self->device = NULL;
+
+    Py_XDECREF(self->ids); self->ids = NULL;
+    Py_XDECREF(self->friendly_name); self->friendly_name = NULL;
+    Py_XDECREF(self->manufacturer_name); self->manufacturer_name = NULL;
+    Py_XDECREF(self->model_name); self->model_name = NULL;
+    Py_XDECREF(self->serial_number); self->serial_number = NULL;
+    Py_XDECREF(self->device_version); self->device_version = NULL;
+
+    self->ob_type->tp_free((PyObject*)self);
+}
+
+static int
+libmtp_Device_init(libmtp_Device *self, PyObject *args, PyObject *kwds)
+{
+    int busnum, devnum, vendor_id, product_id;
+    PyObject *usb_serialnum;
+    char *vendor, *product, *friendly_name, *manufacturer_name, *model_name, *serial_number, *device_version;
+    LIBMTP_raw_device_t rawdev;
+    LIBMTP_mtpdevice_t *dev;
+    size_t i;
+
+    if (!PyArg_ParseTuple(args, "iiiissO", &busnum, &devnum, &vendor_id, &product_id, &vendor, &product, &usb_serialnum)) return -1;
+
+    if (devnum < 0 || devnum > 255 || busnum < 0) { PyErr_SetString(PyExc_TypeError, "Invalid busnum/devnum"); return -1; }
+
+    self->ids = Py_BuildValue("iiiiO", busnum, devnum, vendor_id, product_id, usb_serialnum);
+    if (self->ids == NULL) return -1;
+
+    rawdev.bus_location = (uint32_t)busnum;
+    rawdev.devnum = (uint8_t)devnum;
+    rawdev.device_entry.vendor = vendor;
+    rawdev.device_entry.product = product;
+    rawdev.device_entry.vendor_id = vendor_id;
+    rawdev.device_entry.product_id = product_id;
+    rawdev.device_entry.device_flags = 0x00000000U;
+
+    Py_BEGIN_ALLOW_THREADS;
+    for (i = 0; ; i++) {
+        if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
+        if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == product_id) {
+            rawdev.device_entry.device_flags = calibre_mtp_device_table[i].device_flags;
+        }
+    }
+
+    // Note that contrary to what the libmtp docs imply, we cannot use
+    // LIBMTP_Open_Raw_Device_Uncached as using it causes file listing to fail
+    dev = LIBMTP_Open_Raw_Device(&rawdev);
+    Py_END_ALLOW_THREADS;
+
+    if (dev == NULL) { 
+        PyErr_SetString(PyExc_ValueError, "Unable to open raw device."); 
+        return -1;
+    }
+
+    self->device = dev;
+
+    Py_BEGIN_ALLOW_THREADS;
+    friendly_name = LIBMTP_Get_Friendlyname(self->device);
+    manufacturer_name = LIBMTP_Get_Manufacturername(self->device);
+    model_name = LIBMTP_Get_Modelname(self->device);
+    serial_number = LIBMTP_Get_Serialnumber(self->device);
+    device_version = LIBMTP_Get_Deviceversion(self->device);
+    Py_END_ALLOW_THREADS;
+
+    if (friendly_name != NULL) {
+        self->friendly_name = PyUnicode_FromString(friendly_name);
+        free(friendly_name);
+    }
+    if (self->friendly_name == NULL) { self->friendly_name = Py_None; Py_INCREF(Py_None); }
+
+    if (manufacturer_name != NULL) {
+        self->manufacturer_name = PyUnicode_FromString(manufacturer_name);
+        free(manufacturer_name);
+    }
+    if (self->manufacturer_name == NULL) { self->manufacturer_name = Py_None; Py_INCREF(Py_None); }
+
+    if (model_name != NULL) {
+        self->model_name = PyUnicode_FromString(model_name);
+        free(model_name);
+    }
+    if (self->model_name == NULL) { self->model_name = Py_None; Py_INCREF(Py_None); }
+
+    if (serial_number != NULL) {
+        self->serial_number = PyUnicode_FromString(serial_number);
+        free(serial_number);
+    }
+    if (self->serial_number == NULL) { self->serial_number = Py_None; Py_INCREF(Py_None); }
+
+    if (device_version != NULL) {
+        self->device_version = PyUnicode_FromString(device_version);
+        free(device_version);
+    }
+    if (self->device_version == NULL) { self->device_version = Py_None; Py_INCREF(Py_None); }
+
+    return 0;
+}
+// }}}
+
+// Device.friendly_name {{{
+static PyObject *
+libmtp_Device_friendly_name(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->friendly_name); return self->friendly_name;
+} // }}}
+
+// Device.manufacturer_name {{{
+static PyObject *
+libmtp_Device_manufacturer_name(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->manufacturer_name); return self->manufacturer_name;
+} // }}}
+
+// Device.model_name {{{
+static PyObject *
+libmtp_Device_model_name(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->model_name); return self->model_name;
+} // }}}
+
+// Device.serial_number {{{
+static PyObject *
+libmtp_Device_serial_number(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->serial_number); return self->serial_number;
+} // }}}
+
+// Device.device_version {{{
+static PyObject *
+libmtp_Device_device_version(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->device_version); return self->device_version;
+} // }}}
+
+// Device.ids {{{
+static PyObject *
+libmtp_Device_ids(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->ids); return self->ids;
+} // }}}
+
+// Device.update_storage_info() {{{
+static PyObject*
+libmtp_Device_update_storage_info(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    ENSURE_DEV(NULL);
+    if (LIBMTP_Get_Storage(self->device, LIBMTP_STORAGE_SORTBY_NOTSORTED) < 0) {
+        PyErr_SetString(PyExc_RuntimeError, "Failed to get storage infor for device.");
+        return NULL;
+    }
+    Py_RETURN_NONE;
+}
+// }}}
+
+// Device.storage_info {{{
+static PyObject *
+libmtp_Device_storage_info(libmtp_Device *self, void *closure) {
+    PyObject *ans, *loc;
+    LIBMTP_devicestorage_t *storage;
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    ans = PyList_New(0);
+    if (ans == NULL) { PyErr_NoMemory(); return NULL; }
+
+    for (storage = self->device->storage; storage != NULL; storage = storage->next) {
+        // Ignore read only storage
+        if (storage->StorageType == ST_FixedROM || storage->StorageType == ST_RemovableROM) continue;
+        // Storage IDs with the lower 16 bits 0x0000 are not supposed to be
+        // writeable.
+        if ((storage->id & 0x0000FFFFU) == 0x00000000U) continue;
+        // Also check the access capability to avoid e.g. deletable only storages
+        if (storage->AccessCapability == AC_ReadOnly || storage->AccessCapability == AC_ReadOnly_with_Object_Deletion) continue;
+
+        loc = Py_BuildValue("{s:k,s:O,s:K,s:K,s:K,s:s,s:s}", 
+                "id", storage->id, 
+                "removable", ((storage->StorageType == ST_RemovableRAM) ? Py_True : Py_False),
+                "capacity", storage->MaxCapacity,
+                "freespace_bytes", storage->FreeSpaceInBytes,
+                "freespace_objects", storage->FreeSpaceInObjects,
+                "storage_desc", storage->StorageDescription,
+                "volume_id", storage->VolumeIdentifier
+        );
+
+        if (loc == NULL) return NULL; 
+        if (PyList_Append(ans, loc) != 0) return NULL;
+        Py_DECREF(loc);
+
+    }
+
+    return ans;
+} // }}}
+
+// Device.get_filelist {{{
+static PyObject *
+libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *ans, *fo, *callback = NULL, *errs;
+    ProgressCallback cb;
+    LIBMTP_file_t *f, *tf;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+
+    if (!PyArg_ParseTuple(args, "|O", &callback)) return NULL;
+    cb.obj = callback;
+
+    ans = PyList_New(0);
+    errs = PyList_New(0);
+    if (ans == NULL || errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    cb.state = PyEval_SaveThread();
+    tf = LIBMTP_Get_Filelisting_With_Callback(self->device, report_progress, &cb);
+    PyEval_RestoreThread(cb.state);
+
+    if (tf == NULL) { 
+        dump_errorstack(self->device, errs);
+        return Py_BuildValue("NN", ans, errs);
+    }
+
+    for (f=tf; f != NULL; f=f->next) {
+        fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
+                "id", f->item_id,
+                "parent_id", f->parent_id,
+                "storage_id", f->storage_id,
+                "filename", f->filename,
+                "size", f->filesize,
+                "modtime", f->modificationdate
+        );
+        if (fo == NULL || PyList_Append(ans, fo) != 0) break;
+        Py_DECREF(fo);
+    }
+
+    // Release memory
+    f = tf;
+    while (f != NULL) {
+        tf = f; f = f->next; LIBMTP_destroy_file_t(tf);
+    }
+
+    if (callback != NULL) {
+        // Bug in libmtp where it does not call callback with 100%
+        fo = PyObject_CallMethod(callback, "report_progress", "KK", PyList_Size(ans), PyList_Size(ans));
+        Py_XDECREF(fo);
+    }
+
+    return Py_BuildValue("NN", ans, errs);
+} // }}}
+
+// Device.get_folderlist {{{
+
+int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
+    PyObject *folder, *children;
+
+    children = PyList_New(0);
+    if (children == NULL) { PyErr_NoMemory(); return 1;}
+
+    folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
+            "id", f->folder_id,
+            "parent_d", f->parent_id,
+            "storage_id", f->storage_id,
+            "name", f->name,
+            "children", children);
+    if (folder == NULL) return 1;
+    PyList_Append(parent, folder);
+    Py_DECREF(folder);
+
+    if (f->sibling != NULL) {
+        if (folderiter(f->sibling, parent)) return 1;
+    }
+
+    if (f->child != NULL) {
+        if (folderiter(f->child, children)) return 1;
+    }
+
+    return 0;
+}
+
+static PyObject *
+libmtp_Device_get_folderlist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *ans, *errs;
+    LIBMTP_folder_t *f;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    ans = PyList_New(0);
+    errs = PyList_New(0);
+    if (errs == NULL || ans == NULL) { PyErr_NoMemory(); return NULL; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    f = LIBMTP_Get_Folder_List(self->device);
+    Py_END_ALLOW_THREADS;
+
+    if (f == NULL) {
+        dump_errorstack(self->device, errs);
+        return Py_BuildValue("NN", ans, errs);
+    }
+
+    if (folderiter(f, ans)) return NULL;
+    LIBMTP_destroy_folder_t(f);
+
+    return Py_BuildValue("NN", ans, errs);
+
+} // }}}
+
+static PyMethodDef libmtp_Device_methods[] = {
+    {"update_storage_info", (PyCFunction)libmtp_Device_update_storage_info, METH_VARARGS,
+     "update_storage_info() -> Reread the storage info from the device (total, space, free space, storage locations, etc.)"
+    },
+
+    {"get_filelist", (PyCFunction)libmtp_Device_get_filelist, METH_VARARGS,
+     "get_filelist(callback=None) -> Get the list of files on the device. callback must be an object that has a method named 'report_progress(current, total)'. Returns files, errors."
+    },
+
+    {"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS,
+     "get_folderlist() -> Get the list of folders on the device. Returns files, erros."
+    },
+
+    {NULL}  /* Sentinel */
+};
+
+static PyGetSetDef libmtp_Device_getsetters[] = {
+    {(char *)"friendly_name", 
+     (getter)libmtp_Device_friendly_name, NULL,
+     (char *)"The friendly name of this device, can be None.",
+     NULL},
+
+    {(char *)"manufacturer_name", 
+     (getter)libmtp_Device_manufacturer_name, NULL,
+     (char *)"The manufacturer name of this device, can be None.",
+     NULL},
+
+    {(char *)"model_name", 
+     (getter)libmtp_Device_model_name, NULL,
+     (char *)"The model name of this device, can be None.",
+     NULL},
+
+    {(char *)"serial_number", 
+     (getter)libmtp_Device_serial_number, NULL,
+     (char *)"The serial number of this device, can be None.",
+     NULL},
+
+    {(char *)"device_version", 
+     (getter)libmtp_Device_device_version, NULL,
+     (char *)"The device version of this device, can be None.",
+     NULL},
+
+    {(char *)"ids", 
+     (getter)libmtp_Device_ids, NULL,
+     (char *)"The ids of the device (busnum, devnum, vendor_id, product_id, usb_serialnum)",
+     NULL},
+
+    {(char *)"storage_info",
+     (getter)libmtp_Device_storage_info, NULL,
+     (char *)"Information about the storage locations on the device. Returns a list of dictionaries where each dictionary corresponds to the LIBMTP_devicestorage_struct.",
+     NULL},
+
+    {NULL}  /* Sentinel */
+};
+
+static PyTypeObject libmtp_DeviceType = { // {{{
+    PyObject_HEAD_INIT(NULL)
+    0,                         /*ob_size*/
+    "libmtp.Device",            /*tp_name*/
+    sizeof(libmtp_Device),      /*tp_basicsize*/
+    0,                         /*tp_itemsize*/
+    (destructor)libmtp_Device_dealloc, /*tp_dealloc*/
+    0,                         /*tp_print*/
+    0,                         /*tp_getattr*/
+    0,                         /*tp_setattr*/
+    0,                         /*tp_compare*/
+    0,                         /*tp_repr*/
+    0,                         /*tp_as_number*/
+    0,                         /*tp_as_sequence*/
+    0,                         /*tp_as_mapping*/
+    0,                         /*tp_hash */
+    0,                         /*tp_call*/
+    0,                         /*tp_str*/
+    0,                         /*tp_getattro*/
+    0,                         /*tp_setattro*/
+    0,                         /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,        /*tp_flags*/
+    "Device",                  /* tp_doc */
+    0,		               /* tp_traverse */
+    0,		               /* tp_clear */
+    0,		               /* tp_richcompare */
+    0,		               /* tp_weaklistoffset */
+    0,		               /* tp_iter */
+    0,		               /* tp_iternext */
+    libmtp_Device_methods,             /* tp_methods */
+    0,             /* tp_members */
+    libmtp_Device_getsetters,                         /* tp_getset */
+    0,                         /* tp_base */
+    0,                         /* tp_dict */
+    0,                         /* tp_descr_get */
+    0,                         /* tp_descr_set */
+    0,                         /* tp_dictoffset */
+    (initproc)libmtp_Device_init,      /* tp_init */
+    0,                         /* tp_alloc */
+    0,                 /* tp_new */
+}; // }}}
+
+// }}} End Device object definition
+
+static PyObject *
+libmtp_set_debug_level(PyObject *self, PyObject *args) {
+    int level;
+    if (!PyArg_ParseTuple(args, "i", &level)) return NULL;
+    LIBMTP_Set_Debug(level);
+    Py_RETURN_NONE;
+}
+
+
+static PyObject *
+libmtp_is_mtp_device(PyObject *self, PyObject *args) {
+    int busnum, devnum, vendor_id, prod_id, ans = 0;
+    size_t i;
+
+    if (!PyArg_ParseTuple(args, "iiii", &busnum, &devnum, &vendor_id, &prod_id)) return NULL;
+
+    for (i = 0; ; i++) {
+        if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
+        if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == prod_id) {
+            Py_RETURN_TRUE;
+        }
+    }
+
+    /*
+     * LIBMTP_Check_Specific_Device does not seem to work at least on my linux
+     * system. Need to investigate why later. Most devices are in the device
+     * table so this is not terribly important.
+     */
+    /* LIBMTP_Set_Debug(LIBMTP_DEBUG_ALL); */
+    /* printf("Calling check: %d %d\n", busnum, devnum); */
+    Py_BEGIN_ALLOW_THREADS;
+    ans = LIBMTP_Check_Specific_Device(busnum, devnum);
+    Py_END_ALLOW_THREADS;
+
+    if (ans) Py_RETURN_TRUE;
+
+    Py_RETURN_FALSE;
+
+}
+
+static PyMethodDef libmtp_methods[] = {
+    {"set_debug_level", libmtp_set_debug_level, METH_VARARGS,
+        "set_debug_level(level)\n\nSet the debug level bit mask, see LIBMTP_DEBUG_* constants."
+    },
+
+    {"is_mtp_device", libmtp_is_mtp_device, METH_VARARGS,
+        "is_mtp_device(busnum, devnum, vendor_id, prod_id)\n\nReturn True if the device is recognized as an MTP device by its vendor/product ids. If it is not recognized a probe is done and True returned if the probe succeeds. Note that probing can cause some devices to malfunction, and it is not very reliable, which is why we prefer to use the device database."
+    },
+
+    {NULL, NULL, 0, NULL}
+};
+
+
+PyMODINIT_FUNC
+initlibmtp(void) {
+    PyObject *m;
+
+    libmtp_DeviceType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&libmtp_DeviceType) < 0)
+        return;
+    
+    m = Py_InitModule3("libmtp", libmtp_methods, "Interface to libmtp.");
+    if (m == NULL) return;
+
+    LIBMTP_Init();
+    LIBMTP_Set_Debug(LIBMTP_DEBUG_NONE);
+
+    Py_INCREF(&libmtp_DeviceType);
+    PyModule_AddObject(m, "Device", (PyObject *)&libmtp_DeviceType);
+
+    PyModule_AddStringMacro(m, LIBMTP_VERSION_STRING);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_NONE);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_PTP);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_PLST);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_USB);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_DATA);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_ALL);
+}

src/calibre/devices/mtp/unix/upstream/device-flags.h

@@ -0,0 +1,329 @@
+/**
+ * \file device-flags.h
+ * Special device flags to deal with bugs in specific devices.
+ *
+ * Copyright (C) 2005-2007 Richard A. Low <richard@wentnet.com>
+ * Copyright (C) 2005-2012 Linus Walleij <triad@df.lth.se>
+ * Copyright (C) 2006-2007 Marcus Meissner
+ * Copyright (C) 2007 Ted Bullock
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * This file is supposed to be included by both libmtp and libgphoto2.
+ */
+
+/**
+ * These flags are used to indicate if some or other
+ * device need special treatment. These should be possible
+ * to concatenate using logical OR so please use one bit per
+ * feature and lets pray we don't need more than 32 bits...
+ */
+#define DEVICE_FLAG_NONE 0x00000000
+/**
+ * This means that the PTP_OC_MTP_GetObjPropList is broken
+ * in the sense that it won't return properly formatted metadata
+ * for ALL files on the device when you request an object
+ * property list for object 0xFFFFFFFF with parameter 3 likewise
+ * set to 0xFFFFFFFF. Compare to
+ * DEVICE_FLAG_BROKEN_MTPGETOBJECTPROPLIST which only signify
+ * that it's broken when getting metadata for a SINGLE object.
+ * A typical way the implementation may be broken is that it
+ * may not return a proper count of the objects, and sometimes
+ * (like on the ZENs) objects are simply missing from the list
+ * if you use this. Sometimes it has been used incorrectly to
+ * mask bugs in the code (like handling transactions of data
+ * with size given to -1 (0xFFFFFFFFU), in that case please
+ * help us remove it now the code is fixed. Sometimes this is
+ * used because getting all the objects is just too slow and
+ * the USB transaction will time out if you use this command.
+ */
+#define DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL 0x00000001
+/**
+ * This means that under Linux, another kernel module may
+ * be using this device's USB interface, so we need to detach
+ * it if it is. Typically this is on dual-mode devices that
+ * will present both an MTP compliant interface and device
+ * descriptor *and* a USB mass storage interface. If the USB
+ * mass storage interface is in use, other apps (like our
+ * userspace libmtp through libusb access path) cannot get in
+ * and get cosy with it. So we can remove the offending
+ * application. Typically this means you have to run the program
+ * as root as well.
+ */
+#define DEVICE_FLAG_UNLOAD_DRIVER 0x00000002
+/**
+ * This means that the PTP_OC_MTP_GetObjPropList (9805)
+ * is broken in some way, either it doesn't work at all
+ * (as for Android devices) or it won't properly return all
+ * object properties if parameter 3 is set to 0xFFFFFFFFU.
+ */
+#define DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST 0x00000004
+/**
+ * This means the device doesn't send zero packets to indicate
+ * end of transfer when the transfer boundary occurs at a
+ * multiple of 64 bytes (the USB 1.1 endpoint size). Instead,
+ * exactly one extra byte is sent at the end of the transfer
+ * if the size is an integer multiple of USB 1.1 endpoint size
+ * (64 bytes).
+ *
+ * This behaviour is most probably a workaround due to the fact
+ * that the hardware USB slave controller in the device cannot
+ * handle zero writes at all, and the usage of the USB 1.1
+ * endpoint size is due to the fact that the device will "gear
+ * down" on a USB 1.1 hub, and since 64 bytes is a multiple of
+ * 512 bytes, it will work with USB 1.1 and USB 2.0 alike.
+ */
+#define DEVICE_FLAG_NO_ZERO_READS 0x00000008
+/**
+ * This flag means that the device is prone to forgetting the
+ * OGG container file type, so that libmtp must look at the
+ * filename extensions in order to determine that a file is
+ * actually OGG. This is a clear and present firmware bug, and
+ * while firmware bugs should be fixed in firmware, we like
+ * OGG so much that we back it by introducing this flag.
+ * The error has only been seen on iriver devices. Turning this
+ * flag on won't hurt anything, just that the check against
+ * filename extension will be done for files of "unknown" type.
+ * If the player does not even know (reports) that it supports
+ * ogg even though it does, please use the stronger
+ * OGG_IS_UNKNOWN flag, which will forcedly support ogg on
+ * anything with the .ogg filename extension.
+ */
+#define DEVICE_FLAG_IRIVER_OGG_ALZHEIMER 0x00000010
+/**
+ * This flag indicates a limitation in the filenames a device
+ * can accept - they must be 7 bit (all chars <= 127/0x7F).
+ * It was found first on the Philips Shoqbox, and is a deviation
+ * from the PTP standard which mandates that any unicode chars
+ * may be used for filenames. I guess this is caused by a 7bit-only
+ * filesystem being used intrinsically on the device.
+ */
+#define DEVICE_FLAG_ONLY_7BIT_FILENAMES 0x00000020
+/**
+ * This flag indicates that the device will lock up if you
+ * try to get status of endpoints and/or release the interface
+ * when closing the device. This fixes problems with SanDisk
+ * Sansa devices especially. It may be a side-effect of a
+ * Windows behaviour of never releasing interfaces.
+ */
+#define DEVICE_FLAG_NO_RELEASE_INTERFACE 0x00000040
+/**
+ * This flag was introduced with the advent of Creative ZEN
+ * 8GB. The device sometimes return a broken PTP header
+ * like this: < 1502 0000 0200 01d1 02d1 01d2 >
+ * the latter 6 bytes (representing "code" and "transaction ID")
+ * contain junk. This is breaking the PTP/MTP spec but works
+ * on Windows anyway, probably because the Windows implementation
+ * does not check that these bytes are valid. To interoperate
+ * with devices like this, we need this flag to emulate the
+ * Windows bug. Broken headers has also been found in the
+ * Aricent MTP stack.
+ */
+#define DEVICE_FLAG_IGNORE_HEADER_ERRORS 0x00000080
+/**
+ * The Motorola RAZR2 V8 (others?) has broken set object
+ * proplist causing the metadata setting to fail. (The
+ * set object prop to set individual properties work on
+ * this device, but the metadata is plain ignored on
+ * tracks, though e.g. playlist names can be set.)
+ */
+#define DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST 0x00000100
+/**
+ * The Samsung YP-T10 think Ogg files shall be sent with
+ * the "unknown" (PTP_OFC_Undefined) file type, this gives a
+ * side effect that is a combination of the iRiver Ogg Alzheimer
+ * problem (have to recognized Ogg files on file extension)
+ * and a need to report the Ogg support (the device itself does
+ * not properly claim to support it) and need to set filetype
+ * to unknown when storing Ogg files, even though they're not
+ * actually unknown. Later iRivers seem to need this flag since
+ * they do not report to support OGG even though they actually
+ * do. Often the device supports OGG in USB mass storage mode,
+ * then the firmware simply miss to declare metadata support
+ * for OGG properly.
+ */
+#define DEVICE_FLAG_OGG_IS_UNKNOWN 0x00000200
+/**
+ * The Creative Zen is quite unstable in libmtp but seems to
+ * be better with later firmware versions. However, it still
+ * frequently crashes when setting album art dimensions. This
+ * flag disables setting the dimensions (which seems to make
+ * no difference to how the graphic is displayed).
+ */
+#define DEVICE_FLAG_BROKEN_SET_SAMPLE_DIMENSIONS 0x00000400
+/**
+ * Some devices, particularly SanDisk Sansas, need to always
+ * have their "OS Descriptor" probed in order to work correctly.
+ * This flag provides that extra massage.
+ */
+#define DEVICE_FLAG_ALWAYS_PROBE_DESCRIPTOR 0x00000800
+/**
+ * Samsung has implimented its own playlist format as a .spl file
+ * stored in the normal file system, rather than a proper mtp
+ * playlist. There are multiple versions of the .spl format
+ * identified by a line in the file: VERSION X.XX
+ * Version 1.00 is just a simple playlist.
+ */
+#define DEVICE_FLAG_PLAYLIST_SPL_V1 0x00001000
+/**
+ * Samsung has implimented its own playlist format as a .spl file
+ * stored in the normal file system, rather than a proper mtp
+ * playlist. There are multiple versions of the .spl format
+ * identified by a line in the file: VERSION X.XX
+ * Version 2.00 is playlist but allows DNSe sound settings
+ * to be stored, per playlist.
+ */
+#define DEVICE_FLAG_PLAYLIST_SPL_V2 0x00002000
+/**
+ * The Sansa E250 is know to have this problem which is actually
+ * that the device claims that property PTP_OPC_DateModified
+ * is read/write but will still fail to update it. It can only
+ * be set properly the first time a file is sent.
+ */
+#define DEVICE_FLAG_CANNOT_HANDLE_DATEMODIFIED 0x00004000
+/**
+ * This avoids use of the send object proplist which
+ * is used when creating new objects (not just updating)
+ * The DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST is related
+ * but only concerns the case where the object proplist
+ * is sent in to update an existing object. The Toshiba
+ * Gigabeat MEU202 for example has this problem.
+ */
+#define DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST 0x00008000
+/**
+ * Devices that cannot support reading out battery
+ * level.
+ */
+#define DEVICE_FLAG_BROKEN_BATTERY_LEVEL 0x00010000
+
+/**
+ * Devices that send "ObjectDeleted" events after deletion
+ * of images. (libgphoto2)
+ */
+#define DEVICE_FLAG_DELETE_SENDS_EVENT	0x00020000
+
+/**
+ * Cameras that can capture images. (libgphoto2)
+ */
+#define DEVICE_FLAG_CAPTURE		0x00040000
+
+/**
+ * Cameras that can capture images. (libgphoto2)
+ */
+#define DEVICE_FLAG_CAPTURE_PREVIEW	0x00080000
+
+/**
+ * Nikon broken capture support without proper ObjectAdded events.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_NIKON_BROKEN_CAPTURE	0x00100000
+
+/**
+ * Broken capture support where cameras do not send CaptureComplete events.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_NO_CAPTURE_COMPLETE		0x00400000
+
+/**
+ * Direct PTP match required.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_MATCH_PTP_INTERFACE		0x00800000
+/**
+ * This flag is like DEVICE_FLAG_OGG_IS_UNKNOWN but for FLAC
+ * files instead. Using the unknown filetype for FLAC files.
+ */
+#define DEVICE_FLAG_FLAC_IS_UNKNOWN		0x01000000
+/**
+ * Device needs unique filenames, no two files can be
+ * named the same string.
+ */
+#define DEVICE_FLAG_UNIQUE_FILENAMES		0x02000000
+/**
+ * This flag performs some random magic on the BlackBerry
+ * device to switch from USB mass storage to MTP mode we think.
+ */
+#define DEVICE_FLAG_SWITCH_MODE_BLACKBERRY	0x04000000
+/**
+ * This flag indicates that the device need an extra long
+ * timeout on some operations.
+ */
+#define DEVICE_FLAG_LONG_TIMEOUT		0x08000000
+/**
+ * This flag indicates that the device need an explicit
+ * USB reset after each connection. Some devices don't
+ * like this, so it's not done by default.
+ */
+#define DEVICE_FLAG_FORCE_RESET_ON_CLOSE	0x10000000
+/**
+ * Early Creative Zen (etc) models actually only support
+ * command 9805 (Get object property list) and will hang
+ * if you try to get individual properties of an object.
+ */
+#define DEVICE_FLAG_BROKEN_GET_OBJECT_PROPVAL	0x20000000
+/**
+ * It seems that some devices return an bad data when
+ * using the GetObjectInfo operation. So in these cases
+ * we prefer to override the PTP-compatible object infos
+ * with the MTP property list.
+ *
+ * For example Some Samsung Galaxy S devices contain an MTP
+ * stack that present the ObjectInfo in 64 bit instead of
+ * 32 bit.
+ */
+#define DEVICE_FLAG_PROPLIST_OVERRIDES_OI	0x40000000
+
+/**
+ * All these bug flags need to be set on SONY NWZ Walkman
+ * players, and will be autodetected on unknown devices
+ * by detecting the vendor extension descriptor "sony.net"
+ */
+#define DEVICE_FLAGS_SONY_NWZ_BUGS \
+  (DEVICE_FLAG_UNLOAD_DRIVER | \
+   DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | \
+   DEVICE_FLAG_UNIQUE_FILENAMES | \
+   DEVICE_FLAG_FORCE_RESET_ON_CLOSE )
+/**
+ * All these bug flags need to be set on Android devices,
+ * they claim to support MTP operations they actually
+ * cannot handle, especially 9805 (Get object property list).
+ * These are auto-assigned to devices reporting
+ * "android.com" in their device extension descriptor.
+ */
+#define DEVICE_FLAGS_ANDROID_BUGS \
+  (DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | \
+   DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST | \
+   DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST | \
+   DEVICE_FLAG_UNLOAD_DRIVER | \
+   DEVICE_FLAG_LONG_TIMEOUT )
+/**
+ * All these bug flags appear on a number of SonyEricsson
+ * devices including Android devices not using the stock
+ * Android 4.0+ (Ice Cream Sandwich) MTP stack. It is highly
+ * supected that these bugs comes from an MTP implementation
+ * from Aricent, so it is called the Aricent bug flags as a
+ * shorthand. Especially the header errors that need to be
+ * ignored is typical for this stack.
+ *
+ * After some guesswork we auto-assign these bug flags to
+ * devices that present the "microsoft.com/WPDNA", and
+ * "sonyericsson.com/SE" but NOT the "android.com"
+ * descriptor.
+ */
+#define DEVICE_FLAGS_ARICENT_BUGS \
+  (DEVICE_FLAG_IGNORE_HEADER_ERRORS | \
+   DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST | \
+   DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST )

src/calibre/devices/mtp/unix/upstream/update.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+MP = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/music-players.h;hb=HEAD'
+DF = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/device-flags.h;hb=HEAD'
+
+import urllib, os, shutil
+
+base = os.path.dirname(os.path.abspath(__file__))
+
+for url, fname in [(MP, 'music-players.h'), (DF, 'device-flags.h')]:
+    with open(os.path.join(base, fname), 'wb') as f:
+        shutil.copyfileobj(urllib.urlopen(url), f)
+

src/calibre/devices/prst1/driver.py

@@ -376,6 +376,8 @@ class PRST1(USBMS):
         # Record what the max id being used is as well.
         db_books = {}
         for i, row in enumerate(cursor):
+            if row[0] is None:
+                continue
             lpath = row[0].replace('\\', '/')
             db_books[lpath] = row[1]
             if row[1] < sequence_min:

src/calibre/devices/scanner.py

@@ -7,6 +7,7 @@ manner.
 
 import sys, os, re
 from threading import RLock
+from collections import namedtuple
 
 from calibre import prints, as_unicode
 from calibre.constants import iswindows, isosx, plugins, islinux, isfreebsd
@@ -107,6 +108,15 @@ class WinPNPScanner(object):
 
 win_pnp_drives = WinPNPScanner()
 
+_USBDevice = namedtuple('USBDevice',
+    'vendor_id product_id bcd manufacturer product serial')
+
+class USBDevice(_USBDevice):
+
+    def __init__(self, *args, **kwargs):
+        _USBDevice.__init__(self, *args, **kwargs)
+        self.busnum = self.devnum = -1
+
 class LinuxScanner(object):
 
     SYSFS_PATH = os.environ.get('SYSFS_PATH', '/sys')
@@ -122,6 +132,10 @@ class LinuxScanner(object):
         if not self.ok:
             raise RuntimeError('DeviceScanner requires the /sys filesystem to work.')
 
+        def read(f):
+            with open(f, 'rb') as s:
+                return s.read().strip()
+
         for x in os.listdir(self.base):
             base = os.path.join(self.base, x)
             ven = os.path.join(base, 'idVendor')
@@ -132,31 +146,46 @@ class LinuxScanner(object):
             prod_string = os.path.join(base, 'product')
             dev = []
             try:
-                dev.append(int('0x'+open(ven).read().strip(), 16))
+                # Ignore USB HUBs
+                if read(os.path.join(base, 'bDeviceClass')) == b'09':
+                    continue
             except:
                 continue
             try:
-                dev.append(int('0x'+open(prod).read().strip(), 16))
+                dev.append(int(b'0x'+read(ven), 16))
             except:
                 continue
             try:
-                dev.append(int('0x'+open(bcd).read().strip(), 16))
+                dev.append(int(b'0x'+read(prod), 16))
             except:
                 continue
             try:
-                dev.append(open(man).read().strip())
+                dev.append(int(b'0x'+read(bcd), 16))
             except:
-                dev.append('')
+                continue
             try:
-                dev.append(open(prod_string).read().strip())
+                dev.append(read(man))
             except:
-                dev.append('')
+                dev.append(b'')
             try:
-                dev.append(open(serial).read().strip())
+                dev.append(read(prod_string))
             except:
-                dev.append('')
+                dev.append(b'')
+            try:
+                dev.append(read(serial))
+            except:
+                dev.append(b'')
 
-            ans.add(tuple(dev))
+            dev = USBDevice(*dev)
+            try:
+                dev.busnum = int(read(os.path.join(base, 'busnum')))
+            except:
+                pass
+            try:
+                dev.devnum = int(read(os.path.join(base, 'devnum')))
+            except:
+                pass
+            ans.add(dev)
         return ans
 
 class FreeBSDScanner(object):

src/calibre/devices/smart_device_app/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/devices/smart_device_app/driver.py

@@ -0,0 +1,928 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+'''
+Created on 29 Jun 2012
+
+@author: charles
+'''
+import socket, select, json, inspect, os, traceback, time, sys, random
+import hashlib, threading
+from base64 import b64encode, b64decode
+from functools import wraps
+
+from calibre import prints
+from calibre.constants import numeric_version, DEBUG
+from calibre.devices.errors import (OpenFailed, ControlError, TimeoutError,
+                                    InitialConnectionError)
+from calibre.devices.interface import DevicePlugin
+from calibre.devices.usbms.books import Book, BookList
+from calibre.devices.usbms.deviceconfig import DeviceConfig
+from calibre.devices.usbms.driver import USBMS
+from calibre.ebooks import BOOK_EXTENSIONS
+from calibre.ebooks.metadata import title_sort
+from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS
+from calibre.ebooks.metadata.book.base import Metadata
+from calibre.ebooks.metadata.book.json_codec import JsonCodec
+from calibre.library import current_library_name
+from calibre.utils.ipc import eintr_retry_call
+from calibre.utils.config import from_json, tweaks
+from calibre.utils.date import isoformat, now
+from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to
+from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as
+        unpublish_zeroconf)
+
+def synchronous(tlockname):
+    """A decorator to place an instance based lock around a method """
+
+    def _synched(func):
+        @wraps(func)
+        def _synchronizer(self, *args, **kwargs):
+            with self.__getattribute__(tlockname):
+                return func(self, *args, **kwargs)
+        return _synchronizer
+    return _synched
+
+def do_zeroconf(f, port):
+    f('calibre smart device client',
+        '_calibresmartdeviceapp._tcp', port, {})
+
+
+class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
+    name = 'SmartDevice App Interface'
+    gui_name = _('SmartDevice')
+    icon = I('devices/galaxy_s3.png')
+    description = _('Communicate with Smart Device apps')
+    supported_platforms = ['windows', 'osx', 'linux']
+    author = 'Charles Haley'
+    version = (0, 0, 1)
+
+    # Invalid USB vendor information so the scanner will never match
+    VENDOR_ID                   = [0xffff]
+    PRODUCT_ID                  = [0xffff]
+    BCD                         = [0xffff]
+
+    FORMATS                     = list(BOOK_EXTENSIONS)
+    ALL_FORMATS                 = list(BOOK_EXTENSIONS)
+    HIDE_FORMATS_CONFIG_BOX     = True
+    USER_CAN_ADD_NEW_FORMATS    = False
+    DEVICE_PLUGBOARD_NAME       = 'SMART_DEVICE_APP'
+    CAN_SET_METADATA            = []
+    CAN_DO_DEVICE_DB_PLUGBOARD  = False
+    SUPPORTS_SUB_DIRS           = False
+    MUST_READ_METADATA          = True
+    NEWS_IN_FOLDER              = False
+    SUPPORTS_USE_AUTHOR_SORT    = False
+    WANTS_UPDATED_THUMBNAILS    = True
+    MAX_PATH_LEN                = 100
+    THUMBNAIL_HEIGHT            = 160
+    PREFIX                      = ''
+
+    # Some network protocol constants
+    BASE_PACKET_LEN             = 4096
+    PROTOCOL_VERSION            = 1
+    MAX_CLIENT_COMM_TIMEOUT     = 60.0 # Wait at most N seconds for an answer
+    MAX_UNSUCCESSFUL_CONNECTS   = 5
+
+    opcodes = {
+        'NOOP'                   : 12,
+        'OK'                     : 0,
+        'BOOK_DATA'              : 10,
+        'BOOK_DONE'              : 11,
+        'DELETE_BOOK'            : 13,
+        'DISPLAY_MESSAGE'        : 17,
+        'FREE_SPACE'             : 5,
+        'GET_BOOK_FILE_SEGMENT'  : 14,
+        'GET_BOOK_METADATA'      : 15,
+        'GET_BOOK_COUNT'         : 6,
+        'GET_DEVICE_INFORMATION' : 3,
+        'GET_INITIALIZATION_INFO': 9,
+        'SEND_BOOKLISTS'         : 7,
+        'SEND_BOOK'              : 8,
+        'SEND_BOOK_METADATA'     : 16,
+        'SET_CALIBRE_DEVICE_INFO': 1,
+        'SET_CALIBRE_DEVICE_NAME': 2,
+        'TOTAL_SPACE'            : 4,
+    }
+    reverse_opcodes = dict([(v, k) for k,v in opcodes.iteritems()])
+
+
+    EXTRA_CUSTOMIZATION_MESSAGE = [
+        _('Enable connections at startup') + ':::<p>' +
+            _('Check this box to allow connections when calibre starts') + '</p>',
+        '',
+        _('Security password') + ':::<p>' +
+            _('Enter a password that the device app must use to connect to calibre') + '</p>',
+        '',
+        _('Use fixed network port') + ':::<p>' +
+            _('If checked, use the port number in the "Port" box, otherwise '
+              'the driver will pick a random port') + '</p>',
+        _('Port') + ':::<p>' +
+            _('Enter the port number the driver is to use if the "fixed port" box is checked') + '</p>',
+        _('Print extra debug information') + ':::<p>' +
+            _('Check this box if requested when reporting problems') + '</p>',
+        ]
+    EXTRA_CUSTOMIZATION_DEFAULT = [
+                False,
+                '',
+                '',
+                '',
+                False, '9090',
+                False,
+    ]
+    OPT_AUTOSTART               = 0
+    OPT_PASSWORD                = 2
+    OPT_USE_PORT                = 4
+    OPT_PORT_NUMBER             = 5
+    OPT_EXTRA_DEBUG             = 6
+
+    def __init__(self, path):
+        self.sync_lock = threading.RLock()
+        self.noop_counter = 0
+        self.debug_start_time = time.time()
+        self.debug_time = time.time()
+
+    def _debug(self, *args):
+        if not DEBUG:
+            return
+        total_elapsed = time.time() - self.debug_start_time
+        elapsed = time.time() - self.debug_time
+        print('SMART_DEV (%7.2f:%7.3f) %s'%(total_elapsed, elapsed,
+                                               inspect.stack()[1][3]), end='')
+        for a in args:
+            try:
+                prints('', a, end='')
+            except:
+                prints('', 'value too long', end='')
+        print()
+        self.debug_time = time.time()
+
+    # Various methods required by the plugin architecture
+    @classmethod
+    def _default_save_template(cls):
+        from calibre.library.save_to_disk import config
+        st = cls.SAVE_TEMPLATE if cls.SAVE_TEMPLATE else \
+            config().parse().send_template
+        if st:
+            st = os.path.basename(st)
+        return st
+
+    @classmethod
+    def save_template(cls):
+        st = cls.settings().save_template
+        if st:
+            st = os.path.basename(st)
+        else:
+            st = cls._default_save_template()
+        return st
+
+    # local utilities
+
+    # copied from USBMS. Perhaps this could be a classmethod in usbms?
+    def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None):
+        import uuid
+        if not isinstance(dinfo, dict):
+            dinfo = {}
+        if dinfo.get('device_store_uuid', None) is None:
+            dinfo['device_store_uuid'] = unicode(uuid.uuid4())
+        if dinfo.get('device_name') is None:
+            dinfo['device_name'] = self.get_gui_name()
+        if name is not None:
+            dinfo['device_name'] = name
+        dinfo['location_code'] = location_code
+        dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None)
+        dinfo['calibre_version'] = '.'.join([unicode(i) for i in numeric_version])
+        dinfo['date_last_connected'] = isoformat(now())
+        dinfo['prefix'] = self.PREFIX
+        return dinfo
+
+    # copied with changes from USBMS.Device. In particular, we needed to
+    # remove the 'path' argument and all its uses. Also removed the calls to
+    # filename_callback and sanitize_path_components
+    def _create_upload_path(self, mdata, fname, create_dirs=True):
+        maxlen = self.MAX_PATH_LEN
+
+        special_tag = None
+        if mdata.tags:
+            for t in mdata.tags:
+                if t.startswith(_('News')) or t.startswith('/'):
+                    special_tag = t
+                    break
+
+        settings = self.settings()
+        template = self.save_template()
+        if mdata.tags and _('News') in mdata.tags:
+            try:
+                p = mdata.pubdate
+                date  = (p.year, p.month, p.day)
+            except:
+                today = time.localtime()
+                date = (today[0], today[1], today[2])
+            template = "{title}_%d-%d-%d" % date
+        use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs
+
+        fname = sanitize(fname)
+        ext = os.path.splitext(fname)[1]
+
+        from calibre.library.save_to_disk import get_components
+        from calibre.library.save_to_disk import config
+        opts = config().parse()
+        if not isinstance(template, unicode):
+            template = template.decode('utf-8')
+        app_id = str(getattr(mdata, 'application_id', ''))
+        id_ = mdata.get('id', fname)
+        extra_components = get_components(template, mdata, id_,
+                timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1)
+        if not extra_components:
+            extra_components.append(sanitize(fname))
+        else:
+            extra_components[-1] = sanitize(extra_components[-1]+ext)
+
+        if extra_components[-1] and extra_components[-1][0] in ('.', '_'):
+            extra_components[-1] = 'x' + extra_components[-1][1:]
+
+        if special_tag is not None:
+            name = extra_components[-1]
+            extra_components = []
+            tag = special_tag
+            if tag.startswith(_('News')):
+                if self.NEWS_IN_FOLDER:
+                    extra_components.append('News')
+            else:
+                for c in tag.split('/'):
+                    c = sanitize(c)
+                    if not c: continue
+                    extra_components.append(c)
+            extra_components.append(name)
+
+        if not use_subdirs:
+            # Leave this stuff here in case we later decide to use subdirs
+            extra_components = extra_components[-1:]
+
+        def remove_trailing_periods(x):
+            ans = x
+            while ans.endswith('.'):
+                ans = ans[:-1].strip()
+            if not ans:
+                ans = 'x'
+            return ans
+
+        extra_components = list(map(remove_trailing_periods, extra_components))
+        components = shorten_components_to(maxlen, extra_components)
+        filepath = os.path.join(*components)
+        return filepath
+
+    def _strip_prefix(self, path):
+        if self.PREFIX and path.startswith(self.PREFIX):
+            return path[len(self.PREFIX):]
+        return path
+
+    # JSON booklist encode & decode
+
+    # If the argument is a booklist or contains a book, use the metadata json
+    # codec to first convert it to a string dict
+    def _json_encode(self, op, arg):
+        res = {}
+        for k,v in arg.iteritems():
+            if isinstance(v, (Book, Metadata)):
+                res[k] = self.json_codec.encode_book_metadata(v)
+                series = v.get('series', None)
+                if series:
+                    tsorder = tweaks['save_template_title_series_sorting']
+                    series = title_sort(v.get('series', ''), order=tsorder)
+                else:
+                    series = ''
+                res[k]['_series_sort_'] = series
+            else:
+                res[k] = v
+        return json.dumps([op, res], encoding='utf-8')
+
+    # Network functions
+    def _read_string_from_net(self):
+        data = bytes(0)
+        while True:
+            dex = data.find(b'[')
+            if dex >= 0:
+                break
+            # recv seems to return a pointer into some internal buffer.
+            # Things get trashed if we don't make a copy of the data.
+            self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
+            v = self.device_socket.recv(self.BASE_PACKET_LEN)
+            self.device_socket.settimeout(None)
+            if len(v) == 0:
+                return '' # documentation says the socket is broken permanently.
+            data += v
+        total_len = int(data[:dex])
+        data = data[dex:]
+        pos = len(data)
+        while pos < total_len:
+            self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
+            v = self.device_socket.recv(total_len - pos)
+            self.device_socket.settimeout(None)
+            if len(v) == 0:
+                return '' # documentation says the socket is broken permanently.
+            data += v
+            pos += len(v)
+        return data
+
+    def _call_client(self, op, arg, print_debug_info=True):
+        if op != 'NOOP':
+            self.noop_counter = 0
+        extra_debug = self.settings().extra_customization[self.OPT_EXTRA_DEBUG]
+        if print_debug_info or extra_debug:
+            if extra_debug:
+                self._debug(op, arg)
+            else:
+                self._debug(op)
+        if self.device_socket is None:
+            return None, None
+        try:
+            s = self._json_encode(self.opcodes[op], arg)
+            if print_debug_info and extra_debug:
+                self._debug('send string', s)
+            self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
+            self.device_socket.sendall(('%d' % len(s))+s)
+            self.device_socket.settimeout(None)
+            v = self._read_string_from_net()
+            if print_debug_info and extra_debug:
+                self._debug('received string', v)
+            if v:
+                v = json.loads(v, object_hook=from_json)
+                if print_debug_info and extra_debug:
+                        self._debug('receive after decode') #, v)
+                return (self.reverse_opcodes[v[0]], v[1])
+            self._debug('protocol error -- empty json string')
+        except socket.timeout:
+            self._debug('timeout communicating with device')
+            self._close_device_socket()
+            raise TimeoutError('Device did not respond in reasonable time')
+        except socket.error:
+            self._debug('device went away')
+            self._close_device_socket()
+            raise ControlError('Device closed the network connection')
+        except:
+            self._debug('other exception')
+            traceback.print_exc()
+            self._close_device_socket()
+            raise
+        raise ControlError('Device responded with incorrect information')
+
+    # Write a file as a series of base64-encoded strings.
+    def _put_file(self, infile, lpath, book_metadata, this_book, total_books):
+        close_ = False
+        if not hasattr(infile, 'read'):
+            infile, close_ = open(infile, 'rb'), True
+        infile.seek(0, os.SEEK_END)
+        length = infile.tell()
+        book_metadata.size = length
+        infile.seek(0)
+        self._debug(lpath, length)
+        self._call_client('SEND_BOOK', {'lpath': lpath, 'length': length,
+                               'metadata': book_metadata, 'thisBook': this_book,
+                               'totalBooks': total_books}, print_debug_info=False)
+        self._set_known_metadata(book_metadata)
+        pos = 0
+        failed = False
+        with infile:
+            while True:
+                b = infile.read(self.max_book_packet_len)
+                blen = len(b)
+                if not b:
+                    break;
+                b = b64encode(b)
+                opcode, result = self._call_client('BOOK_DATA',
+                                {'lpath': lpath, 'position': pos, 'data': b},
+                                print_debug_info=False)
+                pos += blen
+                if opcode != 'OK':
+                    self._debug('protocol error', opcode)
+                    failed = True
+                    break
+        self._call_client('BOOK_DONE', {'lpath': lpath})
+        self.time = None
+        if close_:
+            infile.close()
+        return -1 if failed else length
+
+    def _get_smartdevice_option_number(self, opt_string):
+        if opt_string == 'password':
+            return self.OPT_PASSWORD
+        elif opt_string == 'autostart':
+            return self.OPT_AUTOSTART
+        else:
+            return None
+
+    def _compare_metadata(self, mi1, mi2):
+        for key in SERIALIZABLE_FIELDS:
+            if key in ['cover', 'mime']:
+                continue
+            if key == 'user_metadata':
+                meta1 = mi1.get_all_user_metadata(make_copy=False)
+                meta2 = mi1.get_all_user_metadata(make_copy=False)
+                if meta1 != meta2:
+                    self._debug('custom metadata different')
+                    return False
+                for ckey in meta1:
+                    if mi1.get(ckey) != mi2.get(ckey):
+                        self._debug(ckey, mi1.get(ckey), mi2.get(ckey))
+                        return False
+            elif mi1.get(key, None) != mi2.get(key, None):
+                self._debug(key, mi1.get(key), mi2.get(key))
+                return False
+        return True
+
+    def _metadata_already_on_device(self, book):
+        v = self.known_metadata.get(book.lpath, None)
+        if v is not None:
+            return self._compare_metadata(book, v)
+        return False
+
+    def _set_known_metadata(self, book, remove=False):
+        lpath = book.lpath
+        if remove:
+            self.known_metadata[lpath] = None
+        else:
+            self.known_metadata[lpath] = book.deepcopy()
+
+    def _close_device_socket(self):
+        if self.device_socket is not None:
+            try:
+                self.device_socket.close()
+            except:
+                pass
+            self.device_socket = None
+        self.is_connected = False
+
+    # The public interface methods.
+
+    @synchronous('sync_lock')
+    def is_usb_connected(self, devices_on_system, debug=False, only_presence=False):
+        if getattr(self, 'listen_socket', None) is None:
+            self.is_connected = False
+        if self.is_connected:
+            self.noop_counter += 1
+            if only_presence and (self.noop_counter % 5) != 1:
+                try:
+                    ans = select.select((self.device_socket,), (), (), 0)
+                    if len(ans[0]) == 0:
+                        return (True, self)
+                    # The socket indicates that something is there. Given the
+                    # protocol, this can only be a disconnect notification. Fall
+                    # through and actually try to talk to the client.
+                    # This will usually toss an exception if the socket is gone.
+                except:
+                    pass
+            try:
+                if self._call_client('NOOP', dict())[0] is None:
+                    self._close_device_socket()
+            except:
+                self._close_device_socket()
+            return (self.is_connected, self)
+        if getattr(self, 'listen_socket', None) is not None:
+            ans = select.select((self.listen_socket,), (), (), 0)
+            if len(ans[0]) > 0:
+                # timeout in 10 ms to detect rare case where the socket went
+                # way between the select and the accept
+                try:
+                    self.device_socket = None
+                    self.listen_socket.settimeout(0.010)
+                    self.device_socket, ign = eintr_retry_call(
+                            self.listen_socket.accept)
+                    self.listen_socket.settimeout(None)
+                    self.device_socket.settimeout(None)
+                    self.is_connected = True
+                    try:
+                        peer = self.device_socket.getpeername()[0]
+                        attempts = self.connection_attempts.get(peer, 0)
+                        if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
+                            self._debug('too many connection attempts from', peer)
+                            self._close_device_socket()
+                            raise InitialConnectionError(_('Too many connection attempts from %s')%peer)
+                        else:
+                            self.connection_attempts[peer] = attempts + 1
+                    except InitialConnectionError:
+                        raise
+                    except:
+                        pass
+                except socket.timeout:
+                    self._close_device_socket()
+                except socket.error:
+                    x = sys.exc_info()[1]
+                    self._debug('unexpected socket exception', x.args[0])
+                    self._close_device_socket()
+                    raise
+                return (self.is_connected, self)
+        return (False, None)
+
+    @synchronous('sync_lock')
+    def open(self, connected_device, library_uuid):
+        self._debug()
+        if not self.is_connected:
+            # We have been called to retry the connection. Give up immediately
+            raise ControlError('Attempt to open a closed device')
+        self.current_library_uuid = library_uuid
+        self.current_library_name = current_library_name()
+        try:
+            password = self.settings().extra_customization[self.OPT_PASSWORD]
+            if password:
+                challenge = isoformat(now())
+                hasher = hashlib.new('sha1')
+                hasher.update(password.encode('UTF-8'))
+                hasher.update(challenge.encode('UTF-8'))
+                hash_digest = hasher.hexdigest()
+            else:
+                challenge = ''
+                hash_digest = ''
+            opcode, result = self._call_client('GET_INITIALIZATION_INFO',
+                                {'serverProtocolVersion': self.PROTOCOL_VERSION,
+                                'validExtensions': self.ALL_FORMATS,
+                                'passwordChallenge': challenge,
+                                'currentLibraryName': self.current_library_name,
+                                'currentLibraryUUID': library_uuid})
+            if opcode != 'OK':
+                # Something wrong with the return. Close the socket
+                # and continue.
+                self._debug('Protocol error - Opcode not OK')
+                self._close_device_socket()
+                return False
+            if not result.get('versionOK', False):
+                # protocol mismatch
+                self._debug('Protocol error - protocol version mismatch')
+                self._close_device_socket()
+                return False
+            if result.get('maxBookContentPacketLen', 0) <= 0:
+                # protocol mismatch
+                self._debug('Protocol error - bogus book packet length')
+                self._close_device_socket()
+                return False
+            self.max_book_packet_len = result.get('maxBookContentPacketLen',
+                                                  self.BASE_PACKET_LEN)
+            exts = result.get('acceptedExtensions', None)
+            if exts is None or not isinstance(exts, list) or len(exts) == 0:
+                self._debug('Protocol error - bogus accepted extensions')
+                self._close_device_socket()
+                return False
+            self.FORMATS = exts
+            if password:
+                returned_hash = result.get('passwordHash', None)
+                if result.get('passwordHash', None) is None:
+                    # protocol mismatch
+                    self._debug('Protocol error - missing password hash')
+                    self._close_device_socket()
+                    return False
+                if returned_hash != hash_digest:
+                    # bad password
+                    self._debug('password mismatch')
+                    try:
+                        self._call_client("DISPLAY_MESSAGE",
+                                {'messageKind':1,
+                                 'currentLibraryName': self.current_library_name,
+                                 'currentLibraryUUID': library_uuid})
+                    except:
+                        pass
+                    self._close_device_socket()
+                    # Don't bother with a message. The user will be informed on
+                    # the device.
+                    raise OpenFailed('')
+            try:
+                peer = self.device_socket.getpeername()[0]
+                self.connection_attempts[peer] = 0
+            except:
+                pass
+            return True
+        except socket.timeout:
+            self._close_device_socket()
+        except socket.error:
+            x = sys.exc_info()[1]
+            self._debug('unexpected socket exception', x.args[0])
+            self._close_device_socket()
+            raise
+        return False
+
+    @synchronous('sync_lock')
+    def get_device_information(self, end_session=True):
+        self._debug()
+        self.report_progress(1.0, _('Get device information...'))
+        opcode, result = self._call_client('GET_DEVICE_INFORMATION', dict())
+        if opcode == 'OK':
+            self.driveinfo = result['device_info']
+            self._update_driveinfo_record(self.driveinfo, self.PREFIX, 'main')
+            self._call_client('SET_CALIBRE_DEVICE_INFO', self.driveinfo)
+            return (self.get_gui_name(), result['device_version'],
+                    result['version'], '', {'main':self.driveinfo})
+        return (self.get_gui_name(), '', '', '')
+
+    @synchronous('sync_lock')
+    def set_driveinfo_name(self, location_code, name):
+        self._update_driveinfo_record(self.driveinfo, "main", name)
+        self._call_client('SET_CALIBRE_DEVICE_NAME',
+                         {'location_code': 'main', 'name':name})
+
+    @synchronous('sync_lock')
+    def reset(self, key='-1', log_packets=False, report_progress=None,
+            detected_device=None) :
+        self._debug()
+        self.set_progress_reporter(report_progress)
+
+    @synchronous('sync_lock')
+    def set_progress_reporter(self, report_progress):
+        self._debug()
+        self.report_progress = report_progress
+        if self.report_progress is None:
+            self.report_progress = lambda x, y: x
+
+    @synchronous('sync_lock')
+    def card_prefix(self, end_session=True):
+        self._debug()
+        return (None, None)
+
+    @synchronous('sync_lock')
+    def total_space(self, end_session=True):
+        self._debug()
+        opcode, result = self._call_client('TOTAL_SPACE', {})
+        if opcode == 'OK':
+            return (result['total_space_on_device'], 0, 0)
+        # protocol error if we get here
+        return (0, 0, 0)
+
+    @synchronous('sync_lock')
+    def free_space(self, end_session=True):
+        self._debug()
+        opcode, result = self._call_client('FREE_SPACE', {})
+        if opcode == 'OK':
+            return (result['free_space_on_device'], 0, 0)
+        # protocol error if we get here
+        return (0, 0, 0)
+
+    @synchronous('sync_lock')
+    def books(self, oncard=None, end_session=True):
+        self._debug(oncard)
+        if oncard is not None:
+            return BookList(None, None, None)
+        opcode, result = self._call_client('GET_BOOK_COUNT', {})
+        bl = BookList(None, self.PREFIX, self.settings)
+        if opcode == 'OK':
+            count = result['count']
+            for i in range(0, count):
+                self._debug('retrieve metadata book', i)
+                opcode, result = self._call_client('GET_BOOK_METADATA', {'index': i},
+                                                  print_debug_info=False)
+                if opcode == 'OK':
+                    if '_series_sort_' in result:
+                        del result['_series_sort_']
+                    book = self.json_codec.raw_to_book(result, Book, self.PREFIX)
+                    self._set_known_metadata(book)
+                    bl.add_book(book, replace_metadata=True)
+                else:
+                    raise ControlError('book metadata not returned')
+        return bl
+
+    @synchronous('sync_lock')
+    def sync_booklists(self, booklists, end_session=True):
+        self._debug()
+        # If we ever do device_db plugboards, this is where it will go. We will
+        # probably need to send two booklists, one with calibre's data that is
+        # given back by "books", and one that has been plugboarded.
+        self._call_client('SEND_BOOKLISTS', { 'count': len(booklists[0]) } )
+        for i,book in enumerate(booklists[0]):
+            if not self._metadata_already_on_device(book):
+                self._set_known_metadata(book)
+                self._debug('syncing book', book.lpath)
+                opcode, result = self._call_client('SEND_BOOK_METADATA',
+                                                  {'index': i, 'data': book},
+                                                  print_debug_info=False)
+                if opcode != 'OK':
+                    self._debug('protocol error', opcode, i)
+                    raise ControlError('sync_booklists')
+
+    @synchronous('sync_lock')
+    def eject(self):
+        self._debug()
+        self._close_device_socket()
+
+    @synchronous('sync_lock')
+    def post_yank_cleanup(self):
+        self._debug()
+
+    @synchronous('sync_lock')
+    def upload_books(self, files, names, on_card=None, end_session=True,
+                     metadata=None):
+        self._debug(names)
+
+        paths = []
+        names = iter(names)
+        metadata = iter(metadata)
+
+        for i, infile in enumerate(files):
+            mdata, fname = metadata.next(), names.next()
+            lpath = self._create_upload_path(mdata, fname, create_dirs=False)
+            if not hasattr(infile, 'read'):
+                infile = USBMS.normalize_path(infile)
+            book = Book(self.PREFIX, lpath, other=mdata)
+            length = self._put_file(infile, lpath, book, i, len(files))
+            if length < 0:
+                raise ControlError('Sending book %s to device failed' % lpath)
+            paths.append((lpath, length))
+            # No need to deal with covers. The client will get the thumbnails
+            # in the mi structure
+            self.report_progress((i+1) / float(len(files)), _('Transferring books to device...'))
+
+        self.report_progress(1.0, _('Transferring books to device...'))
+        self._debug('finished uploading %d books'%(len(files)))
+        return paths
+
+    @synchronous('sync_lock')
+    def add_books_to_metadata(self, locations, metadata, booklists):
+        self._debug('adding metadata for %d books'%(len(metadata)))
+
+        metadata = iter(metadata)
+        for i, location in enumerate(locations):
+            self.report_progress((i+1) / float(len(locations)),
+                                 _('Adding books to device metadata listing...'))
+            info = metadata.next()
+            lpath = location[0]
+            length = location[1]
+            lpath = self._strip_prefix(lpath)
+            book = Book(self.PREFIX, lpath, other=info)
+            if book.size is None:
+                book.size = length
+            b = booklists[0].add_book(book, replace_metadata=True)
+            if b:
+                b._new_book = True
+        self.report_progress(1.0, _('Adding books to device metadata listing...'))
+        self._debug('finished adding metadata')
+
+    @synchronous('sync_lock')
+    def delete_books(self, paths, end_session=True):
+        self._debug(paths)
+        for path in paths:
+            # the path has the prefix on it (I think)
+            path = self._strip_prefix(path)
+            opcode, result = self._call_client('DELETE_BOOK', {'lpath': path})
+            if opcode == 'OK':
+                self._debug('removed book with UUID', result['uuid'])
+            else:
+                raise ControlError('Protocol error - delete books')
+
+    @synchronous('sync_lock')
+    def remove_books_from_metadata(self, paths, booklists):
+        self._debug(paths)
+        for i, path in enumerate(paths):
+            path = self._strip_prefix(path)
+            self.report_progress((i+1) / float(len(paths)), _('Removing books from device metadata listing...'))
+            for bl in booklists:
+                for book in bl:
+                    if path == book.path:
+                        bl.remove_book(book)
+                        self._set_known_metadata(book, remove=True)
+        self.report_progress(1.0, _('Removing books from device metadata listing...'))
+        self._debug('finished removing metadata for %d books'%(len(paths)))
+
+
+    @synchronous('sync_lock')
+    def get_file(self, path, outfile, end_session=True):
+        self._debug(path)
+        eof = False
+        position = 0
+        while not eof:
+            opcode, result = self._call_client('GET_BOOK_FILE_SEGMENT',
+                                    {'lpath' : path, 'position': position},
+                                    print_debug_info=False )
+            if opcode == 'OK':
+                if not result['eof']:
+                    data = b64decode(result['data'])
+                    if len(data) != result['next_position'] - position:
+                        self._debug('position mismatch', result['next_position'], position)
+                    position = result['next_position']
+                    outfile.write(data)
+                else:
+                    eof = True
+            else:
+                raise ControlError('request for book data failed')
+
+    @synchronous('sync_lock')
+    def set_plugboards(self, plugboards, pb_func):
+        self._debug()
+        self.plugboards = plugboards
+        self.plugboard_func = pb_func
+
+    @synchronous('sync_lock')
+    def startup(self):
+        self.listen_socket = None
+
+    @synchronous('sync_lock')
+    def startup_on_demand(self):
+        if getattr(self, 'listen_socket', None) is not None:
+            # we are already running
+            return
+        if len(self.opcodes) != len(self.reverse_opcodes):
+            self._debug(self.opcodes, self.reverse_opcodes)
+        self.is_connected = False
+        self.listen_socket = None
+        self.device_socket = None
+        self.json_codec = JsonCodec()
+        self.known_metadata = {}
+        self.debug_time = time.time()
+        self.debug_start_time = time.time()
+        self.max_book_packet_len = 0
+        self.noop_counter = 0
+        self.connection_attempts = {}
+        try:
+            self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        except:
+            self._debug('creation of listen socket failed')
+            return
+
+        i = 0
+        while i < 100: # try up to 100 random port numbers
+            if self.settings().extra_customization[self.OPT_USE_PORT]:
+                i = 100
+                try:
+                    port = int(self.settings().extra_customization[self.OPT_PORT_NUMBER])
+                except:
+                    port = 0
+            else:
+                i += 1
+                port = random.randint(8192, 32000)
+            try:
+                self._debug('try port', port)
+                self.listen_socket.bind(('', port))
+                break
+            except socket.error:
+                port = 0
+            except:
+                self._debug('Unknown exception while allocating listen socket')
+                traceback.print_exc()
+                raise
+        if port == 0:
+            self._debug('Failed to allocate a port');
+            self.listen_socket.close()
+            self.listen_socket = None
+            self.is_connected = False
+            return
+
+        try:
+            self.listen_socket.listen(0)
+        except:
+            self._debug('listen on socket failed', port)
+            self.listen_socket.close()
+            self.listen_socket = None
+            self.is_connected = False
+            return
+
+        try:
+            do_zeroconf(publish_zeroconf, port)
+        except:
+            self._debug('registration with bonjour failed')
+            self.listen_socket.close()
+            self.listen_socket = None
+            self.is_connected = False
+            return
+
+        self._debug('listening on port', port)
+        self.port = port
+
+    @synchronous('sync_lock')
+    def shutdown(self):
+        if getattr(self, 'listen_socket', None) is not None:
+            do_zeroconf(unpublish_zeroconf, self.port)
+            self.listen_socket.close()
+            self.listen_socket = None
+            self.is_connected = False
+
+    # Methods for dynamic control
+
+    @synchronous('sync_lock')
+    def is_dynamically_controllable(self):
+        return 'smartdevice'
+
+    @synchronous('sync_lock')
+    def start_plugin(self):
+        self.startup_on_demand()
+
+    @synchronous('sync_lock')
+    def stop_plugin(self):
+        self.shutdown()
+
+    @synchronous('sync_lock')
+    def get_option(self, opt_string, default=None):
+        opt = self._get_smartdevice_option_number(opt_string)
+        if opt is not None:
+            return self.settings().extra_customization[opt]
+        return default
+
+    @synchronous('sync_lock')
+    def set_option(self, opt_string, value):
+        opt = self._get_smartdevice_option_number(opt_string)
+        if opt is not None:
+            config = self._configProxy()
+            ec = config['extra_customization']
+            ec[opt] = value
+            config['extra_customization'] = ec
+
+    @synchronous('sync_lock')
+    def is_running(self):
+        return getattr(self, 'listen_socket', None) is not None
+
+

src/calibre/ebooks/conversion/plugins/epub_input.py

@@ -198,11 +198,13 @@ class EPUBInput(InputFormatPlugin):
                 ('application/vnd.adobe-page-template+xml','application/text'):
                     not_for_spine.add(id_)
 
+        seen = set()
         for x in list(opf.iterspine()):
             ref = x.get('idref', None)
-            if ref is None or ref in not_for_spine:
+            if not ref or ref in not_for_spine or ref in seen:
                 x.getparent().remove(x)
                 continue
+            seen.add(ref)
 
         if len(list(opf.iterspine())) == 0:
             raise ValueError('No valid entries in the spine of this EPUB')

src/calibre/ebooks/conversion/plumber.py

@@ -326,7 +326,7 @@ OptionRecommendation(name='page_breaks_before',
             recommended_value="//*[name()='h1' or name()='h2']",
             level=OptionRecommendation.LOW,
             help=_('An XPath expression. Page breaks are inserted '
-            'before the specified elements.')
+                'before the specified elements. To disable use the expression: /')
         ),
 
 OptionRecommendation(name='remove_fake_margins',

src/calibre/ebooks/fb2/fb2ml.py

@@ -352,6 +352,7 @@ class FB2MLizer(object):
         @return: List of string representing the XHTML converted to FB2 markup.
         '''
         from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace
+        elem = elem_tree
 
         # Ensure what we are converting is not a string and that the fist tag is part of the XHTML namespace.
         if not isinstance(elem_tree.tag, basestring) or namespace(elem_tree.tag) != XHTML_NS:

src/calibre/ebooks/metadata/book/json_codec.py

@@ -117,8 +117,8 @@ class JsonCodec(object):
     def __init__(self):
         self.field_metadata = FieldMetadata()
 
-    def encode_to_file(self, file, booklist):
-        file.write(json.dumps(self.encode_booklist_metadata(booklist),
+    def encode_to_file(self, file_, booklist):
+        file_.write(json.dumps(self.encode_booklist_metadata(booklist),
                               indent=2, encoding='utf-8'))
 
     def encode_booklist_metadata(self, booklist):
@@ -156,21 +156,28 @@ class JsonCodec(object):
         else:
             return object_to_unicode(value)
 
-    def decode_from_file(self, file, booklist, book_class, prefix):
+    def decode_from_file(self, file_, booklist, book_class, prefix):
         js = []
         try:
-            js = json.load(file, encoding='utf-8')
+            js = json.load(file_, encoding='utf-8')
             for item in js:
-                book = book_class(prefix, item.get('lpath', None))
-                for key in item.keys():
-                    meta = self.decode_metadata(key, item[key])
+                booklist.append(self.raw_to_book(item, book_class, prefix))
+        except:
+            print 'exception during JSON decode_from_file'
+            traceback.print_exc()
+
+    def raw_to_book(self, json_book, book_class, prefix):
+        try:
+            book = book_class(prefix, json_book.get('lpath', None))
+            for key,val in json_book.iteritems():
+                meta = self.decode_metadata(key, val)
                 if key == 'user_metadata':
                     book.set_all_user_metadata(meta)
                 else:
                     if key == 'classifiers':
                         key = 'identifiers'
                     setattr(book, key, meta)
-                booklist.append(book)
+            return book
         except:
             print 'exception during JSON decoding'
             traceback.print_exc()

src/calibre/ebooks/metadata/odt.py

@@ -1,5 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+#
 # Copyright (C) 2006 Søren Roug, European Environment Agency
 #
 # This is free software.  You may redistribute it under the terms
@@ -17,12 +19,20 @@
 #
 # Contributor(s):
 #
+from __future__ import division
+
 import zipfile, re
 import xml.sax.saxutils
 from cStringIO import StringIO
 
 from odf.namespaces import OFFICENS, DCNS, METANS
-from calibre.ebooks.metadata import MetaInformation, string_to_authors
+from odf.opendocument import load as odLoad
+from odf.draw import Image as odImage, Frame as odFrame
+
+from calibre.ebooks.metadata import MetaInformation, string_to_authors, check_isbn
+from calibre.utils.magick.draw import identify_data
+from calibre.utils.date import parse_date
+from calibre.utils.localization import canonicalize_lang
 
 whitespace = re.compile(r'\s+')
 
@@ -125,6 +135,10 @@ class odfmetaparser(xml.sax.saxutils.XMLGenerator):
         else:
             texttag = self._tag
         self.seenfields[texttag] = self.data()
+        # OpenOffice has the habit to capitalize custom properties, so we add a
+        # lowercase version for easy access
+        if texttag[:4].lower() == u'opf.':
+            self.seenfields[texttag.lower()] = self.data()
 
         if field in self.deletefields:
             self.output.dowrite = True
@@ -141,7 +155,7 @@ class odfmetaparser(xml.sax.saxutils.XMLGenerator):
     def data(self):
         return normalize(''.join(self._data))
 
-def get_metadata(stream):
+def get_metadata(stream, extract_cover=True):
     zin = zipfile.ZipFile(stream, 'r')
     odfs = odfmetaparser()
     parser = xml.sax.make_parser()
@@ -162,7 +176,90 @@ def get_metadata(stream):
     if data.has_key('language'):
         mi.language = data['language']
     if data.get('keywords', ''):
-        mi.tags = data['keywords'].split(',')
+        mi.tags = [x.strip() for x in data['keywords'].split(',') if x.strip()]
+    opfmeta = False # we need this later for the cover
+    opfnocover = False
+    if data.get('opf.metadata','') == 'true':
+        # custom metadata contains OPF information
+        opfmeta = True
+        if data.get('opf.titlesort', ''):
+            mi.title_sort = data['opf.titlesort']
+        if data.get('opf.authors', ''):
+            mi.authors = string_to_authors(data['opf.authors'])
+        if data.get('opf.authorsort', ''):
+            mi.author_sort = data['opf.authorsort']
+        if data.get('opf.isbn', ''):
+            isbn = check_isbn(data['opf.isbn'])
+            if isbn is not None:
+                mi.isbn = isbn
+        if data.get('opf.publisher', ''):
+            mi.publisher = data['opf.publisher']
+        if data.get('opf.pubdate', ''):
+            mi.pubdate = parse_date(data['opf.pubdate'], assume_utc=True)
+        if data.get('opf.series', ''):
+            mi.series = data['opf.series']
+            if data.get('opf.seriesindex', ''):
+                try:
+                    mi.series_index = float(data['opf.seriesindex'])
+                except ValueError:
+                    mi.series_index = 1.0
+        if data.get('opf.language', ''):
+            cl = canonicalize_lang(data['opf.language'])
+            if cl:
+                mi.languages = [cl]
+        opfnocover = data.get('opf.nocover', 'false') == 'true'
+    if not opfnocover:
+        try:
+            read_cover(stream, zin, mi, opfmeta, extract_cover)
+        except:
+            pass # Do not let an error reading the cover prevent reading other data
 
     return mi
 
+def read_cover(stream, zin, mi, opfmeta, extract_cover):
+    # search for an draw:image in a draw:frame with the name 'opf.cover'
+    # if opf.metadata prop is false, just use the first image that
+    # has a proper size (borrowed from docx)
+    otext = odLoad(stream)
+    cover_href = None
+    cover_data = None
+    cover_frame = None
+    for frm in otext.topnode.getElementsByType(odFrame):
+        img = frm.getElementsByType(odImage)
+        if len(img) > 0: # there should be only one
+            i_href = img[0].getAttribute('href')
+            try:
+                raw = zin.read(i_href)
+            except KeyError:
+                continue
+            try:
+                width, height, fmt = identify_data(raw)
+            except:
+                continue
+        else:
+            continue
+        if opfmeta and frm.getAttribute('name').lower() == u'opf.cover':
+            cover_href = i_href
+            cover_data = (fmt, raw)
+            cover_frame = frm.getAttribute('name') # could have upper case
+            break
+        if cover_href is None and 0.8 <= height/width <= 1.8 and height*width >= 12000:
+            cover_href = i_href
+            cover_data = (fmt, raw)
+            if not opfmeta:
+                break
+
+    if cover_href is not None:
+        mi.cover = cover_href
+        mi.odf_cover_frame = cover_frame
+        if extract_cover:
+            if not cover_data:
+                raw = zin.read(cover_href)
+                try:
+                    width, height, fmt = identify_data(raw)
+                except:
+                    pass
+                else:
+                    cover_data = (fmt, raw)
+            mi.cover_data = cover_data
+

src/calibre/ebooks/metadata/opf2.py

@@ -286,15 +286,17 @@ class Spine(ResourceCollection): # {{{
     @staticmethod
     def from_opf_spine_element(itemrefs, manifest):
         s = Spine(manifest)
+        seen = set()
         for itemref in itemrefs:
             idref = itemref.get('idref', None)
             if idref is not None:
                 path = s.manifest.path_for_id(idref)
-                if path:
+                if path and path not in seen:
                     r = Spine.Item(lambda x:idref, path, is_path=True)
                     r.is_linear = itemref.get('linear', 'yes') == 'yes'
                     r.idref = idref
                     s.append(r)
+                    seen.add(path)
         return s
 
     @staticmethod

src/calibre/ebooks/mobi/reader/ncx.py

@@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import os
 
+from calibre import replace_entities
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.mobi.reader.headers import NULL_INDEX
 from calibre.ebooks.mobi.reader.index import read_index
@@ -88,7 +89,8 @@ def build_toc(index_entries):
     for lvl in sorted(levels):
         for item in level_map[lvl]:
             parent = num_map[item['parent']]
-            child = parent.add_item(item['href'], item['idtag'], item['text'])
+            child = parent.add_item(item['href'], item['idtag'],
+                    replace_entities(item['text'], encoding=None))
             num_map[item['num']] = child
 
     # Set play orders in depth first order

src/calibre/ebooks/mobi/writer2/serializer.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re
+import re, unicodedata
 
 from calibre.ebooks.oeb.base import (OEB_DOCS, XHTML, XHTML_NS, XML_NS,
         namespace, prefixname, urlnormalize)
@@ -355,6 +355,8 @@ class Serializer(object):
         text = text.replace(u'\u00AD', '') # Soft-hyphen
         if quot:
             text = text.replace('"', '&quot;')
+        if isinstance(text, unicode):
+            text = unicodedata.normalize('NFC', text)
         self.buf.write(text.encode('utf-8'))
 
     def fixup_links(self):

src/calibre/ebooks/mobi/writer8/skeleton.py

@@ -76,15 +76,13 @@ def tostring(raw, **kwargs):
 
 class Chunk(object):
 
-    def __init__(self, raw, parent_tag):
+    def __init__(self, raw, selector):
         self.raw = raw
         self.starts_tags = []
         self.ends_tags = []
         self.insert_pos = None
-        self.parent_tag = parent_tag
-        self.parent_is_body = False
-        self.is_last_chunk = False
         self.is_first_chunk = False
+        self.selector = "%s-//*[@aid='%s']"%selector
 
     def __len__(self):
         return len(self.raw)
@@ -97,11 +95,6 @@ class Chunk(object):
         return 'Chunk(len=%r insert_pos=%r starts_tags=%r ends_tags=%r)'%(
                 len(self.raw), self.insert_pos, self.starts_tags, self.ends_tags)
 
-    @property
-    def selector(self):
-        typ = 'S' if (self.is_last_chunk and not self.parent_is_body) else 'P'
-        return "%s-//*[@aid='%s']"%(typ, self.parent_tag)
-
     __str__ = __repr__
 
 class Skeleton(object):
@@ -251,13 +244,13 @@ class Chunker(object):
 
     def step_into_tag(self, tag, chunks):
         aid = tag.get('aid')
-        is_body = tag.tag == 'body'
+        self.chunk_selector = ('P', aid)
 
         first_chunk_idx = len(chunks)
 
         # First handle any text
         if tag.text and tag.text.strip(): # Leave pure whitespace in the skel
-            chunks.extend(self.chunk_up_text(tag.text, aid))
+            chunks.extend(self.chunk_up_text(tag.text))
             tag.text = None
 
         # Now loop over children
@@ -266,21 +259,21 @@ class Chunker(object):
             if child.tag == etree.Entity:
                 chunks.append(raw)
                 if child.tail:
-                    chunks.extend(self.chunk_up_text(child.tail, aid))
+                    chunks.extend(self.chunk_up_text(child.tail))
                 continue
             raw = close_self_closing_tags(raw)
             if len(raw) > CHUNK_SIZE and child.get('aid', None):
                 self.step_into_tag(child, chunks)
                 if child.tail and child.tail.strip(): # Leave pure whitespace
-                    chunks.extend(self.chunk_up_text(child.tail, aid))
+                    chunks.extend(self.chunk_up_text(child.tail))
                     child.tail = None
             else:
                 if len(raw) > CHUNK_SIZE:
                     self.log.warn('Tag %s has no aid and a too large chunk'
                             ' size. Adding anyway.'%child.tag)
-                chunks.append(Chunk(raw, aid))
+                chunks.append(Chunk(raw, self.chunk_selector))
                 if child.tail:
-                    chunks.extend(self.chunk_up_text(child.tail, aid))
+                    chunks.extend(self.chunk_up_text(child.tail))
                 tag.remove(child)
 
         if len(chunks) <= first_chunk_idx and chunks:
@@ -293,12 +286,9 @@ class Chunker(object):
             my_chunks = chunks[first_chunk_idx:]
             if my_chunks:
                 my_chunks[0].is_first_chunk = True
-                my_chunks[-1].is_last_chunk = True
-                if is_body:
-                    for chunk in my_chunks:
-                        chunk.parent_is_body = True
+        self.chunk_selector = ('S', aid)
 
-    def chunk_up_text(self, text, parent_tag):
+    def chunk_up_text(self, text):
         text = text.encode('utf-8')
         ans = []
 
@@ -314,7 +304,7 @@ class Chunker(object):
         while rest:
             start, rest = split_multibyte_text(rest)
             ans.append(b'<span class="AmznBigTextBlock">' + start + '</span>')
-        return [Chunk(x, parent_tag) for x in ans]
+        return [Chunk(x, self.chunk_selector) for x in ans]
 
     def merge_small_chunks(self, chunks):
         ans = chunks[:1]

src/calibre/ebooks/odt/input.py

@@ -10,6 +10,9 @@ import os
 
 from lxml import etree
 from odf.odf2xhtml import ODF2XHTML
+from odf.opendocument import load as odLoad
+from odf.draw import Frame as odFrame, Image as odImage
+from odf.namespaces import TEXTNS as odTEXTNS
 
 from calibre import CurrentDir, walk
 
@@ -138,22 +141,84 @@ class Extract(ODF2XHTML):
                 r.selectorText = '.'+replace_name
         return sheet.cssText, sel_map
 
+    def search_page_img(self, mi, log):
+        for frm in self.document.topnode.getElementsByType(odFrame):
+            try:
+                if frm.getAttrNS(odTEXTNS,u'anchor-type') == 'page':
+                    log.warn('Document has Pictures anchored to Page, will all end up before first page!')
+                    break
+            except ValueError:
+                pass
+
+    def filter_cover(self, mi, log):
+        # filter the Element tree (remove the detected cover)
+        if mi.cover and mi.odf_cover_frame:
+            for frm in self.document.topnode.getElementsByType(odFrame):
+                # search the right frame
+                if frm.getAttribute('name') == mi.odf_cover_frame:
+                    img = frm.getElementsByType(odImage)
+                    # only one draw:image allowed in the draw:frame
+                    if len(img) == 1 and img[0].getAttribute('href') == mi.cover:
+                        # ok, this is the right frame with the right image
+                        # check if there are more childs
+                        if len(frm.childNodes) != 1:
+                            break
+                        # check if the parent paragraph more childs
+                        para = frm.parentNode
+                        if para.tagName != 'text:p' or len(para.childNodes) != 1:
+                            break
+                        # now it should be safe to remove the text:p
+                        parent = para.parentNode
+                        parent.removeChild(para)
+                        log("Removed cover image paragraph from document...")
+                        break
+
+    def filter_load(self, odffile, mi, log):
+        """ This is an adaption from ODF2XHTML. It adds a step between
+            load and parse of the document where the Element tree can be
+            modified.
+        """
+        # first load the odf structure
+        self.lines = []
+        self._wfunc = self._wlines
+        if isinstance(odffile, basestring) \
+                or hasattr(odffile, 'read'): # Added by Kovid
+            self.document = odLoad(odffile)
+        else:
+            self.document = odffile
+        # filter stuff
+        self.search_page_img(mi, log)
+        try:
+            self.filter_cover(mi, log)
+        except:
+            pass
+        # parse the modified tree and generate xhtml
+        self._walknode(self.document.topnode)
+
     def __call__(self, stream, odir, log):
         from calibre.utils.zipfile import ZipFile
-        from calibre.ebooks.metadata.meta import get_metadata
+        from calibre.ebooks.metadata.odt import get_metadata
         from calibre.ebooks.metadata.opf2 import OPFCreator
 
-
         if not os.path.exists(odir):
             os.makedirs(odir)
         with CurrentDir(odir):
             log('Extracting ODT file...')
-            html = self.odf2xhtml(stream)
+            stream.seek(0)
+            mi = get_metadata(stream, 'odt')
+            if not mi.title:
+                mi.title = _('Unknown')
+            if not mi.authors:
+                mi.authors = [_('Unknown')]
+            self.filter_load(stream, mi, log)
+            html = self.xhtml()
             # A blanket img specification like this causes problems
             # with EPUB output as the containing element often has
             # an absolute height and width set that is larger than
             # the available screen real estate
             html = html.replace('img { width: 100%; height: 100%; }', '')
+            # odf2xhtml creates empty title tag
+            html = html.replace('<title></title>','<title>%s</title>'%(mi.title,))
             try:
                 html = self.fix_markup(html, log)
             except:
@@ -162,12 +227,6 @@ class Extract(ODF2XHTML):
                 f.write(html.encode('utf-8'))
             zf = ZipFile(stream, 'r')
             self.extract_pictures(zf)
-            stream.seek(0)
-            mi = get_metadata(stream, 'odt')
-            if not mi.title:
-                mi.title = _('Unknown')
-            if not mi.authors:
-                mi.authors = [_('Unknown')]
             opf = OPFCreator(os.path.abspath(os.getcwdu()), mi)
             opf.create_manifest([(os.path.abspath(f), None) for f in
                 walk(os.getcwdu())])

src/calibre/ebooks/oeb/display/full_screen.coffee

@@ -0,0 +1,65 @@
+#!/usr/bin/env coffee
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+###
+ Copyright 2012, Kovid Goyal <kovid at kovidgoyal.net>
+ Released under the GPLv3 License
+###
+
+
+log = window.calibre_utils.log
+
+class FullScreen
+    # This class is a namespace to expose functions via the
+    # window.full_screen object. The most important functions are:
+
+    constructor: () ->
+        if not this instanceof arguments.callee
+            throw new Error('FullScreen constructor called as function')
+        this.in_full_screen = false
+        this.initial_left_margin = null
+        this.initial_right_margin = null
+
+    save_margins: () ->
+        bs = document.body.style
+        this.initial_left_margin = bs.marginLeft
+        this.initial_right_margin = bs.marginRight
+
+    on: (max_text_width, in_paged_mode) ->
+        if in_paged_mode
+            window.paged_display.max_col_width = max_text_width
+        else
+            s = document.body.style
+            s.maxWidth = max_text_width + 'px'
+            s.marginLeft = 'auto'
+            s.marginRight = 'auto'
+        window.addEventListener('click', this.handle_click, false)
+
+    off: (in_paged_mode) ->
+        window.removeEventListener('click', this.handle_click, false)
+        if in_paged_mode
+            window.paged_display.max_col_width = -1
+        else
+            s = document.body.style
+            s.maxWidth = 'none'
+            if this.initial_left_margin != null
+                s.marginLeft = this.initial_left_margin
+            if this.initial_right_margin != null
+                s.marginRight = this.initial_right_margin
+
+    handle_click: (event) ->
+        if event.target != document.documentElement or event.button != 0
+            return
+        res = null
+        if window.paged_display.in_paged_mode
+            res = window.paged_display.click_for_page_turn(event)
+        else
+            br = document.body.getBoundingClientRect()
+            if not (br.left <= event.clientX <= br.right)
+                res = event.clientX < br.left
+        if res != null
+            window.py_bridge.page_turn_requested(res)
+
+if window?
+    window.full_screen = new FullScreen()
+

src/calibre/ebooks/oeb/display/paged.coffee

@@ -26,6 +26,7 @@ class PagedDisplay
         this.current_margin_side = 0
         this.is_full_screen_layout = false
         this.max_col_width = -1
+        this.current_page_height = null
         this.document_margins = null
         this.use_document_margins = false
 
@@ -74,25 +75,12 @@ class PagedDisplay
         # start_time = new Date().getTime()
         body_style = window.getComputedStyle(document.body)
         bs = document.body.style
-        # When laying body out in columns, webkit bleeds the top margin of the
-        # first block element out above the columns, leading to an extra top
-        # margin for the page. We compensate for that here. Computing the
-        # boundingrect of body is very expensive with column layout, so we do
-        # it before the column layout is applied.
         first_layout = false
         if not this.in_paged_mode
-            bs.setProperty('margin-top', '0px')
-            extra_margin = document.body.getBoundingClientRect().top
-            if extra_margin <= this.margin_top
-                extra_margin = 0
-            margin_top = (this.margin_top - extra_margin) + 'px'
             # Check if the current document is a full screen layout like
             # cover, if so we treat it specially.
             single_screen = (document.body.scrollWidth < window.innerWidth + 25 and document.body.scrollHeight < window.innerHeight + 25)
             first_layout = true
-        else
-            # resize event
-            margin_top = body_style.marginTop
 
         ww = window.innerWidth
 
@@ -116,16 +104,23 @@ class PagedDisplay
             col_width = Math.max(100, ((ww - adjust)/n) - 2*sm)
         this.page_width = col_width + 2*sm
         this.screen_width = this.page_width * this.cols_per_screen
+        this.current_page_height = window.innerHeight - this.margin_top - this.margin_bottom
 
         fgcolor = body_style.getPropertyValue('color')
 
         bs.setProperty('-webkit-column-gap', (2*sm)+'px')
         bs.setProperty('-webkit-column-width', col_width+'px')
         bs.setProperty('-webkit-column-rule-color', fgcolor)
+
+        # Without this, webkit bleeds the margin of the first block(s) of body
+        # above the columns, which causes them to effectively be added to the
+        # page margins (the margin collapse algorithm)
+        bs.setProperty('-webkit-margin-collapse', 'separate')
+
         bs.setProperty('overflow', 'visible')
         bs.setProperty('height', (window.innerHeight - this.margin_top - this.margin_bottom) + 'px')
         bs.setProperty('width', (window.innerWidth - 2*sm)+'px')
-        bs.setProperty('margin-top', margin_top)
+        bs.setProperty('margin-top', this.margin_top + 'px')
         bs.setProperty('margin-bottom', this.margin_bottom+'px')
         bs.setProperty('margin-left', sm+'px')
         bs.setProperty('margin-right', sm+'px')
@@ -167,9 +162,15 @@ class PagedDisplay
         # that this method use getBoundingClientRect() which means it will
         # force a relayout if the render tree is dirty.
         images = []
+        vimages = []
+        maxh = this.current_page_height
         for img in document.getElementsByTagName('img')
             previously_limited = calibre_utils.retrieve(img, 'width-limited', false)
+            data = calibre_utils.retrieve(img, 'img-data', null)
             br = img.getBoundingClientRect()
+            if data == null
+                data = {'left':br.left, 'right':br.right, 'height':br.height, 'display': img.style.display}
+                calibre_utils.store(img, 'img-data', data)
             left = calibre_utils.viewport_to_document(br.left, 0, doc=img.ownerDocument)[0]
             col = this.column_at(left) * this.page_width
             rleft = left - col - this.current_margin_side
@@ -178,23 +179,28 @@ class PagedDisplay
             col_width = this.page_width - 2*this.current_margin_side
             if previously_limited or rright > col_width
                 images.push([img, col_width - rleft])
+            previously_limited = calibre_utils.retrieve(img, 'height-limited', false)
+            if previously_limited or br.height > maxh
+                vimages.push(img)
+            if previously_limited
+                img.style.setProperty('-webkit-column-break-before', 'auto')
+                img.style.setProperty('display', data.display)
+            img.style.setProperty('-webkit-column-break-inside', 'avoid')
 
         for [img, max_width] in images
             img.style.setProperty('max-width', max_width+'px')
             calibre_utils.store(img, 'width-limited', true)
 
-    check_top_margin: () ->
-        # This is needed to handle the case when a descendant of body specifies
-        # a top margin as a percentage, which messes up the top margin
-        # calculations above
-        tm = document.body.getBoundingClientRect().top
-        if tm != this.margin_top
-            document.body.style.setProperty('margin-top', '0px')
-            tm = document.body.getBoundingClientRect().top
-            if tm <= this.margin_top
-                tm = 0
-            m = this.margin_top - tm
-            document.body.style.setProperty('margin-top', m+'px')
+        for img in vimages
+            data = calibre_utils.retrieve(img, 'img-data', null)
+            img.style.setProperty('-webkit-column-break-before', 'always')
+            img.style.setProperty('max-height', maxh+'px')
+            if data.height > maxh
+                # This is needed to force the image onto a new page, without
+                # it, the webkit algorithm may still decide to split the image
+                # by keeping it part of its parent block
+                img.style.setProperty('display', 'block')
+            calibre_utils.store(img, 'height-limited', true)
 
     scroll_to_pos: (frac) ->
         # Scroll to the position represented by frac (number between 0 and 1)
@@ -395,6 +401,18 @@ class PagedDisplay
             log('Viewport cfi:', ans)
         return ans
 
+    click_for_page_turn: (event) ->
+        # Check if the click event event should generate a apge turn. Returns
+        # null if it should not, true if it is a backwards page turn, false if
+        # it is a forward apge turn.
+        left_boundary = this.current_margin_side
+        right_bondary = this.screen_width - this.current_margin_side
+        if left_boundary > event.clientX
+            return true
+        if right_bondary < event.clientX
+            return false
+        return null
+
 if window?
     window.paged_display = new PagedDisplay()
 

src/calibre/ebooks/oeb/transforms/structure.py

@@ -82,10 +82,17 @@ class DetectStructure(object):
 
     def detect_chapters(self):
         self.detected_chapters = []
+
+        def find_matches(expr, doc):
+            try:
+                return XPath(expr)(doc)
+            except:
+                self.log.warn('Invalid chapter expression, ignoring: %s'%expr)
+                return []
+
         if self.opts.chapter:
-            chapter_xpath = XPath(self.opts.chapter)
             for item in self.oeb.spine:
-                for x in chapter_xpath(item.data):
+                for x in find_matches(self.opts.chapter, item.data):
                     self.detected_chapters.append((item, x))
 
             chapter_mark = self.opts.chapter_mark
@@ -164,11 +171,19 @@ class DetectStructure(object):
         added = OrderedDict()
         added2 = OrderedDict()
         counter = 1
+
+        def find_matches(expr, doc):
+            try:
+                return XPath(expr)(doc)
+            except:
+                self.log.warn('Invalid ToC expression, ignoring: %s'%expr)
+                return []
+
         for document in self.oeb.spine:
             previous_level1 = list(added.itervalues())[-1] if added else None
             previous_level2 = list(added2.itervalues())[-1] if added2 else None
 
-            for elem in XPath(self.opts.level1_toc)(document.data):
+            for elem in find_matches(self.opts.level1_toc, document.data):
                 text, _href = self.elem_to_link(document, elem, counter)
                 counter += 1
                 if text:
@@ -178,7 +193,7 @@ class DetectStructure(object):
                     #node.add(_('Top'), _href)
 
             if self.opts.level2_toc is not None and added:
-                for elem in XPath(self.opts.level2_toc)(document.data):
+                for elem in find_matches(self.opts.level2_toc, document.data):
                     level1 = None
                     for item in document.data.iterdescendants():
                         if item in added:
@@ -196,7 +211,8 @@ class DetectStructure(object):
                             break
 
                 if self.opts.level3_toc is not None and added2:
-                    for elem in XPath(self.opts.level3_toc)(document.data):
+                    for elem in find_matches(self.opts.level3_toc,
+                            document.data):
                         level2 = None
                         for item in document.data.iterdescendants():
                             if item in added2:

src/calibre/ebooks/pdf/writer.py

@@ -202,7 +202,6 @@ class PDFWriter(QObject): # {{{
         paged_display.set_geometry(1, 0, 0, 0);
         paged_display.layout();
         paged_display.fit_images();
-        paged_display.check_top_margin();
         ''')
         mf = self.view.page().mainFrame()
         while True:
@@ -221,7 +220,7 @@ class PDFWriter(QObject): # {{{
             self.tmp_path = PersistentTemporaryDirectory('_pdf_output_parts')
 
     def insert_cover(self):
-        if self.cover_data is None:
+        if not isinstance(self.cover_data, bytes):
             return
         item_path = os.path.join(self.tmp_path, 'cover.pdf')
         printer = get_pdf_printer(self.opts, output_file_name=item_path,

src/calibre/ebooks/pml/pmlml.py

@@ -220,7 +220,7 @@ class PMLMLizer(object):
     def dump_text(self, elem, stylizer, page, tag_stack=[]):
         from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace
 
-        if not isinstance(elem_tree.tag, basestring) or namespace(elem_tree.tag) != XHTML_NS:
+        if not isinstance(elem.tag, basestring) or namespace(elem.tag) != XHTML_NS:
             p = elem.getparent()
             if p is not None and isinstance(p.tag, basestring) and namespace(p.tag) == XHTML_NS \
                     and elem.tail:

src/calibre/ebooks/rb/rbml.py

@@ -142,7 +142,7 @@ class RBMLizer(object):
     def dump_text(self, elem, stylizer, page, tag_stack=[]):
         from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace
 
-        if not isinstance(elem_tree.tag, basestring) or namespace(elem_tree.tag) != XHTML_NS:
+        if not isinstance(elem.tag, basestring) or namespace(elem.tag) != XHTML_NS:
             p = elem.getparent()
             if p is not None and isinstance(p.tag, basestring) and namespace(p.tag) == XHTML_NS \
                     and elem.tail:

src/calibre/gui2/actions/delete.py

@@ -139,6 +139,21 @@ class DeleteAction(InterfaceAction):
             return set([])
         return set(map(self.gui.library_view.model().id, rows))
 
+    def remove_format_by_id(self, book_id, fmt):
+        title = self.gui.current_db.title(book_id, index_is_id=True)
+        if not confirm('<p>'+(_(
+            'The %(fmt)s format will be <b>permanently deleted</b> from '
+            '%(title)s. Are you sure?')%dict(fmt=fmt, title=title))
+                            +'</p>', 'library_delete_specific_format', self.gui):
+            return
+
+        self.gui.library_view.model().db.remove_format(book_id, fmt,
+                index_is_id=True, notify=False)
+        self.gui.library_view.model().refresh_ids([book_id])
+        self.gui.library_view.model().current_changed(self.gui.library_view.currentIndex(),
+                self.gui.library_view.currentIndex())
+        self.gui.tags_view.recount()
+
     def delete_selected_formats(self, *args):
         ids = self._get_selected_ids()
         if not ids:

src/calibre/gui2/actions/device.py

@@ -14,7 +14,8 @@ from calibre.utils.smtp import config as email_config
 from calibre.constants import iswindows, isosx
 from calibre.customize.ui import is_disabled
 from calibre.devices.bambook.driver import BAMBOOK
-from calibre.gui2 import info_dialog
+from calibre.gui2.dialogs.smartdevice import SmartdeviceDialog
+from calibre.gui2 import info_dialog, question_dialog
 
 class ShareConnMenu(QMenu): # {{{
 
@@ -24,8 +25,12 @@ class ShareConnMenu(QMenu): # {{{
 
     config_email = pyqtSignal()
     toggle_server = pyqtSignal()
+    control_smartdevice = pyqtSignal()
     dont_add_to = frozenset(['context-menu-device'])
 
+    DEVICE_MSGS = [_('Start wireless device connection'),
+            _('Stop wireless device connection')]
+
     def __init__(self, parent=None):
         QMenu.__init__(self, parent)
         mitem = self.addAction(QIcon(I('devices/folder.png')), _('Connect to folder'))
@@ -56,6 +61,11 @@ class ShareConnMenu(QMenu): # {{{
             _('Start Content Server'))
         self.toggle_server_action.triggered.connect(lambda x:
                 self.toggle_server.emit())
+        self.control_smartdevice_action = \
+            self.addAction(QIcon(I('dot_red.png')),
+            self.DEVICE_MSGS[0])
+        self.control_smartdevice_action.triggered.connect(lambda x:
+                self.control_smartdevice.emit())
         self.addSeparator()
 
         self.email_actions = []
@@ -80,6 +90,9 @@ class ShareConnMenu(QMenu): # {{{
             text = _('Stop Content Server') + ' [%s]'%get_external_ip()
         self.toggle_server_action.setText(text)
 
+    def hide_smartdevice_menus(self):
+        self.control_smartdevice_action.setVisible(False)
+
     def build_email_entries(self, sync_menu):
         from calibre.gui2.device import DeviceAction
         for ac in self.email_actions:
@@ -158,6 +171,7 @@ class ConnectShareAction(InterfaceAction):
     def genesis(self):
         self.share_conn_menu = ShareConnMenu(self.gui)
         self.share_conn_menu.toggle_server.connect(self.toggle_content_server)
+        self.share_conn_menu.control_smartdevice.connect(self.control_smartdevice)
         self.share_conn_menu.config_email.connect(partial(
             self.gui.iactions['Preferences'].do_config,
             initial_plugin=('Sharing', 'Email')))
@@ -200,8 +214,37 @@ class ConnectShareAction(InterfaceAction):
             if not self.stopping_msg.isVisible():
                 self.stopping_msg.exec_()
             return
-
-
         self.gui.content_server = None
         self.stopping_msg.accept()
 
+    def control_smartdevice(self):
+        dm = self.gui.device_manager
+        running = dm.is_running('smartdevice')
+        if running:
+            dm.stop_plugin('smartdevice')
+            if dm.get_option('smartdevice', 'autostart'):
+                if not question_dialog(self.gui, _('Disable autostart'),
+                        _('Do you want wireless device connections to be'
+                            ' started automatically when calibre starts?')):
+                    dm.set_option('smartdevice', 'autostart', False)
+        else:
+            sd_dialog = SmartdeviceDialog(self.gui)
+            sd_dialog.exec_()
+        self.set_smartdevice_action_state()
+
+    def check_smartdevice_menus(self):
+        if not self.gui.device_manager.is_enabled('smartdevice'):
+            self.share_conn_menu.hide_smartdevice_menus()
+
+    def set_smartdevice_action_state(self):
+        from calibre.utils.mdns import get_external_ip
+        running = self.gui.device_manager.is_running('smartdevice')
+        if not running:
+            text = self.share_conn_menu.DEVICE_MSGS[0]
+        else:
+            text = self.share_conn_menu.DEVICE_MSGS[1]  + ' [%s]'%get_external_ip()
+        icon = 'green' if running else 'red'
+        ac = self.share_conn_menu.control_smartdevice_action
+        ac.setIcon(QIcon(I('dot_%s.png'%icon)))
+        ac.setText(text)
+

src/calibre/gui2/actions/plugin_updates.py

@@ -31,3 +31,5 @@ class PluginUpdaterAction(InterfaceAction):
 
         d = PluginUpdaterDialog(self.gui, initial_filter=initial_filter)
         d.exec_()
+        if d.do_restart:
+            self.gui.quit(restart=True)

src/calibre/gui2/actions/preferences.py

@@ -45,6 +45,8 @@ class PreferencesAction(InterfaceAction):
         d = PluginUpdaterDialog(self.gui,
                 initial_filter=FILTER_NOT_INSTALLED)
         d.exec_()
+        if d.do_restart:
+            self.gui.quit(restart=True)
 
     def do_config(self, checked=False, initial_plugin=None,
             close_after_initial=False):

src/calibre/gui2/actions/save_to_disk.py

@@ -73,7 +73,9 @@ class SaveToDiskAction(InterfaceAction):
         self.save_to_disk(False, single_dir=True,
                 single_format=prefs['output_format'])
 
-    def save_to_disk(self, checked, single_dir=False, single_format=None):
+    def save_to_disk(self, checked, single_dir=False, single_format=None,
+            rows=None, write_opf=None, save_cover=None):
+        if rows is None:
             rows = self.gui.current_view().selectionModel().selectedRows()
         if not rows or len(rows) == 0:
             return error_dialog(self.gui, _('Cannot save to disk'),
@@ -105,6 +107,10 @@ class SaveToDiskAction(InterfaceAction):
                     opts.write_opf = False
                     opts.template = opts.send_template
             opts.single_dir = single_dir
+            if write_opf is not None:
+                opts.write_opf = write_opf
+            if save_cover is not None:
+                opts.save_cover = save_cover
             self._saver = Saver(self.gui, self.gui.library_view.model().db,
                     Dispatcher(self._books_saved), rows, path, opts,
                     spare_server=self.gui.spare_server)
@@ -114,6 +120,13 @@ class SaveToDiskAction(InterfaceAction):
             self.gui.device_manager.save_books(
                     Dispatcher(self.books_saved), paths, path)
 
+    def save_library_format_by_ids(self, book_ids, fmt, single_dir=True):
+        if isinstance(book_ids, int):
+            book_ids = [book_ids]
+        rows = list(self.gui.library_view.ids_to_rows(book_ids).itervalues())
+        rows = [self.gui.library_view.model().index(r, 0) for r in rows]
+        self.save_to_disk(True, single_dir=single_dir, single_format=fmt,
+                rows=rows, write_opf=False, save_cover=False)
 
     def _books_saved(self, path, failures, error):
         self._saver = None

src/calibre/gui2/book_details.py

@@ -5,8 +5,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import (QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl,
-    QPropertyAnimation, QEasingCurve, QApplication, QFontInfo,
+from PyQt4.Qt import (QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, QIcon,
+    QPropertyAnimation, QEasingCurve, QApplication, QFontInfo, QAction,
     QSizePolicy, QPainter, QRect, pyqtProperty, QLayout, QPalette, QMenu)
 from PyQt4.QtWebKit import QWebView
 
@@ -382,6 +382,8 @@ class CoverView(QWidget): # {{{
 class BookInfo(QWebView):
 
     link_clicked = pyqtSignal(object)
+    remove_format = pyqtSignal(int, object)
+    save_format = pyqtSignal(int, object)
 
     def __init__(self, vertical, parent=None):
         QWebView.__init__(self, parent)
@@ -395,6 +397,23 @@ class BookInfo(QWebView):
         palette.setBrush(QPalette.Base, Qt.transparent)
         self.page().setPalette(palette)
         self.css = P('templates/book_details.css', data=True).decode('utf-8')
+        for x, icon in [('remove', 'trash.png'), ('save', 'save.png')]:
+            ac = QAction(QIcon(I(icon)), '', self)
+            ac.current_fmt = None
+            ac.triggered.connect(getattr(self, '%s_format_triggerred'%x))
+            setattr(self, '%s_format_action'%x, ac)
+
+    def context_action_triggered(self, which):
+        f = getattr(self, '%s_format_action'%which).current_fmt
+        if f:
+            book_id, fmt = f
+            getattr(self, '%s_format'%which).emit(book_id, fmt)
+
+    def remove_format_triggerred(self):
+        self.context_action_triggered('remove')
+
+    def save_format_triggerred(self):
+        self.context_action_triggered('save')
 
     def link_activated(self, link):
         self._link_clicked = True
@@ -420,6 +439,34 @@ class BookInfo(QWebView):
         else:
             ev.ignore()
 
+    def contextMenuEvent(self, ev):
+        p = self.page()
+        mf = p.mainFrame()
+        r = mf.hitTestContent(ev.pos())
+        url = unicode(r.linkUrl().toString()).strip()
+        menu = p.createStandardContextMenu()
+        ca = self.pageAction(p.Copy)
+        for action in list(menu.actions()):
+            if action is not ca:
+                menu.removeAction(action)
+        if not r.isNull() and url.startswith('format:'):
+            parts = url.split(':')
+            try:
+                book_id, fmt = int(parts[1]), parts[2]
+            except:
+                import traceback
+                traceback.print_exc()
+            else:
+                for a, t in [('remove', _('Delete the %s format')),
+                    ('save', _('Save the %s format to disk'))]:
+                    ac = getattr(self, '%s_format_action'%a)
+                    ac.current_fmt = (book_id, fmt)
+                    ac.setText(t%parts[2])
+                    menu.addAction(ac)
+        if len(menu.actions()) > 0:
+            menu.exec_(ev.globalPos())
+
+
 # }}}
 
 class DetailsLayout(QLayout): # {{{
@@ -513,6 +560,8 @@ class BookDetails(QWidget): # {{{
     show_book_info = pyqtSignal()
     open_containing_folder = pyqtSignal(int)
     view_specific_format = pyqtSignal(int, object)
+    remove_specific_format = pyqtSignal(int, object)
+    save_specific_format = pyqtSignal(int, object)
     remote_file_dropped = pyqtSignal(object, object)
     files_dropped = pyqtSignal(object, object)
     cover_changed = pyqtSignal(object, object)
@@ -579,6 +628,8 @@ class BookDetails(QWidget): # {{{
         self.book_info = BookInfo(vertical, self)
         self._layout.addWidget(self.book_info)
         self.book_info.link_clicked.connect(self.handle_click)
+        self.book_info.remove_format.connect(self.remove_specific_format)
+        self.book_info.save_format.connect(self.save_specific_format)
         self.setCursor(Qt.PointingHandCursor)
 
     def handle_click(self, link):

src/calibre/gui2/comments_editor.py

@@ -8,6 +8,7 @@ __docformat__ = 'restructuredtext en'
 import re, os
 
 from lxml import html
+import sip
 
 from PyQt4.Qt import (QApplication, QFontInfo, QSize, QWidget, QPlainTextEdit,
     QToolBar, QVBoxLayout, QAction, QIcon, Qt, QTabWidget, QUrl,
@@ -42,6 +43,7 @@ class PageAction(QAction): # {{{
         self.page_action.trigger()
 
     def update_state(self, *args):
+        if sip.isdeleted(self) or sip.isdeleted(self.page_action): return
         if self.isCheckable():
             self.setChecked(self.page_action.isChecked())
         self.setEnabled(self.page_action.isEnabled())

src/calibre/gui2/convert/bulk.py

@@ -4,7 +4,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import shutil, importlib
+import shutil
 
 from PyQt4.Qt import QString, SIGNAL
 
@@ -86,17 +86,9 @@ class BulkConfig(Config):
         sd = widget_factory(StructureDetectionWidget)
         toc = widget_factory(TOCWidget)
 
-        output_widget = None
-        name = self.plumber.output_plugin.name.lower().replace(' ', '_')
-        try:
-            output_widget = importlib.import_module(
-                    'calibre.gui2.convert.'+name)
-            pw = output_widget.PluginWidget
-            pw.ICON = I('back.png')
-            pw.HELP = _('Options specific to the output format.')
-            output_widget = widget_factory(pw)
-        except ImportError:
-            pass
+        output_widget = self.plumber.output_plugin.gui_configuration_widget(
+                self.stack, self.plumber.get_option_by_name,
+                self.plumber.get_option_help, self.db)
 
         while True:
             c = self.stack.currentWidget()

src/calibre/gui2/convert/single.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import cPickle, shutil, importlib
+import cPickle, shutil
 
 from PyQt4.Qt import QString, SIGNAL, QAbstractListModel, Qt, QVariant, QFont
 
@@ -187,29 +187,12 @@ class Config(ResizableDialog, Ui_Dialog):
         toc = widget_factory(TOCWidget)
         debug = widget_factory(DebugWidget)
 
-        output_widget = None
-        name = self.plumber.output_plugin.name.lower().replace(' ', '_')
-        try:
-            output_widget = importlib.import_module(
-                    'calibre.gui2.convert.'+name)
-            pw = output_widget.PluginWidget
-            pw.ICON = I('back.png')
-            pw.HELP = _('Options specific to the output format.')
-            output_widget = widget_factory(pw)
-        except ImportError:
-            pass
-        input_widget = None
-        name = self.plumber.input_plugin.name.lower().replace(' ', '_')
-        try:
-            input_widget = importlib.import_module(
-                    'calibre.gui2.convert.'+name)
-            pw = input_widget.PluginWidget
-            pw.ICON = I('forward.png')
-            pw.HELP = _('Options specific to the input format.')
-            input_widget = widget_factory(pw)
-        except ImportError:
-            pass
-
+        output_widget = self.plumber.output_plugin.gui_configuration_widget(
+                self.stack, self.plumber.get_option_by_name,
+                self.plumber.get_option_help, self.db, self.book_id)
+        input_widget = self.plumber.input_plugin.gui_configuration_widget(
+                self.stack, self.plumber.get_option_by_name,
+                self.plumber.get_option_help, self.db, self.book_id)
         while True:
             c = self.stack.currentWidget()
             if not c: break

src/calibre/gui2/device.py

@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 # Imports {{{
 import os, traceback, Queue, time, cStringIO, re, sys
-from threading import Thread
+from threading import Thread, Event
 
 from PyQt4.Qt import (QMenu, QAction, QActionGroup, QIcon, SIGNAL,
                      Qt, pyqtSignal, QDialog, QObject, QVBoxLayout,
@@ -13,7 +13,8 @@ from PyQt4.Qt import (QMenu, QAction, QActionGroup, QIcon, SIGNAL,
 from calibre.customize.ui import (available_input_formats, available_output_formats,
     device_plugins)
 from calibre.devices.interface import DevicePlugin
-from calibre.devices.errors import UserFeedback, OpenFeedback
+from calibre.devices.errors import (UserFeedback, OpenFeedback, OpenFailed,
+                                    InitialConnectionError)
 from calibre.gui2.dialogs.choose_format_device import ChooseFormatDeviceDialog
 from calibre.utils.ipc.job import BaseJob
 from calibre.devices.scanner import DeviceScanner
@@ -144,6 +145,9 @@ class DeviceManager(Thread): # {{{
         self.open_feedback_msg = open_feedback_msg
         self._device_information = None
         self.current_library_uuid = None
+        self.call_shutdown_on_disconnect = False
+        self.devices_initialized = Event()
+        self.dynamic_plugins = {}
 
     def report_progress(self, *args):
         pass
@@ -169,6 +173,8 @@ class DeviceManager(Thread): # {{{
                     self.open_feedback_msg(dev.get_gui_name(), e)
                     self.ejected_devices.add(dev)
                 continue
+            except OpenFailed:
+                raise
             except:
                 tb = traceback.format_exc()
                 if DEBUG or tb not in self.reported_errors:
@@ -197,6 +203,13 @@ class DeviceManager(Thread): # {{{
             self.ejected_devices.remove(self.connected_device)
         else:
             self.connected_slot(False, self.connected_device_kind)
+        if self.call_shutdown_on_disconnect:
+            # The current device is an instance of a plugin class instantiated
+            # to handle this connection, probably as a mounted device. We are
+            # now abandoning the instance that we created, so we tell it that it
+            # is being shut down.
+            self.connected_device.shutdown()
+            self.call_shutdown_on_disconnect = False
         self.connected_device = None
         self._device_information = None
 
@@ -215,12 +228,17 @@ class DeviceManager(Thread): # {{{
                             only_presence=True, debug=True)
                 self.connected_device_removed()
         else:
+            try:
                 possibly_connected_devices = []
                 for device in self.devices:
                     if device in self.ejected_devices:
                         continue
+                    try:
                         possibly_connected, detected_device = \
                                 self.scanner.is_device_connected(device)
+                    except InitialConnectionError as e:
+                        self.open_feedback_msg(device.get_gui_name(), e)
+                        continue
                     if possibly_connected:
                         possibly_connected_devices.append((device, detected_device))
                 if possibly_connected_devices:
@@ -233,6 +251,9 @@ class DeviceManager(Thread): # {{{
                                            device_kind='usb'):
                             if DEBUG:
                                 prints('Device connect failed again, giving up')
+            except OpenFailed as e:
+                if e.show_me:
+                    traceback.print_exc()
 
     # Mount devices that don't use USB, such as the folder device and iTunes
     # This will be called on the GUI thread. Because of this, we must store
@@ -265,7 +286,24 @@ class DeviceManager(Thread): # {{{
             except Queue.Empty:
                 pass
 
+    def run_startup(self, dev):
+        name = 'unknown'
+        try:
+            name = dev.__class__.__name__
+            dev.startup()
+        except:
+            prints('Startup method for device %s threw exception'%name)
+            traceback.print_exc()
+
     def run(self):
+        # Do any device-specific startup processing.
+        for d in self.devices:
+            self.run_startup(d)
+            n = d.is_dynamically_controllable()
+            if n:
+                self.dynamic_plugins[n] = d
+        self.devices_initialized.set()
+
         while self.keep_going:
             kls = None
             while True:
@@ -277,15 +315,23 @@ class DeviceManager(Thread): # {{{
             if kls is not None:
                 try:
                     dev = kls(folder_path)
+                    # We just created a new device instance. Call its startup
+                    # method and set the flag to call the shutdown method when
+                    # it disconnects.
+                    self.run_startup(dev)
+                    self.call_shutdown_on_disconnect = True
                     self.do_connect([[dev, None],], device_kind=device_kind)
                 except:
                     prints('Unable to open %s as device (%s)'%(device_kind, folder_path))
                     traceback.print_exc()
             else:
                 self.detect_device()
+
+            do_sleep = True
             while True:
                 job = self.next()
                 if job is not None:
+                    do_sleep = False
                     self.current_job = job
                     if self.device is not None:
                         self.device.set_progress_reporter(job.report_progress)
@@ -293,8 +339,16 @@ class DeviceManager(Thread): # {{{
                     self.current_job = None
                 else:
                     break
+            if do_sleep:
                 time.sleep(self.sleep_time)
 
+        # We are exiting. Call the shutdown method for each plugin
+        for p in self.devices:
+            try:
+                p.shutdown()
+            except:
+                pass
+
     def create_job_step(self, func, done, description, to_job, args=[], kwargs={}):
         job = DeviceJob(func, done, self.job_manager,
                         args=args, kwargs=kwargs, description=description)
@@ -475,6 +529,44 @@ class DeviceManager(Thread): # {{{
         if self.connected_device:
             self.connected_device.set_driveinfo_name(location_code, name)
 
+    # dynamic plugin interface
+
+    # This is a helper function that handles queueing with the device manager
+    def _call_request(self, name, method, *args, **kwargs):
+        d = self.dynamic_plugins.get(name, None)
+        if d:
+            return getattr(d, method)(*args, **kwargs)
+        return kwargs.get('default', None)
+
+    # The dynamic plugin methods below must be called on the GUI thread. They
+    # will switch to the device thread before calling the plugin.
+
+    def start_plugin(self, name):
+        self._call_request(name, 'start_plugin')
+
+    def stop_plugin(self, name):
+        self._call_request(name, 'stop_plugin')
+
+    def get_option(self, name, opt_string, default=None):
+        return self._call_request(name, 'get_option', opt_string, default=default)
+
+    def set_option(self, name, opt_string, opt_value):
+        self._call_request(name, 'set_option', opt_string, opt_value)
+
+    def is_running(self, name):
+        if self._call_request(name, 'is_running'):
+            return True
+        return False
+
+    def is_enabled(self, name):
+        try:
+            d = self.dynamic_plugins.get(name, None)
+            if d:
+                return True
+        except:
+            pass
+        return False
+
     # }}}
 
 class DeviceAction(QAction): # {{{
@@ -675,6 +767,7 @@ class DeviceMixin(object): # {{{
                 self.job_manager, Dispatcher(self.status_bar.show_message),
                 Dispatcher(self.show_open_feedback))
         self.device_manager.start()
+        self.device_manager.devices_initialized.wait()
         if tweaks['auto_connect_to_folder']:
             self.connect_to_folder_named(tweaks['auto_connect_to_folder'])
 

src/calibre/gui2/device_drivers/configwidget.py

@@ -43,6 +43,9 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
         self.connect(self.column_up, SIGNAL('clicked()'), self.up_column)
         self.connect(self.column_down, SIGNAL('clicked()'), self.down_column)
 
+        if device.HIDE_FORMATS_CONFIG_BOX:
+            self.groupBox.hide()
+
         if supports_subdirs:
             self.opt_use_subdirs.setChecked(self.settings.use_subdirs)
         else:

src/calibre/gui2/device_drivers/configwidget.ui

@@ -103,6 +103,19 @@
    <item row="6" column="0">
     <layout class="QGridLayout" name="extra_layout"/>
    </item>
+   <item row="7" column="0">
+    <spacer name="verticalSpacer_2">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>40</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
    <item row="4" column="0">
     <widget class="QLabel" name="label">
      <property name="text">

src/calibre/gui2/dialogs/confirm_delete.ui

@@ -1,7 +1,8 @@
-<ui version="4.0" >
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
  <class>Dialog</class>
- <widget class="QDialog" name="Dialog" >
-  <property name="geometry" >
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
    <rect>
     <x>0</x>
     <y>0</y>
@@ -9,51 +10,63 @@
     <height>300</height>
    </rect>
   </property>
-  <property name="windowTitle" >
+  <property name="windowTitle">
    <string>Are you sure?</string>
   </property>
-  <property name="windowIcon" >
-   <iconset resource="../../../../resources/images.qrc" >
+  <property name="windowIcon">
+   <iconset resource="../../../../resources/images.qrc">
     <normaloff>:/images/dialog_warning.png</normaloff>:/images/dialog_warning.png</iconset>
   </property>
-  <layout class="QGridLayout" name="gridLayout" >
-   <item row="0" column="0" >
-    <layout class="QHBoxLayout" name="horizontalLayout" >
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="0" column="0">
+    <layout class="QHBoxLayout" name="horizontalLayout">
      <item>
-      <widget class="QLabel" name="label" >
-       <property name="pixmap" >
-        <pixmap resource="../../../../resources/images.qrc" >:/images/dialog_warning.png</pixmap>
+      <widget class="QLabel" name="label">
+       <property name="pixmap">
+        <pixmap resource="../../../../resources/images.qrc">:/images/dialog_warning.png</pixmap>
        </property>
       </widget>
      </item>
      <item>
-      <widget class="QLabel" name="msg" >
-       <property name="text" >
+      <widget class="QLabel" name="msg">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Expanding" vsizetype="Preferred">
+         <horstretch>0</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="minimumSize">
+        <size>
+         <width>300</width>
+         <height>0</height>
+        </size>
+       </property>
+       <property name="text">
         <string>TextLabel</string>
        </property>
-       <property name="wordWrap" >
+       <property name="wordWrap">
         <bool>true</bool>
        </property>
       </widget>
      </item>
     </layout>
    </item>
-   <item row="1" column="0" >
-    <widget class="QCheckBox" name="again" >
-     <property name="text" >
+   <item row="1" column="0">
+    <widget class="QCheckBox" name="again">
+     <property name="text">
       <string>&amp;Show this warning again</string>
      </property>
-     <property name="checked" >
+     <property name="checked">
       <bool>true</bool>
      </property>
     </widget>
    </item>
-   <item row="2" column="0" >
-    <widget class="QDialogButtonBox" name="buttonBox" >
-     <property name="orientation" >
+   <item row="2" column="0">
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
       <enum>Qt::Horizontal</enum>
      </property>
-     <property name="standardButtons" >
+     <property name="standardButtons">
       <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
      </property>
     </widget>
@@ -61,7 +74,7 @@
   </layout>
  </widget>
  <resources>
-  <include location="../../../../resources/images.qrc" />
+  <include location="../../../../resources/images.qrc"/>
  </resources>
  <connections>
   <connection>
@@ -70,11 +83,11 @@
    <receiver>Dialog</receiver>
    <slot>accept()</slot>
    <hints>
-    <hint type="sourcelabel" >
+    <hint type="sourcelabel">
      <x>248</x>
      <y>254</y>
     </hint>
-    <hint type="destinationlabel" >
+    <hint type="destinationlabel">
      <x>157</x>
      <y>274</y>
     </hint>
@@ -86,11 +99,11 @@
    <receiver>Dialog</receiver>
    <slot>reject()</slot>
    <hints>
-    <hint type="sourcelabel" >
+    <hint type="sourcelabel">
      <x>316</x>
      <y>260</y>
     </hint>
-    <hint type="destinationlabel" >
+    <hint type="destinationlabel">
      <x>286</x>
      <y>274</y>
     </hint>

src/calibre/gui2/dialogs/message_box.py

@@ -9,7 +9,7 @@ import sys
 
 from PyQt4.Qt import (QDialog, QIcon, QApplication, QSize, QKeySequence,
     QAction, Qt, QTextBrowser, QDialogButtonBox, QVBoxLayout, QGridLayout,
-    QLabel, QPlainTextEdit, QTextDocument)
+    QLabel, QPlainTextEdit, QTextDocument, QCheckBox, pyqtSignal)
 
 from calibre.constants import __version__, isfrozen
 from calibre.gui2.dialogs.message_box_ui import Ui_Dialog
@@ -270,21 +270,23 @@ class ErrorNotification(MessageBox): # {{{
 class JobError(QDialog): # {{{
 
     WIDTH = 600
+    do_pop = pyqtSignal()
 
-    def __init__(self, gui):
-        QDialog.__init__(self, gui)
+    def __init__(self, parent):
+        QDialog.__init__(self, parent)
         self.setAttribute(Qt.WA_DeleteOnClose, False)
-        self.gui = gui
         self.queue = []
+        self.do_pop.connect(self.pop, type=Qt.QueuedConnection)
 
         self._layout = l = QGridLayout()
         self.setLayout(l)
         self.icon = QIcon(I('dialog_error.png'))
         self.setWindowIcon(self.icon)
         self.icon_label = QLabel()
-        self.icon_label.setPixmap(self.icon.pixmap(128, 128))
-        self.icon_label.setMaximumSize(QSize(128, 128))
+        self.icon_label.setPixmap(self.icon.pixmap(68, 68))
+        self.icon_label.setMaximumSize(QSize(68, 68))
         self.msg_label = QLabel('<p>&nbsp;')
+        self.msg_label.setStyleSheet('QLabel { margin-top: 1ex; }')
         self.msg_label.setWordWrap(True)
         self.msg_label.setTextFormat(Qt.RichText)
         self.det_msg = QPlainTextEdit(self)
@@ -302,15 +304,23 @@ class JobError(QDialog): # {{{
         self.det_msg_toggle.clicked.connect(self.toggle_det_msg)
         self.det_msg_toggle.setToolTip(
                 _('Show detailed information about this error'))
+        self.suppress = QCheckBox(self)
 
         l.addWidget(self.icon_label, 0, 0, 1, 1)
-        l.addWidget(self.msg_label,  0, 1, 1, 1, Qt.AlignLeft|Qt.AlignTop)
+        l.addWidget(self.msg_label,  0, 1, 1, 1)
         l.addWidget(self.det_msg,    1, 0, 1, 2)
-
-        l.addWidget(self.bb, 2, 0, 1, 2, Qt.AlignRight|Qt.AlignBottom)
+        l.addWidget(self.suppress,   2, 0, 1, 2, Qt.AlignLeft|Qt.AlignBottom)
+        l.addWidget(self.bb,         3, 0, 1, 2, Qt.AlignRight|Qt.AlignBottom)
+        l.setColumnStretch(1, 100)
 
         self.setModal(False)
-        self.base_height = max(200, self.sizeHint().height() + 20)
+        self.suppress.setVisible(False)
+        self.do_resize()
+
+    def update_suppress_state(self):
+        self.suppress.setText(_(
+            'Hide the remaining %d error messages'%len(self.queue)))
+        self.suppress.setVisible(len(self.queue) > 3)
         self.do_resize()
 
     def copy_to_clipboard(self, *args):
@@ -332,9 +342,11 @@ class JobError(QDialog): # {{{
         self.do_resize()
 
     def do_resize(self):
-        h = self.base_height
-        if self.det_msg.isVisible():
-            h += 250
+        h = self.sizeHint().height()
+        self.setMinimumHeight(0) # Needed as this gets set if det_msg is shown
+        # Needed otherwise re-showing the box after showing det_msg causes the box
+        # to not reduce in height
+        self.setMaximumHeight(h)
         self.resize(QSize(self.WIDTH, h))
 
     def showEvent(self, ev):
@@ -342,16 +354,50 @@ class JobError(QDialog): # {{{
         self.bb.button(self.bb.Close).setFocus(Qt.OtherFocusReason)
         return ret
 
+    def show_error(self, title, msg, det_msg=u''):
+        self.queue.append((title, msg, det_msg))
+        self.update_suppress_state()
+        self.pop()
+
+    def pop(self):
+        if not self.queue or self.isVisible(): return
+        title, msg, det_msg = self.queue.pop(0)
+        self.setWindowTitle(title)
+        self.msg_label.setText(msg)
+        self.det_msg.setPlainText(det_msg)
+        self.det_msg.setVisible(False)
+        self.det_msg_toggle.setText(self.show_det_msg)
+        self.det_msg_toggle.setVisible(True)
+        self.suppress.setChecked(False)
+        self.update_suppress_state()
+        if not det_msg:
+            self.det_msg_toggle.setVisible(False)
+        self.do_resize()
+        self.show()
+
+    def done(self, r):
+        if self.suppress.isChecked():
+            self.queue = []
+        QDialog.done(self, r)
+        self.do_pop.emit()
+
 # }}}
 
 if __name__ == '__main__':
     app = QApplication([])
-    from calibre.gui2.preferences import init_gui
-    gui = init_gui()
-    d = JobError(gui)
-    d.show()
+    d = JobError(None)
+    d.show_error('test title', 'some long meaningless test message', 'det msg')
+    d.show_error('test title', 'some long meaningless test message', 'det msg')
+    d.show_error('test title', 'some long meaningless test message', 'det msg')
+    d.show_error('test title', 'some long meaningless test message', 'det msg')
+    d.show_error('test title', 'some long meaningless test message', 'det msg')
+    d.show_error('test title', 'some long meaningless test message', 'det msg')
+    app.setQuitOnLastWindowClosed(False)
+    def checkd():
+        if not d.queue:
+            app.quit()
+    app.lastWindowClosed.connect(checkd)
     app.exec_()
-    gui.shutdown()
 
 # if __name__ == '__main__':
 #     app = QApplication([])