Sync to trunk.

resources/recipes/boston.com.recipe

@@ -0,0 +1,49 @@
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.boston.com
+'''
+
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class BusinessStandard(BasicNewsRecipe):
+    title                  = 'Boston'
+    __author__             = 'Darko Miletic'
+    description            = 'News from Boston'
+    oldest_article         = 7
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    delay                  = 1
+    use_embedded_content   = False
+    encoding               = 'cp1252'
+    publisher              = 'Boston'
+    category               = 'news, boston, usa, world'
+    language               = 'en'
+
+    conversion_options = {
+                             'comments'        : description
+                            ,'tags'            : category
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                         }
+
+    keep_only_tags   = [dict(name='div', attrs={'class':'story'})]
+    remove_tags      = [dict(name=['object','link','script','iframe'])]
+
+    feeds = [
+               (u'Top Stories'  , u'http://feeds.boston.com/boston/topstories'              )
+              ,(u'Patriots news', u'http://feeds.boston.com/boston/sports/football/patriots')
+              ,(u'National news', u'http://feeds.boston.com/boston/news/nation'             )
+              ,(u'World news'   , u'http://feeds.boston.com/boston/news/world'              )
+            ]
+
+    def print_version(self, url):
+        return url + '?mode=PF'
+
+    def get_article_url(self, article):
+        rawarticle = article.get('pheedo_origlink', None)
+        artls, sep, rsep = rawarticle.rpartition('/?')
+        if artls == '':
+           artls = rawarticle.rpartition('?')[0]
+        return artls
+

resources/recipes/clarion_ledger.recipe

@@ -0,0 +1,20 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ClarionLedger(BasicNewsRecipe):
+    title          = u'Clarion Ledger'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    language = 'en'
+    __author__ = 'cr4zyd'
+
+    feeds          = [(u'Local News', u'http://www.clarionledger.com/apps/pbcs.dll/oversikt?Category=RSS01'), (u'Breaking News', u'http://www.clarionledger.com/apps/pbcs.dll/section?Category=RSS'), (u'Sports', u'http://www.clarionledger.com/apps/pbcs.dll/oversikt?Category=RSS02'), (u'Business', u'http://www.clarionledger.com/apps/pbcs.dll/oversikt?Category=RSS03')]
+
+    keep_only_tags = [dict(name='div', attrs={'class':'article-headline'}),
+     dict(name='div', attrs={'class':'article-bodytext'})]
+    remove_tags = [dict(name=['img','script','li']),
+     dict(name='p', attrs={'class':'ratingbyline'}),
+     dict(name='div', attrs={'class':'article-tools'}),
+     dict(name='div', attrs={'class':'article-pagination article-pagination-top'}),
+     dict(name='div', attrs={'class':'article-pagination article-pagination-bottom'}),
+     dict(name='div', attrs={'class':'articleflex-container'})]

resources/recipes/dosisdiarias.recipe

@@ -0,0 +1,31 @@
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+http://www.dosisdiarias.com
+'''
+
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class DosisDiarias(BasicNewsRecipe):
+    title                  = 'Alberto Montt en dosis diarias'
+    __author__             = 'Darko Miletic'
+    description            = 'Mire sin compromiso y si le gusta vuelva'
+    oldest_article         = 5
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    use_embedded_content   = True
+    encoding               = 'utf-8'
+    publisher              = 'Alberto Montt'
+    category               = 'comic, blog, spanish'
+    language               = 'es'
+
+    conversion_options = {
+                             'comments'        : description
+                            ,'tags'            : category
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                         }
+
+    remove_tags = [dict(name='div',attrs={'class':'feedflare'})]
+
+    feeds = [(u'Dosis diaria', u'http://feeds.feedburner.com/montt' )]

resources/recipes/elmundo.recipe

@@ -1,4 +1,3 @@
-#!/usr/bin/env  python
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
@@ -19,22 +18,19 @@ class ElMundo(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'iso8859_15'
-    cover_url             = 'http://estaticos02.cache.el-mundo.net/papel/imagenes/v2.0/logoverde.gif'
+    language              = 'es'
-    remove_javascript     = True
 
-    html2lrf_options = [
+    conversion_options = {
-                          '--comment', description
+                             'comments'  : description
-                        , '--category', category
+                            ,'tags'      : category
-                        , '--publisher', publisher
+                            ,'language'  : language
-                        ]
+                            ,'publisher' : publisher
+                         }
 
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
+    keep_only_tags     = [dict(name='div', attrs={'class':'noticia'})]
+    remove_tags_before = dict(attrs={'class':['titular','antetitulo']     })
+    remove_tags_after  = dict(name='div' , attrs={'id':['desarrollo_noticia','tamano']})
 
-
-    keep_only_tags = [
-                        dict(name='div', attrs={'id':['bloqueprincipal','noticia']})
-                       ,dict(name='div', attrs={'class':['contenido_noticia_01']})
-                     ]
     remove_tags = [
                      dict(name='div', attrs={'class':['herramientas','publicidad_google']})
                     ,dict(name='div', attrs={'id':'modulo_multimedia' })
@@ -44,6 +40,8 @@ class ElMundo(BasicNewsRecipe):
 
     feeds = [
               (u'Portada'         , u'http://rss.elmundo.es/rss/descarga.htm?data2=4' )
+             ,(u'Deportes'        , u'http://rss.elmundo.es/rss/descarga.htm?data2=14')
+             ,(u'Economia'        , u'http://rss.elmundo.es/rss/descarga.htm?data2=7' )
              ,(u'Espana'          , u'http://rss.elmundo.es/rss/descarga.htm?data2=8' )
              ,(u'Internacional'   , u'http://rss.elmundo.es/rss/descarga.htm?data2=9' )
              ,(u'Cultura'         , u'http://rss.elmundo.es/rss/descarga.htm?data2=6' )
@@ -51,10 +49,3 @@ class ElMundo(BasicNewsRecipe):
              ,(u'Comunicacion'    , u'http://rss.elmundo.es/rss/descarga.htm?data2=26')
              ,(u'Television'      , u'http://rss.elmundo.es/rss/descarga.htm?data2=76')
             ]
-
-    def preprocess_html(self, soup):
-        for item in soup.findAll(style=True):
-            del item['style']
-        return soup
-
-    language = 'es'

resources/recipes/lrb.recipe

@@ -1,4 +1,3 @@
-#!/usr/bin/env  python
 
 __license__   = 'GPL v3'
 __copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
@@ -12,30 +11,29 @@ class LondonReviewOfBooks(BasicNewsRecipe):
     title                 = u'London Review of Books'
     __author__            = u'Darko Miletic'
     description           = u'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
+    category              = 'news, literature, England'
+    publisher             = 'London Review of Books'
     oldest_article        = 7
     max_articles_per_feed = 100
     language              = 'en_GB'
-
     no_stylesheets        = True
     use_embedded_content  = False
-    encoding              = 'cp1252'
+    encoding              = 'utf-8'
 
+    conversion_options = {  
+                             'comments'  : description
+                            ,'tags'      : category
+                            ,'language'  : language
+                            ,'publisher' : publisher
+                         }
+    
+    keep_only_tags = [dict(name='div' , attrs={'id'   :'main'})]
     remove_tags = [
-                    dict(name='div' , attrs={'id'   :'otherarticles'})
+                    dict(name='div' , attrs={'class':['pagetools','issue-nav-controls','nocss']})
-                   ,dict(name='div' , attrs={'class':'pagetools'    })
+                   ,dict(name='div' , attrs={'id'   :['mainmenu','precontent','otherarticles']     })
-                   ,dict(name='div' , attrs={'id'   :'mainmenu'     })
+                   ,dict(name='span', attrs={'class':['inlineright','article-icons']})
-                   ,dict(name='div' , attrs={'id'   :'precontent'   })
+                   ,dict(name='ul'  , attrs={'class':'article-controls'})
-                   ,dict(name='div' , attrs={'class':'nocss'        })
+                   ,dict(name='p'   , attrs={'class':'meta-info'       })
-                   ,dict(name='span', attrs={'class':'inlineright'  })
                   ]
 
     feeds = [(u'London Review of Books', u'http://www.lrb.co.uk/lrbrss.xml')]
-
-    def print_version(self, url):
-        main, split, rest = url.rpartition('/')
-        return main + '/print/' + rest
-
-    def postprocess_html(self, soup, first_fetch):
-        for t in soup.findAll(['table', 'tr', 'td']):
-            t.name = 'div'
-        return soup

resources/recipes/politico.recipe

@@ -1,4 +1,5 @@
 #!/usr/bin/env  python
+# -*- coding: cp1252 -*-
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
@@ -13,7 +14,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class Politico(BasicNewsRecipe):
 
     title                 = 'Politico'
-    __author__            = 'Darko Miletic'
+    __author__            = 'Darko Miletic and Sujata Raman'
     description           = 'Political news from USA'
     publisher             = 'Capitol News Company, LLC'
     category              = 'news, politics, USA'
@@ -22,10 +23,9 @@ class Politico(BasicNewsRecipe):
     use_embedded_content  = False
     no_stylesheets        = True
     remove_javascript     = True
-    encoding              = 'cp1252'
+    encoding              = 'UTF-8'
     language = 'en'
 
-    
     html2lrf_options = [
                           '--comment', description
                         , '--category', category
@@ -35,7 +35,19 @@ class Politico(BasicNewsRecipe):
 
     html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True'
 
-    remove_tags       = [dict(name=['notags','embed','object','link','img'])]
+    remove_tags       = [
+                         dict(name=['notags','embed','object','link','img']),
+                         ]
+
+    extra_css = '''
+                body{font-family:Arial,Sans-serif;}
+                element.style{color:#FF0000;font-family:Arial,Sans-serif;}
+                .author{color:#808080;font-size:x-small;}
+                a{ color:#003399;}
+                .byline{color:#696969 ; font-size:x-small;}
+                .story{color:#000000;}
+                td{color:#000000;}
+                '''
 
     feeds = [
               (u'Top Stories' , u'http://www.politico.com/rss/politicopicks.xml' )
@@ -48,17 +60,23 @@ class Politico(BasicNewsRecipe):
               ,(u'Roger Simon' , u'http://www.politico.com/rss/rogersimon.xml'    )
               ,(u'Suite Talk'  , u'http://www.politico.com/rss/suitetalk.xml'     )
               ,(u'Playbook'    , u'http://www.politico.com/rss/playbook.xml'      )
-              ,(u'The Huddle'  , u'http://www.politico.com/rss/huddle.xml'        )
+              #(u'The Huddle'  , u'http://www.politico.com/rss/huddle.xml' )
             ]
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Language" content="en-US"/>'
+        mtag = '<meta http-equiv="Content-Language" content="en"/>'
         soup.head.insert(0,mtag)
         for item in soup.findAll(style=True):
             del item['style']
         return soup
 
-    url_pat = re.compile(r'<a href="([^"]+printstory\.cfm[^"]+)"')
+    url_pat = re.compile(r'<a href="([^"]+print.*\.cfm[^"]+)"')
+
+    def postprocess_html(self, soup, first):
+
+            for tag in soup.findAll(name=['table', 'tr', 'td']):
+                tag.name = 'div'
+            return soup
 
     def print_version(self, url):
         raw = self.index_to_soup(url, raw=True)

resources/recipes/praguemonitor.recipe

@@ -0,0 +1,32 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+praguemonitor.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class PragueDailyMonitor(BasicNewsRecipe):
+    title                 = u'Prague Daily Monitor'
+    __author__            = u'Darko Miletic'
+    description           = u'Czech news in English'
+    category              = 'news, politics, Czech republic'
+    publisher             = 'Prague Daily Monitor'
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    language              = 'en'
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+
+    conversion_options = {
+                             'comments'  : description
+                            ,'tags'      : category
+                            ,'language'  : language
+                            ,'publisher' : publisher
+                         }
+
+    keep_only_tags = [dict(name='div' , attrs={'id':['content-header','content-area']})]
+
+    feeds = [(u'All Articles', u'http://feeds.feedburner.com/PragueDailyMonitor?format=xml')]

resources/recipes/sueddeutsche.recipe

@@ -26,9 +26,12 @@ class Sueddeutsche(BasicNewsRecipe):
                         dict(name='div', attrs={'id':["artikel","contentTable"]}) ,
                          ]
     remove_tags = [ dict(name='link'), dict(name='iframe'),
-                    dict(name='div', attrs={'id':["themenbox","artikelfoot","CAD_AD","rechteSpalte"]}),
+                    dict(name='div', attrs={'id':["themenbox","artikelfoot","CAD_AD","SKY_AD","NT1_AD","rechteSpalte"]}),
                     dict(name='div', attrs={'class':["similar-article-box","artikelliste","nteaser301bg","pages closed"]}),
+                    dict(name='div', attrs={'class':["listHeader","listHeader2","hr2","item","videoBigButton"]}),
                     dict(name='p', attrs={'class':["ressortartikeln",]}),
+                    dict(name='div', attrs={'style':["position:relative;"]}),
+                    dict(name='span', attrs={'class':["nlinkheaderteaserschwarz",]}),
                     dict(name='table', attrs={'class':["kommentare","footer","pageBoxBot","pageAktiv","bgcontent"]}),
                     dict(name='ul', attrs={'class':["breadcrumb","articles","activities"]}),
                     dict(name='p', text = "ANZEIGE")
@@ -66,3 +69,4 @@ class Sueddeutsche(BasicNewsRecipe):
 
 
 
+

resources/recipes/welt.recipe

@@ -15,12 +15,13 @@ class weltDe(BasicNewsRecipe):
     __author__ = 'Oliver Niesner'
     use_embedded_content   = False
     timefmt = ' [%d %b %Y]'
-    max_articles_per_feed = 15 # reduced to this value to prevent too many articles (suggested by Gregory Riker
+    max_articles_per_feed = 15
+    linearize_tables = True
     no_stylesheets = True
     remove_stylesheets = True
     remove_javascript = True
-    language = 'de'
     encoding = 'iso-8859-1'
+    BasicNewsRecipe.summary_length = 200
 
 
     remove_tags = [dict(id='jumplinks'),
@@ -43,10 +44,14 @@ class weltDe(BasicNewsRecipe):
 		   dict(id='servicesBox'),
 		   dict(id='toggleAdvancedSearch'),
 		   dict(id='mainNav'),
-		   dict(id='ratingBox5136466_1'),
-		   dict(id='ratingBox5136466_2'),
 		   dict(id='articleInlineMediaBox0'),
 		   dict(id='sectionSponsor'),
+		   dict(id='sprucharea'),
+		   dict(id='xmsg_recommendEmail'),
+		   dict(id='xmsg_recommendSms'),
+		   dict(id='xmsg_comment'),
+		   dict(id='additionalNavWrapper'),
+		   dict(id='imagebox'),
 		   #dict(id=''),
                    dict(name='span'),
 		   dict(name='div', attrs={'class':'printURL'}),
@@ -65,10 +70,21 @@ class weltDe(BasicNewsRecipe):
 		   dict(name='ul', attrs={'class':'optionsSubNav clear'}),
 		   dict(name='li', attrs={'class':'next'}),
 		   dict(name='li', attrs={'class':'prev'}),
+		   dict(name='li', attrs={'class':'last'}),
+		   dict(name='table', attrs={'class':'textGallery'}),
 		   dict(name='li', attrs={'class':'active'})]
 
     remove_tags_after = [dict(id='tw_link_widget')]
 
+    extra_css = '''
+                    h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small; color: #003399;}
+                    a{font-family:Arial,Helvetica,sans-serif; font-size: x-small; font-style:italic;}
+                    .dachzeile p{font-family:Arial,Helvetica,sans-serif; font-size: x-small; }
+                    h1{ font-family:Arial,Helvetica,sans-serif;  font-size:x-large; font-weight:bold;}
+                    .artikelTeaser{font-family:Arial,Helvetica,sans-serif; font-size: x-small; font-weight:bold; }
+                    body{font-family:Arial,Helvetica,sans-serif; }
+                    .photo {font-family:Arial,Helvetica,sans-serif; font-size: x-small; color: #666666;}                 '''
+
     feeds =  [ ('Politik', 'http://welt.de/politik/?service=Rss'),
 	       ('Deutsche Dinge', 'http://www.welt.de/deutsche-dinge/?service=Rss'),
 	       ('Wirtschaft', 'http://welt.de/wirtschaft/?service=Rss'),

resources/templates/fb2.xsl

@@ -187,7 +187,7 @@
 	<!-- section/title -->
 	<xsl:template match="fb:body/fb:title">
         <xsl:element name="h1">
-            <xsl:apply-templates mode="title"/>
+            <xsl:apply-templates />
         </xsl:element>
 	</xsl:template>
 

setup/installer/linux/freeze.py

@@ -108,7 +108,7 @@ class LinuxFreeze(Command):
                     'glib', 'gobject']
 
         packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg',
-                    'dateutil', 'dns', 'email']
+                    'dateutil', 'dns', 'email', 'dbus']
 
         includes += ['calibre.gui2.convert.'+x.split('/')[-1].rpartition('.')[0] for x in \
                 glob.glob('src/calibre/gui2/convert/*.py')]

src/calibre/devices/__init__.py

@@ -5,7 +5,9 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Device drivers.
 '''
 
-import time
+import sys, os, time, pprint
+from functools import partial
+from StringIO import StringIO
 
 DAY_MAP   = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6)
 MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12)
@@ -24,3 +26,96 @@ def strftime(epoch, zone=time.gmtime):
     src[0] = INVERSE_DAY_MAP[int(src[0][:-1])]+','
     src[2] = INVERSE_MONTH_MAP[int(src[2])]
     return ' '.join(src)
+
+def debug():
+    from calibre.customize.ui import device_plugins
+    from calibre.devices.scanner import DeviceScanner
+    from calibre.constants import iswindows, isosx, __version__
+    from calibre import prints
+    oldo, olde = sys.stdout, sys.stderr
+
+    buf = StringIO()
+    sys.stdout = sys.stderr = buf
+    try:
+        out = partial(prints, file=buf)
+        out('Version:', __version__)
+        s = DeviceScanner()
+        s.scan()
+        devices = (s.devices)
+        if not iswindows:
+            devices = [list(x) for x in devices]
+            for d in devices:
+                for i in range(3):
+                    d[i] = hex(d[i])
+        out('USB devices on system:')
+        out(pprint.pformat(devices))
+        if iswindows:
+            if iswindows:
+                import pythoncom
+                pythoncom.CoInitialize()
+                try:
+                    wmi = __import__('wmi', globals(), locals(), [], -1)
+                    drives = []
+                    out('Drives detected:')
+                    out('\t', '(ID, Partitions, Drive letter)')
+                    for drive in wmi.WMI(find_classes=False).Win32_DiskDrive():
+                        if drive.Partitions == 0:
+                            continue
+                        try:
+                            partition = drive.associators("Win32_DiskDriveToDiskPartition")[0]
+                            logical_disk = partition.associators('Win32_LogicalDiskToPartition')[0]
+                            prefix = logical_disk.DeviceID+os.sep
+                            drives.append((str(drive.PNPDeviceID), drive.Index, prefix))
+                        except IndexError:
+                            drives.append((str(drive.PNPDeviceID), 'No mount points found'))
+                    for drive in drives:
+                        out('\t', drive)
+                finally:
+                    pythoncom.CoUninitialize()
+
+        ioreg = None
+        if isosx:
+            from calibre.devices.usbms.device import Device
+            ioreg = Device.run_ioreg()
+        connected_devices = []
+        for dev in device_plugins():
+            out('Looking for', dev.__class__.__name__)
+            connected = s.is_device_connected(dev, debug=True)
+            if connected:
+                connected_devices.append(dev)
+
+        errors = {}
+        success = False
+        for dev in connected_devices:
+            out('Device possibly connected:', dev.__class__.name)
+            out('Trying to open device...', end=' ')
+            try:
+                dev.open()
+                out('OK')
+            except:
+                import traceback
+                errors[dev] = traceback.format_exc()
+                out('failed')
+                continue
+            success = True
+            if hasattr(dev, '_main_prefix'):
+                out('Main memory:', repr(dev._main_prefix))
+            out('Total space:', dev.total_space())
+            break
+        if not success and errors:
+            out('Opening of the following devices failed')
+            for dev,msg in errors.items():
+                out(dev)
+                out(msg)
+                out(' ')
+
+        if ioreg is not None:
+            out(' ')
+            out('IOREG Output')
+            out(ioreg)
+
+        return buf.getvalue().decode('utf-8')
+    finally:
+        sys.stdout = oldo
+        sys.stderr = olde
+

src/calibre/devices/eb600/driver.py

@@ -117,9 +117,15 @@ class ITALICA(EB600):
     name = 'Italica Device Interface'
     gui_name = 'Italica'
 
-    FORMATS = ['epub', 'pdf', 'txt']
+    FORMATS = ['epub', 'rtf', 'fb2', 'html', 'prc', 'mobi', 'pdf', 'txt']
 
     VENDOR_NAME = 'ITALICA'
     WINDOWS_MAIN_MEM = 'EREADER'
+    WINDOWS_CARD_A_MEM = WINDOWS_MAIN_MEM
 
     OSX_MAIN_MEM = 'Italica eReader Media'
+    OSX_CARD_A_MEM = OSX_MAIN_MEM
+
+    MAIN_MEMORY_VOLUME_LABEL  = 'Italica Main Memory'
+    STORAGE_CARD_VOLUME_LABEL = 'Italica Storage Card'
+

src/calibre/devices/prs500/driver.py

@@ -166,7 +166,7 @@ class PRS500(DeviceConfig, DevicePlugin):
             try:
                 if not dev.handle:
                     dev.open()
-                if not dev.in_session:
+                if not getattr(dev, 'in_session', False):
                     dev.send_validated_command(BeginEndSession(end=False))
                     dev.in_session = True
                 res = func(*args, **kwargs)

src/calibre/devices/usbms/device.py

@@ -711,7 +711,9 @@ class Device(DeviceConfig, DevicePlugin):
             candidates = self.get_main_ebook_dir()
             if isinstance(candidates, basestring):
                 candidates = [candidates]
-            candidates = [os.path.join(self._main_prefix, *(x.split('/'))) for x
+            candidates = [
+                    ((os.path.join(self._main_prefix, *(x.split('/')))) if x else
+                    self._main_prefix) for x
                     in candidates]
             existing = [x for x in candidates if os.path.exists(x)]
             if not existing:

src/calibre/devices/usbms/driver.py

@@ -63,7 +63,7 @@ class USBMS(CLI, Device):
         if isinstance(ebook_dirs, basestring):
             ebook_dirs = [ebook_dirs]
         for ebook_dir in ebook_dirs:
-            ebook_dir = os.path.join(prefix, *(ebook_dir.split('/')))
+            ebook_dir = os.path.join(prefix, *(ebook_dir.split('/'))) if ebook_dir else prefix
             if not os.path.exists(ebook_dir): continue
             # Get all books in the ebook_dir directory
             if self.SUPPORTS_SUB_DIRS:

src/calibre/ebooks/__init__.py

@@ -25,7 +25,7 @@ class DRMError(ValueError):
 BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'htm', 'xhtm',
                    'html', 'xhtml', 'pdf', 'pdb', 'prc', 'mobi', 'azw', 'doc',
                    'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'oebzip',
-                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1']
+                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml']
 
 class HTMLRenderer(object):
 

src/calibre/ebooks/conversion/plumber.py

@@ -804,6 +804,11 @@ OptionRecommendation(name='language',
         if line_height < 1e-4:
             line_height = None
 
+        if self.opts.linearize_tables and \
+                self.output_plugin.file_type not in ('mobi', 'lrf'):
+            from calibre.ebooks.oeb.transforms.linearize_tables import LinearizeTables
+            LinearizeTables()(self.oeb, self.opts)
+
         flattener = CSSFlattener(fbase=fbase, fkey=fkey,
                 lineh=line_height,
                 untable=self.output_plugin.file_type in ('mobi','lit'),
@@ -812,10 +817,6 @@ OptionRecommendation(name='language',
         self.opts.insert_blank_line = oibl
         self.opts.remove_paragraph_spacing = orps
 
-        if self.opts.linearize_tables and \
-                self.output_plugin.file_type not in ('mobi', 'lrf'):
-            from calibre.ebooks.oeb.transforms.linearize_tables import LinearizeTables
-            LinearizeTables()(self.oeb, self.opts)
         pr(0.9)
         self.flush()
 

src/calibre/ebooks/mobi/input.py

@@ -16,10 +16,16 @@ class MOBIInput(InputFormatPlugin):
                 accelerators):
         from calibre.ebooks.mobi.reader import MobiReader
         from lxml import html
+        parse_cache = {}
+        try:
             mr = MobiReader(stream, log, options.input_encoding,
                         options.debug_pipeline)
-        parse_cache = {}
             mr.extract_content('.', parse_cache)
+        except:
+            mr = MobiReader(stream, log, options.input_encoding,
+                        options.debug_pipeline, try_extra_data_fix=True)
+            mr.extract_content('.', parse_cache)
+
         raw = parse_cache.pop('calibre_raw_mobi_markup', False)
         if raw:
             if isinstance(raw, unicode):

src/calibre/ebooks/mobi/reader.py

@@ -108,7 +108,7 @@ class EXTHHeader(object):
 
 class BookHeader(object):
 
-    def __init__(self, raw, ident, user_encoding, log):
+    def __init__(self, raw, ident, user_encoding, log, try_extra_data_fix=False):
         self.log = log
         self.compression_type = raw[:2]
         self.records, self.records_size = struct.unpack('>HH', raw[8:12])
@@ -141,7 +141,8 @@ class BookHeader(object):
                 self.codec = 'cp1252' if user_encoding is None else user_encoding
                 log.warn('Unknown codepage %d. Assuming %s' % (self.codepage,
                     self.codec))
-            if ident == 'TEXTREAD' or self.length < 0xE4 or 0xE8 < self.length:
+            if ident == 'TEXTREAD' or self.length < 0xE4 or 0xE8 < self.length \
+                or (try_extra_data_fix and self.length == 0xE4):
                 self.extra_flags = 0
             else:
                 self.extra_flags, = struct.unpack('>H', raw[0xF2:0xF4])
@@ -229,7 +230,8 @@ class MobiReader(object):
     PAGE_BREAK_PAT = re.compile(r'(<[/]{0,1}mbp:pagebreak\s*[/]{0,1}>)+', re.IGNORECASE)
     IMAGE_ATTRS = ('lowrecindex', 'recindex', 'hirecindex')
 
-    def __init__(self, filename_or_stream, log, user_encoding=None, debug=None):
+    def __init__(self, filename_or_stream, log, user_encoding=None, debug=None,
+            try_extra_data_fix=False):
         self.log = log
         self.debug = debug
         self.embedded_mi = None
@@ -284,7 +286,7 @@ class MobiReader(object):
 
 
         self.book_header = BookHeader(self.sections[0][0], self.ident,
-            user_encoding, self.log)
+            user_encoding, self.log, try_extra_data_fix=try_extra_data_fix)
         self.name = self.name.decode(self.book_header.codec, 'replace')
 
     def extract_content(self, output_dir, parse_cache):
@@ -701,7 +703,9 @@ class MobiReader(object):
         if self.book_header.ancient and '<html' not in self.mobi_html[:300].lower():
             self.mobi_html = self.mobi_html.replace('\r ', '\n\n ')
         self.mobi_html = self.mobi_html.replace('\0', '')
+        if self.book_header.codec == 'cp1252':
             self.mobi_html = self.mobi_html.replace('\x1e', '') # record separator
+            self.mobi_html = self.mobi_html.replace('\x02', '') # start of text
         return processed_records
 
 

src/calibre/ebooks/oeb/transforms/split.py

@@ -91,7 +91,6 @@ class Split(object):
                             False))
                 except:
                     pass
-
         page_breaks = set([])
         for selector, before in self.page_break_selectors:
             body = item.data.xpath('//h:body', namespaces=NAMESPACES)
@@ -382,6 +381,7 @@ class FlowSplitter(object):
                 p[i:i+1] = new_pres
 
         split_point, before = self.find_split_point(root)
+        self.log.debug('\t\t\tSplit point:', split_point.tag, tree.getpath(split_point))
         if split_point is None:
             raise SplitError(self.item.href, root)
 
@@ -396,6 +396,9 @@ class FlowSplitter(object):
                     '\t\t\tCommitted sub-tree #%d (%d KB)'%(
                                len(self.split_trees), size/1024.))
             else:
+                self.log.debug(
+                        '\t\t\tSplit tree still too large: %d KB' % \
+                                (size/1024.))
                 self.split_to_size(t)
 
     def find_split_point(self, root):

src/calibre/ebooks/pdf/reflow.py

@@ -78,7 +78,7 @@ class HorizontalBox(object):
     def append(self, t):
         self.texts.append(t)
 
-    def sort(self):
+    def sort(self, left_margin, right_margin):
         self.texts.sort(cmp=lambda x,y: cmp(x.left, y.left))
         self.top, self.bottom = sys.maxint, 0
         for t in self.texts:
@@ -86,6 +86,27 @@ class HorizontalBox(object):
             self.bottom = max(self.bottom, t.bottom)
         self.left = self.texts[0].left
         self.right = self.texts[-1].right
+        self.gaps = []
+        for i, t in enumerate(self.texts[1:]):
+            gap = Interval(self.texts[i].right, t.left)
+            if gap.width > 3:
+                self.gaps.append(gap)
+        left = Interval(left_margin, self.texts[0].left)
+        if left.width > 3:
+            self.gaps.insert(0, left)
+        right = Interval(self.texts[-1].right, right_margin)
+        if right.width > 3:
+            self.gaps.append(right)
+
+    def has_intersection_with(self, gap):
+        for g in self.gaps:
+            if g.intersection(gap):
+                return True
+        return False
+
+    def identify_columns(self, column_gaps):
+        self.number_of_columns = len(column_gaps) + 1
+
 
 class Page(object):
 
@@ -138,19 +159,24 @@ class Page(object):
 
 
         for hb in self.horizontal_boxes:
-            hb.sort()
+            hb.sort(self.left_margin, self.right_margin)
 
         self.horizontal_boxes.sort(cmp=lambda x,y: cmp(x.bottom, y.bottom))
 
     def identify_columns(self):
 
         def neighborhood(i):
-            if i == 0:
+            if i == len(self.horizontal_boxes)-1:
-                return self.horizontal_boxes[1:3]
+                return self.horizontal_boxes[i-2:i]
+            if i == len(self.horizontal_boxes)-2:
                 return (self.horizontal_boxes[i-1], self.horizontal_boxes[i+1])
+            return self.horizontal_boxes[i+1], self.horizontal_boxes[i+2]
 
         for i, hbox in enumerate(self.horizontal_boxes):
-            pass
+            n1, n2 = neighborhood(i)
+            for gap in hbox.gaps:
+                gap.is_column_gap =  n1.has_intersection_with(gap) and \
+                    n2.has_intersection_with(gap)
 
 
 

src/calibre/gui2/__init__.py

@@ -52,7 +52,7 @@ def _config():
               help=_('Columns to be displayed in the book list'))
     c.add_opt('autolaunch_server', default=False, help=_('Automatically launch content server on application startup'))
     c.add_opt('oldest_news', default=60, help=_('Oldest news kept in database'))
-    c.add_opt('systray_icon', default=True, help=_('Show system tray icon'))
+    c.add_opt('systray_icon', default=False, help=_('Show system tray icon'))
     c.add_opt('upload_news_to_device', default=True,
               help=_('Upload downloaded news to device'))
     c.add_opt('delete_news_from_library_on_upload', default=False,

src/calibre/gui2/convert/xexp_edit.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>400</width>
+    <width>422</width>
     <height>64</height>
    </rect>
   </property>
@@ -30,7 +30,20 @@
       </widget>
      </item>
      <item>
-      <widget class="QLineEdit" name="edit"/>
+      <widget class="HistoryLineEdit" name="edit">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+         <horstretch>100</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="minimumSize">
+        <size>
+         <width>350</width>
+         <height>0</height>
+        </size>
+       </property>
+      </widget>
      </item>
     </layout>
    </item>
@@ -54,8 +67,28 @@
      </property>
     </widget>
    </item>
+   <item row="0" column="2">
+    <spacer name="horizontalSpacer">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>20</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
   </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>HistoryLineEdit</class>
+   <extends>QComboBox</extends>
+   <header>calibre/gui2/widgets.h</header>
+  </customwidget>
+ </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>
  </resources>

src/calibre/gui2/convert/xpath_wizard.py

@@ -70,6 +70,11 @@ class XPathEdit(QWidget, Ui_Edit):
         if wiz.exec_() == wiz.Accepted:
             self.edit.setText(wiz.xpath)
 
+    def setObjectName(self, *args):
+        QWidget.setObjectName(self, *args)
+        if hasattr(self, 'edit'):
+            self.edit.initialize('xpath_edit_'+unicode(self.objectName()))
+
 
     def set_msg(self, msg):
         self.msg.setText(msg)
@@ -95,4 +100,11 @@ class XPathEdit(QWidget, Ui_Edit):
         return True
 
 
-
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+    w = XPathEdit()
+    w.setObjectName('test')
+    w.show()
+    app.exec_()
+    print w.xpath

src/calibre/gui2/dialogs/config/__init__.py

@@ -457,6 +457,12 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
                 self.open_config_dir)
         self.opt_get_social_metadata.setChecked(config['get_social_metadata'])
         self.opt_enforce_cpu_limit.setChecked(config['enforce_cpu_limit'])
+        self.device_detection_button.clicked.connect(self.debug_device_detection)
+
+    def debug_device_detection(self):
+        from calibre.gui2.dialogs.config.device_debug import DebugDevice
+        d = DebugDevice(self)
+        d.exec_()
 
     def open_config_dir(self):
         from calibre.utils.config import config_dir

src/calibre/gui2/dialogs/config/config.ui

@@ -15,7 +15,7 @@
    <string>Preferences</string>
   </property>
   <property name="windowIcon">
-   <iconset resource="../../../../work/calibre/resources/images.qrc">
+   <iconset>
     <normaloff>:/images/config.svg</normaloff>:/images/config.svg</iconset>
   </property>
   <layout class="QGridLayout">
@@ -148,7 +148,7 @@
                     <string>...</string>
                    </property>
                    <property name="icon">
-                    <iconset resource="../../../../work/calibre/resources/images.qrc">
+                    <iconset>
                      <normaloff>:/images/mimetypes/dir.svg</normaloff>:/images/mimetypes/dir.svg</iconset>
                    </property>
                   </widget>
@@ -285,7 +285,7 @@
                        <string>...</string>
                       </property>
                       <property name="icon">
-                       <iconset resource="../../../../work/calibre/resources/images.qrc">
+                       <iconset>
                         <normaloff>:/images/arrow-up.svg</normaloff>:/images/arrow-up.svg</iconset>
                       </property>
                      </widget>
@@ -309,7 +309,7 @@
                        <string>...</string>
                       </property>
                       <property name="icon">
-                       <iconset resource="../../../../work/calibre/resources/images.qrc">
+                       <iconset>
                         <normaloff>:/images/arrow-down.svg</normaloff>:/images/arrow-down.svg</iconset>
                       </property>
                      </widget>
@@ -473,7 +473,7 @@
                          <string>...</string>
                         </property>
                         <property name="icon">
-                         <iconset resource="../../../../work/calibre/resources/images.qrc">
+                         <iconset>
                           <normaloff>:/images/arrow-up.svg</normaloff>:/images/arrow-up.svg</iconset>
                         </property>
                        </widget>
@@ -497,7 +497,7 @@
                          <string>...</string>
                         </property>
                         <property name="icon">
-                         <iconset resource="../../../../work/calibre/resources/images.qrc">
+                         <iconset>
                           <normaloff>:/images/arrow-down.svg</normaloff>:/images/arrow-down.svg</iconset>
                         </property>
                        </widget>
@@ -557,7 +557,7 @@
                     <string>&amp;Add email</string>
                    </property>
                    <property name="icon">
-                    <iconset resource="../../../../work/calibre/resources/images.qrc">
+                    <iconset>
                      <normaloff>:/images/plus.svg</normaloff>:/images/plus.svg</iconset>
                    </property>
                    <property name="iconSize">
@@ -584,7 +584,7 @@
                     <string>&amp;Remove email</string>
                    </property>
                    <property name="icon">
-                    <iconset resource="../../../../work/calibre/resources/images.qrc">
+                    <iconset>
                      <normaloff>:/images/minus.svg</normaloff>:/images/minus.svg</iconset>
                    </property>
                    <property name="iconSize">
@@ -649,21 +649,21 @@
                </property>
               </widget>
              </item>
-             <item row="2" column="0" colspan="2">
+             <item row="3" column="0" colspan="2">
               <widget class="QPushButton" name="compact_button">
                <property name="text">
                 <string>&amp;Check database integrity</string>
                </property>
               </widget>
              </item>
-             <item row="4" column="0" colspan="2">
+             <item row="5" column="0" colspan="2">
               <widget class="QPushButton" name="button_osx_symlinks">
                <property name="text">
                 <string>&amp;Install command line tools</string>
                </property>
               </widget>
              </item>
-             <item row="3" column="0" colspan="2">
+             <item row="4" column="0" colspan="2">
               <widget class="QPushButton" name="button_open_config_dir">
                <property name="text">
                 <string>Open calibre &amp;configuration directory</string>
@@ -677,6 +677,13 @@
                </property>
               </widget>
              </item>
+             <item row="2" column="0" colspan="2">
+              <widget class="QPushButton" name="device_detection_button">
+               <property name="text">
+                <string>Debug &amp;device detection</string>
+               </property>
+              </widget>
+             </item>
             </layout>
            </widget>
            <widget class="QWidget" name="page_4">
@@ -966,7 +973,7 @@
                      <string>...</string>
                     </property>
                     <property name="icon">
-                     <iconset resource="../../../../work/calibre/resources/images.qrc">
+                     <iconset>
                       <normaloff>:/images/document_open.svg</normaloff>:/images/document_open.svg</iconset>
                     </property>
                    </widget>

src/calibre/gui2/dialogs/config/device_debug.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import QDialog, QVBoxLayout, QPlainTextEdit, QTimer, \
+    QDialogButtonBox, QPushButton, QApplication, QIcon
+
+class DebugDevice(QDialog):
+
+    def __init__(self, parent=None):
+        QDialog.__init__(self, parent)
+        self._layout = QVBoxLayout(self)
+        self.setLayout(self._layout)
+        self.log = QPlainTextEdit(self)
+        self._layout.addWidget(self.log)
+        self.log.setPlainText(_('Getting debug information')+'...')
+        self.copy = QPushButton(_('Copy to &clipboard'))
+        self.copy.setDefault(True)
+        self.setWindowTitle(_('Debug device detection'))
+        self.setWindowIcon(QIcon(I('debug.svg')))
+        self.copy.clicked.connect(self.copy_to_clipboard)
+        self.ok = QPushButton('&OK')
+        self.ok.setAutoDefault(False)
+        self.ok.clicked.connect(self.accept)
+        self.bbox = QDialogButtonBox(self)
+        self.bbox.addButton(self.copy, QDialogButtonBox.ActionRole)
+        self.bbox.addButton(self.ok, QDialogButtonBox.AcceptRole)
+        self._layout.addWidget(self.bbox)
+        self.resize(750, 500)
+        self.bbox.setEnabled(False)
+        QTimer.singleShot(1000, self.debug)
+
+    def debug(self):
+        try:
+            from calibre.devices import debug
+            raw = debug()
+            self.log.setPlainText(raw)
+        finally:
+            self.bbox.setEnabled(True)
+
+    def copy_to_clipboard(self):
+        QApplication.clipboard().setText(self.log.toPlainText())
+
+if __name__ == '__main__':
+    app = QApplication([])
+    d = DebugDevice()
+    d.exec_()

src/calibre/gui2/notify.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from calibre.constants import islinux
+
+class Notifier(object):
+
+    DEFAULT_TIMEOUT = 5000
+
+    def get_msg_parms(self, timeout, body, summary):
+        if summary is None:
+            summary = 'calibre'
+        if timeout == 0:
+            timeout = self.DEFAULT_TIMEOUT
+        return timeout, body, summary
+
+    def __call__(self, body, summary=None, replaces_id=None, timeout=0):
+        raise NotImplementedError
+
+
+class DBUSNotifier(Notifier):
+
+    ICON = I('notify.png')
+
+    def __init__(self, server, path):
+        self.ok, self.err = True, None
+        try:
+            import dbus
+            self.dbus = dbus
+            self._notify = dbus.SessionBus().get_object(server, path)
+        except Exception, err:
+            self.ok = False
+            self.err = str(err)
+
+
+class KDENotifier(DBUSNotifier):
+
+    def __init__(self):
+        DBUSNotifier.__init__(self, 'org.kde.VisualNotifications',
+                '/VisualNotifications')
+
+    def __call__(self, body, summary=None, replaces_id=None, timeout=0):
+        if replaces_id is None:
+            replaces_id = self.dbus.UInt32()
+        event_id = ''
+        timeout, body, summary = self.get_msg_parms(timeout, body, summary)
+        self._notify.Notify('calibre', replaces_id, event_id, self.ICON, summary, body,
+                self.dbus.Array(signature='s'), self.dbus.Dictionary(signature='sv'),
+                timeout)
+
+class FDONotifier(DBUSNotifier):
+
+    def __init__(self):
+        DBUSNotifier.__init__(self, 'org.freedesktop.Notifications',
+                '/org/freedesktop/Notifications')
+
+    def __call__(self, body, summary=None, replaces_id=None, timeout=0):
+        if replaces_id is None:
+            replaces_id = self.dbus.UInt32()
+        timeout, body, summary = self.get_msg_parms(timeout, body, summary)
+        self._notify.Notify('calibre', replaces_id, self.ICON, summary, body,
+                self.dbus.Array(signature='s'), self.dbus.Dictionary(signature='sv'),
+                timeout)
+
+class QtNotifier(Notifier):
+
+    def __init__(self, systray=None):
+        self.systray = systray
+        self.ok = self.systray is not None and self.systray.supportsMessages()
+
+    def __call__(self, body, summary=None, replaces_id=None, timeout=0):
+        timeout, body, summary = self.get_msg_parms(timeout, body, summary)
+        if self.systray is not None:
+            self.systray.showMessage(summary, body, self.systray.Information,
+                    timeout)
+
+def get_notifier(systray=None):
+    ans = None
+    if islinux:
+        ans = KDENotifier()
+        if not ans.ok:
+            ans = FDONotifier()
+            if not ans.ok:
+                ans = None
+    if ans is None:
+        ans = QtNotifier(systray)
+        if not ans.ok:
+            ans = None
+    return ans
+
+
+if __name__ == '__main__':
+    n = KDENotifier()
+    n('hello')
+    n = FDONotifier()
+    n('hello')
+    '''
+    from PyQt4.Qt import QApplication, QSystemTrayIcon, QIcon
+    app = QApplication([])
+    ic = QIcon(I('notify.png'))
+    tray = QSystemTrayIcon(ic)
+    tray.setVisible(True)
+    n = QtNotifier(tray)
+    n('hello')
+    '''

src/calibre/gui2/status.py

@@ -9,6 +9,7 @@ from calibre import fit_image, preferred_encoding, isosx
 from calibre.gui2 import qstring_to_unicode, config
 from calibre.gui2.widgets import IMAGE_EXTENSIONS
 from calibre.gui2.progress_indicator import ProgressIndicator
+from calibre.gui2.notify import get_notifier
 from calibre.ebooks import BOOK_EXTENSIONS
 
 class BookInfoDisplay(QWidget):
@@ -218,6 +219,7 @@ class StatusBar(QStatusBar):
     def __init__(self, jobs_dialog, systray=None):
         QStatusBar.__init__(self)
         self.systray = systray
+        self.notifier = get_notifier(systray)
         self.movie_button = MovieButton(jobs_dialog)
         self.cover_flow_button = CoverFlowButton()
         self.tag_view_button = TagViewButton()
@@ -247,13 +249,13 @@ class StatusBar(QStatusBar):
 
     def showMessage(self, msg, timeout=0):
         ret = QStatusBar.showMessage(self, msg, timeout)
-        if self.systray is not None and not config['disable_tray_notification']:
+        if self.notifier is not None and not config['disable_tray_notification']:
             if isosx and isinstance(msg, unicode):
                 try:
                     msg = msg.encode(preferred_encoding)
                 except UnicodeEncodeError:
                     msg = msg.encode('utf-8')
-            self.systray.showMessage('calibre', msg, self.systray.Information, 10000)
+            self.notifier(msg)
         return ret
 
     def jobs(self):

src/calibre/gui2/viewer/documentview.py

@@ -587,20 +587,32 @@ class DocumentView(QWebView):
             if self.manager is not None:
                 self.manager.next_document()
         else:
+            oopos = self.document.ypos
+            #print '\nOriginal position:', oopos
             self.document.set_bottom_padding(0)
             opos = self.document.ypos
+            #print 'After set padding=0:', self.document.ypos
+            if opos < oopos:
+                if self.manager is not None:
+                    self.manager.next_document()
+                return
             lower_limit = opos + delta_y # Max value of top y co-ord after scrolling
             max_y = self.document.height - window_height # The maximum possible top y co-ord
             if max_y < lower_limit:
+                #print 'Setting padding to:', lower_limit - max_y
                 self.document.set_bottom_padding(lower_limit - max_y)
             max_y = self.document.height - window_height
             lower_limit = min(max_y, lower_limit)
+            #print 'Scroll to:', lower_limit
             if lower_limit > opos:
                 self.document.scroll_to(self.document.xpos, lower_limit)
             actually_scrolled = self.document.ypos - opos
+            #print 'After scroll pos:', self.document.ypos
             self.find_next_blank_line(window_height - actually_scrolled)
+            #print 'After blank line pos:', self.document.ypos
             if self.manager is not None:
                 self.manager.scrolled(self.scroll_fraction)
+            #print 'After all:', self.document.ypos
 
     def scroll_by(self, x=0, y=0, notify=True):
         old_pos = self.document.ypos

src/calibre/gui2/widgets.py

@@ -11,7 +11,7 @@ from PyQt4.Qt import QListView, QIcon, QFont, QLabel, QListWidget, \
                         QAbstractListModel, QVariant, Qt, SIGNAL, \
                         QRegExp, QSettings, QSize, QModelIndex, \
                         QAbstractButton, QPainter, QLineEdit, QComboBox, \
-                        QMenu, QStringListModel, QCompleter
+                        QMenu, QStringListModel, QCompleter, QStringList
 
 from calibre.gui2 import human_readable, NONE, TableView, \
                          qstring_to_unicode, error_dialog
@@ -21,9 +21,11 @@ from calibre import fit_image
 from calibre.utils.fonts import fontconfig
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.metadata.meta import metadata_from_filename
-from calibre.utils.config import prefs
+from calibre.utils.config import prefs, XMLConfig
 from calibre.gui2.progress_indicator import ProgressIndicator as _ProgressIndicator
 
+history = XMLConfig('history')
+
 class ProgressIndicator(QWidget):
 
     def __init__(self, *args):
@@ -506,16 +508,16 @@ class LineEditECM(object):
         menu.exec_(event.globalPos())
 
     def upper_case(self):
-        self.setText(qstring_to_unicode(self.text()).upper())
+        self.setText(unicode(self.text()).upper())
 
     def lower_case(self):
-        self.setText(qstring_to_unicode(self.text()).lower())
+        self.setText(unicode(self.text()).lower())
 
     def swap_case(self):
-        self.setText(qstring_to_unicode(self.text()).swapcase())
+        self.setText(unicode(self.text()).swapcase())
 
     def title_case(self):
-        self.setText(qstring_to_unicode(self.text()).title())
+        self.setText(unicode(self.text()).title())
 
 
 class EnLineEdit(LineEditECM, QLineEdit):
@@ -620,7 +622,7 @@ class EnComboBox(QComboBox):
         self.setLineEdit(EnLineEdit(self))
 
     def text(self):
-        return qstring_to_unicode(self.currentText())
+        return unicode(self.currentText())
 
     def setText(self, text):
         idx = self.findText(text, Qt.MatchFixedString)
@@ -629,6 +631,43 @@ class EnComboBox(QComboBox):
             idx = 0
         self.setCurrentIndex(idx)
 
+class HistoryLineEdit(QComboBox):
+
+    def __init__(self, *args):
+        QComboBox.__init__(self, *args)
+        self.setEditable(True)
+        self.setInsertPolicy(self.NoInsert)
+        self.setMaxCount(10)
+
+    @property
+    def store_name(self):
+        return 'lineedit_history_'+self._name
+
+    def initialize(self, name):
+        self._name = name
+        self.addItems(QStringList(history.get(self.store_name, [])))
+        self.setEditText('')
+        self.lineEdit().editingFinished.connect(self.save_history)
+
+    def save_history(self):
+        items = []
+        ct = unicode(self.currentText())
+        if ct:
+            items.append(ct)
+        for i in range(self.count()):
+            item = unicode(self.itemText(i))
+            if item not in items:
+                items.append(item)
+
+        history.set(self.store_name, items)
+
+    def setText(self, t):
+        self.setEditText(t)
+        self.lineEdit().setCursorPosition(0)
+
+    def text(self):
+        return self.currentText()
+
 class PythonHighlighter(QSyntaxHighlighter):
 
     Rules = []

src/calibre/manual/faq.rst

@@ -81,7 +81,7 @@ Device Integration
 
 What devices does |app| support?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-At the moment |app| has full support for the SONY PRS 300/500/505/600/700, Barnes & Noble Nook, Cybook Gen 3/Opus, Amazon Kindle 1/2/DX, Netronix EB600, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook 360, Android phones and the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk.
+At the moment |app| has full support for the SONY PRS 300/500/505/600/700, Barnes & Noble Nook, Cybook Gen 3/Opus, Amazon Kindle 1/2/DX, Netronix EB600, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook 360, Italica, various Android phones and the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk.
 
 How can I help get my device supported in |app|?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -91,17 +91,14 @@ We just need some information from you:
 
   * What e-book formats does your device support?
   * Is there a special directory on the device in which all e-book files should be placed?
-  * We also need the output from running the following command in a terminal, both with the device
+  * We also need information about your device that |app| will collect automatically. First, if your
-    connected and without::
+    device supports SD cards, insert them. Then connect your device. In calibre go to Preferences->Advanced
+    and click the "Debug device detection" button. This will create some debug output. Copy it to a file
+    and repeat the process, this time with your device disconnected.
+  * Send both the above outputs to us with the other information and we will write a device driver for your 
+    device.
 
-        calibre-debug -d
+Once you send us the output for a particular operating system, support for the device in that operating system
-
-  * If your device supports SD cards, run the above command with the cards inserted.
-
-To run the above command, on Windows you should use the full path to calibre-debug.exe
-On OSX, you should go to Preferences->Advanced and click "Install command line tools".
-
-Once you send us the output for a particular operating system, support for the device
 will appear in the next release of |app|.
 
 
@@ -124,6 +121,11 @@ If you do need to reset your metadata due to problems caused by using both
 at the same time, then just delete the media.xml file on the Reader using
 your PC's file explorer and it will be recreated after disconnection.
 
+With recent reader iterations, SONY, in all its wisdom has decided to try to force you to
+use their software. If you install it, it auto-launches whenever you connect the reader.
+If you don't want to uninstall it altogether, there are a couple of tricks you can use. The
+simplest is to simply re-name the executable file that launches the library program. More detail
+`here http://www.mobileread.com/forums/showthread.php?t=65809`_.
 
 Can I use the collections feature of the SONY reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/web/fetch/simple.py

@@ -194,7 +194,8 @@ class RecursiveFetcher(object):
                 purl[i] = quote(purl[i])
             url = urlparse.urlunparse(purl)
         try:
-            with closing(self.browser.open_novisit(url, timeout=self.timeout)) as f:
+            open_func = getattr(self.browser, 'open_novisit', self.browser.open)
+            with closing(open_func(url, timeout=self.timeout)) as f:
                 data = response(f.read()+f.read())
                 data.newurl = f.geturl()
         except urllib2.URLError, err: