Merge from trunk

Changelog.yaml

@@ -19,6 +19,81 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.9.15
+  date: 2013-01-18
+
+  new features:
+    - title: "Linux MTP driver: Detect devices that have MTP interfaces even if their USB ids are not known"
+
+    - title: "Content server: Allow picking a random book by clicking the 'Random book' link on the start page. You can also refresh the random book page to get a new random book"
+
+    - title: "E-book viewer: Add an option to hide the toolbars in the viewer window (Preferences->Miscellaneous->Show controls in the viewr preferences). You can unhide them by right clicking in the viewer window."
+
+    - title: "Kobo driver: Speedup initial connect by avoiding unnecessary update of series metadata in some situations."
+      tickets: [1099190]
+
+    - title: "Get Books: Allow the store plugins to be dynamically loaded so that future website changes of a store dont require a calibre update to fix Get Books."
+
+    - title: "Wireless driver: Always replace file when resending a previously sent book to the device, even if the title/author have changed."
+
+    - title: "Add PocketBook Pro 912 driver."
+      tickets: [1099571]
+
+    - title: "When creating/exporting epub and mobi files, add the calibre book identifier as a special field in the book's metadata. This allows third party tools to identify the book record in calibre to which the file belongs."
+
+    - title: "Wireless driver: Add support for using the book uuid as the filename"
+
+    - title: "Remove the experimental tag from the subset fonts feature, since there has been only one reported problem (now fixed) with it in the two months since it was released"
+ 
+  bug fixes:
+    - title: "Get Books: Update the amazon, waterstones and libri.de plugins to account for website changes"
+
+    - title: "MOBI Input: Do not choke on MOBI files with incorrectly encoded titles."
+      tickets: [1100601]
+
+    - title: "Font subsetting: Fix a bug in the parsing of the GSUB table that could cause some ligatures to not be included in the subset font"
+
+    - title: "E-book-viewer: Fix TOC links without anchors not scrolling to the top of the current flow"
+
+    - title: "LIT Input: Handle lit files that set an incorrect XML mimetype for their text."
+      tickets: [1099621]
+
+    - title: "Catalogs: Fix 'X' being droppen from isbns on export"
+      tickets: [1098325]
+
+    - title: "Fix an error when editing date in the main book list and all visible dates are blank."
+      tickets: [1098675]
+
+    - title: "Fix calibre-smtp using incorrect escaping for non-ascii attachment filenames"
+      tickets: [1098478]
+
+    - title: "Conversion: When subsetting fonts, handle multiple @font-face rules referring to the same physical font"
+
+    - title: "Content server: Update metadata when serving azw3 files"
+
+    - title: "CHM Input: Handle chm files that contain files with url unsafe filenames."
+      tickets: [1100610]
+
+    - title: "Content server: Fix custom icons for top level categories incorrect."
+      tickets: [1095016]
+
+    - title: "Kobo driver: When resending a file to the device, update the filesize in the Kobo db to prevent the device from deleting the file."
+      tickets: [1100607]
+
+  improved recipes:
+    - The Chronicle of Higher Education
+    - Smithsonian Magazine
+    - Philosophy Now
+    - The Economist
+    - Business Week Magazine
+
+  new recipes:
+    - title: Asco de Vida 
+      author: Krittika Goyal
+
+    - title: Schattenblick 
+      author: ThB
+
 - version: 0.9.14
   date: 2013-01-11
 

recipes/bwmagazine2.recipe

@@ -38,7 +38,7 @@ class BusinessWeekMagazine(BasicNewsRecipe):
             title=self.tag_to_string(div.a).strip()
             url=div.a['href']
             soup0 = self.index_to_soup(url)
-            urlprint=soup0.find('li', attrs={'class':'print'}).a['href']
+            urlprint=soup0.find('li', attrs={'class':'print tracked'}).a['href']
             articles.append({'title':title, 'url':urlprint, 'description':'', 'date':''})
 
 
@@ -55,7 +55,7 @@ class BusinessWeekMagazine(BasicNewsRecipe):
             title=self.tag_to_string(div.a).strip()
             url=div.a['href']
             soup0 = self.index_to_soup(url)
-            urlprint=soup0.find('li', attrs={'class':'print'}).a['href']
+            urlprint=soup0.find('li', attrs={'class':'print tracked'}).a['href']
             articles.append({'title':title, 'url':urlprint, 'description':desc, 'date':''})
 
             if articles:

recipes/economist.recipe

@@ -41,10 +41,11 @@ class Economist(BasicNewsRecipe):
     remove_tags = [
             dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
             dict(attrs={'class':['dblClkTrk', 'ec-article-info',
-                'share_inline_header', 'related-items']}),
+                'share_inline_header', 'related-items',
+                                 'main-content-container']}),
             {'class': lambda x: x and 'share-links-header' in x},
     ]
-    keep_only_tags = [dict(id='ec-article-body')]
+    keep_only_tags = [dict(name='article')]
     no_stylesheets = True
     preprocess_regexps = [(re.compile('</html>.*', re.DOTALL),
         lambda x:'</html>')]

recipes/economist_free.recipe

@@ -41,10 +41,11 @@ class Economist(BasicNewsRecipe):
     remove_tags = [
             dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']),
             dict(attrs={'class':['dblClkTrk', 'ec-article-info',
-                'share_inline_header', 'related-items']}),
+                'share_inline_header', 'related-items',
+                                 'main-content-container']}),
             {'class': lambda x: x and 'share-links-header' in x},
     ]
-    keep_only_tags = [dict(id='ec-article-body')]
+    keep_only_tags = [dict(name='article')]
     no_stylesheets = True
     preprocess_regexps = [(re.compile('</html>.*', re.DOTALL),
         lambda x:'</html>')]

setup/iso_639/es.po

@@ -9,14 +9,14 @@ msgstr ""
 "Project-Id-Version: calibre\n"
 "Report-Msgid-Bugs-To: FULL NAME <EMAIL@ADDRESS>\n"
 "POT-Creation-Date: 2011-11-25 14:01+0000\n"
-"PO-Revision-Date: 2012-12-28 09:13+0000\n"
+"PO-Revision-Date: 2013-01-12 08:34+0000\n"
 "Last-Translator: Jellby <Unknown>\n"
 "Language-Team: Español; Castellano <>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2012-12-29 05:00+0000\n"
-"X-Generator: Launchpad (build 16378)\n"
+"X-Launchpad-Export-Date: 2013-01-13 04:37+0000\n"
+"X-Generator: Launchpad (build 16420)\n"
 
 #. name for aaa
 msgid "Ghotuo"
@@ -9652,7 +9652,7 @@ msgstr "Haruku"
 
 #. name for hrm
 msgid "Miao; Horned"
-msgstr ""
+msgstr "Miao blanco"
 
 #. name for hro
 msgid "Haroi"
@@ -9756,7 +9756,7 @@ msgstr ""
 
 #. name for huj
 msgid "Miao; Northern Guiyang"
-msgstr ""
+msgstr "Miao de Guiyang septentrional"
 
 #. name for huk
 msgid "Hulung"
@@ -16280,7 +16280,7 @@ msgstr ""
 
 #. name for mmr
 msgid "Miao; Western Xiangxi"
-msgstr ""
+msgstr "Miao de Xiangxi occidental"
 
 #. name for mmt
 msgid "Malalamai"
@@ -17064,7 +17064,7 @@ msgstr ""
 
 #. name for muq
 msgid "Miao; Eastern Xiangxi"
-msgstr ""
+msgstr "Miao de Xiangxi oriental"
 
 #. name for mur
 msgid "Murle"
@@ -22836,7 +22836,7 @@ msgstr ""
 
 #. name for sfm
 msgid "Miao; Small Flowery"
-msgstr ""
+msgstr "Pequeño miao florido"
 
 #. name for sfs
 msgid "South African Sign Language"

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 9, 14)
+numeric_version = (0, 9, 15)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -1402,7 +1402,6 @@ class StoreEmpikStore(StoreBase):
 
     headquarters = 'PL'
     formats = ['EPUB', 'MOBI', 'PDF']
-    affiliate = True
 
 class StoreEscapeMagazineStore(StoreBase):
     name = 'EscapeMagazine'

src/calibre/devices/kobo/driver.py

@@ -1706,6 +1706,7 @@ class KOBOTOUCH(KOBO):
     def upload_books(self, files, names, on_card=None, end_session=True,
                      metadata=None):
         debug_print('KoboTouch:upload_books - %d books'%(len(files)))
+        debug_print('KoboTouch:upload_books - files=', files)
 
         result = super(KOBOTOUCH, self).upload_books(files, names, on_card, end_session, metadata)
 #        debug_print('KoboTouch:upload_books - result=', result)
@@ -1717,7 +1718,7 @@ class KOBOTOUCH(KOBO):
                                                                 '.kobo/KoboReader.sqlite'))) as connection:
                     connection.text_factory = lambda x: unicode(x, "utf-8", "ignore")
                     cursor = connection.cursor()
-                    query = "DELETE FROM content WHERE ContentID = ? AND Accessibility = 1 AND IsDownloaded = 'false'"
+                    cleanup_query = "DELETE FROM content WHERE ContentID = ? AND Accessibility = 1 AND IsDownloaded = 'false'"
 
                     for fname, cycle in result:
                         show_debug = self.is_debugging_title(fname)
@@ -1726,9 +1727,11 @@ class KOBOTOUCH(KOBO):
                             debug_print('KoboTouch:upload_books: fname=', fname)
                             debug_print('KoboTouch:upload_books: contentID=', contentID)
 
-                        t = (contentID,)
+                        cleanup_values = (contentID,)
 #                        debug_print('KoboTouch:upload_books: Delete record left if deleted on Touch')
-                        cursor.execute(query, t)
+                        cursor.execute(cleanup_query, cleanup_values)
+                        
+                        self.set_filesize_in_device_database(connection, contentID, fname)
 
                     connection.commit()
 
@@ -2183,6 +2186,43 @@ class KOBOTOUCH(KOBO):
         connection.commit()
         cursor.close()
 
+    def set_filesize_in_device_database(self, connection, contentID, fpath):
+        show_debug = self.is_debugging_title(fpath)
+        if show_debug:
+            debug_print('KoboTouch:set_filesize_in_device_database contentID="%s"'%contentID)
+
+        test_query = 'SELECT ___FileSize '     \
+                        'FROM content '        \
+                        'WHERE ContentID = ? ' \
+                        ' AND ContentType = 6'
+        test_values = (contentID, )
+
+        updatequery = 'UPDATE content '         \
+                        'SET ___FileSize = ? '  \
+                        'WHERE ContentId = ? '  \
+                        'AND ContentType = 6'
+
+        cursor = connection.cursor()
+        cursor.execute(test_query, test_values)
+        result = cursor.fetchone()
+        if result is None:
+            if show_debug:
+                debug_print('        Did not find a record - new book on device')
+        elif os.path.exists(fpath):
+            file_size = os.stat(self.normalize_path(fpath)).st_size
+            if show_debug:
+                debug_print('        Found a record - will update - ___FileSize=', result[0], ' file_size=', file_size)
+            if file_size != int(result[0]):
+                update_values = (file_size, contentID, )
+                cursor.execute(updatequery, update_values)
+                if show_debug:
+                    debug_print('        Size updated.')
+
+        connection.commit()
+        cursor.close()
+
+#        debug_print("KoboTouch:set_filesize_in_device_database - end")
+
     def delete_empty_bookshelves(self, connection):
         debug_print("KoboTouch:delete_empty_bookshelves - start")
 

src/calibre/ebooks/chm/reader.py

@@ -100,7 +100,7 @@ class CHMReader(CHMFile):
     def ExtractFiles(self, output_dir=os.getcwdu(), debug_dump=False):
         html_files = set([])
         try:
-            x = self.GetEncoding()
+            x = self.get_encoding()
             codecs.lookup(x)
             enc = x
         except:

src/calibre/ebooks/conversion/plugins/chm_input.py

@@ -7,8 +7,6 @@ import os
 
 from calibre.customize.conversion import InputFormatPlugin
 from calibre.ptempfile import TemporaryDirectory
-from calibre.utils.localization import get_lang
-from calibre.utils.filenames import ascii_filename
 from calibre.constants import filesystem_encoding
 
 class CHMInput(InputFormatPlugin):
@@ -57,22 +55,39 @@ class CHMInput(InputFormatPlugin):
             mainpath = os.path.join(tdir, mainname)
 
             metadata = get_metadata_from_reader(self._chm_reader)
+            encoding = self._chm_reader.get_encoding() or options.input_encoding or 'cp1252'
             self._chm_reader.CloseCHM()
-            #print tdir
-            #from calibre import ipython
-            #ipython()
+            # print tdir, mainpath
+            # from calibre import ipython
+            # ipython()
 
             options.debug_pipeline = None
             options.input_encoding = 'utf-8'
-            # try a custom conversion:
-            #oeb = self._create_oebbook(mainpath, tdir, options, log, metadata)
-            # try using html converter:
-            htmlpath = self._create_html_root(mainpath, log)
+            htmlpath, toc = self._create_html_root(mainpath, log, encoding)
             oeb = self._create_oebbook_html(htmlpath, tdir, options, log, metadata)
             options.debug_pipeline = odi
-            #log.debug('DEBUG: Not removing tempdir %s' % tdir)
+            if toc.count() > 1:
+                oeb.toc = self.parse_html_toc(oeb.spine[0])
+                oeb.manifest.remove(oeb.spine[0])
+                oeb.auto_generated_toc = False
         return oeb
 
+    def parse_html_toc(self, item):
+        from calibre.ebooks.oeb.base import TOC, XPath
+        dx = XPath('./h:div')
+        ax = XPath('./h:a[1]')
+
+        def do_node(parent, div):
+            for child in dx(div):
+                a = ax(child)[0]
+                c = parent.add(a.text, a.attrib['href'])
+                do_node(c, child)
+
+        toc = TOC()
+        root = XPath('//h:div[1]')(item.data)[0]
+        do_node(toc, root)
+        return toc
+
     def _create_oebbook_html(self, htmlpath, basedir, opts, log, mi):
         # use HTMLInput plugin to generate book
         from calibre.customize.builtins import HTMLInput
@@ -81,104 +96,71 @@ class CHMInput(InputFormatPlugin):
         oeb = htmlinput.create_oebbook(htmlpath, basedir, opts, log, mi)
         return oeb
 
-
-    def _create_oebbook(self, hhcpath, basedir, opts, log, mi):
-        import uuid
+    def _create_html_root(self, hhcpath, log, encoding):
         from lxml import html
-        from calibre.ebooks.conversion.plumber import create_oebbook
-        from calibre.ebooks.oeb.base import DirContainer
-        oeb = create_oebbook(log, None, opts,
-                encoding=opts.input_encoding, populate=False)
-        self.oeb = oeb
-
-        metadata = oeb.metadata
-        if mi.title:
-            metadata.add('title', mi.title)
-        if mi.authors:
-            for a in mi.authors:
-                metadata.add('creator', a, attrib={'role':'aut'})
-        if mi.publisher:
-            metadata.add('publisher', mi.publisher)
-        if mi.isbn:
-            metadata.add('identifier', mi.isbn, attrib={'scheme':'ISBN'})
-        if not metadata.language:
-            oeb.logger.warn(u'Language not specified')
-            metadata.add('language', get_lang().replace('_', '-'))
-        if not metadata.creator:
-            oeb.logger.warn('Creator not specified')
-            metadata.add('creator', _('Unknown'))
-        if not metadata.title:
-            oeb.logger.warn('Title not specified')
-            metadata.add('title', _('Unknown'))
-
-        bookid = str(uuid.uuid4())
-        metadata.add('identifier', bookid, id='uuid_id', scheme='uuid')
-        for ident in metadata.identifier:
-            if 'id' in ident.attrib:
-                self.oeb.uid = metadata.identifier[0]
-                break
-
+        from urllib import unquote as _unquote
+        from calibre.ebooks.oeb.base import urlquote
+        from calibre.ebooks.chardet import xml_to_unicode
         hhcdata = self._read_file(hhcpath)
+        hhcdata = hhcdata.decode(encoding)
+        hhcdata = xml_to_unicode(hhcdata, verbose=True,
+                            strip_encoding_pats=True, resolve_entities=True)[0]
         hhcroot = html.fromstring(hhcdata)
-        chapters = self._process_nodes(hhcroot)
+        toc = self._process_nodes(hhcroot)
         #print "============================="
         #print "Printing hhcroot"
         #print etree.tostring(hhcroot, pretty_print=True)
         #print "============================="
-        log.debug('Found %d section nodes' % len(chapters))
-
-        if len(chapters) > 0:
-            path0 = chapters[0][1]
-            subpath = os.path.dirname(path0)
-            htmlpath = os.path.join(basedir, subpath)
-
-            oeb.container = DirContainer(htmlpath, log)
-            for chapter in chapters:
-                title = chapter[0]
-                basename = os.path.basename(chapter[1])
-                self._add_item(oeb, title, basename)
-
-            oeb.container = DirContainer(htmlpath, oeb.log)
-        return oeb
-
-    def _create_html_root(self, hhcpath, log):
-        from lxml import html
-        hhcdata = self._read_file(hhcpath)
-        hhcroot = html.fromstring(hhcdata)
-        chapters = self._process_nodes(hhcroot)
-        #print "============================="
-        #print "Printing hhcroot"
-        #print etree.tostring(hhcroot, pretty_print=True)
-        #print "============================="
-        log.debug('Found %d section nodes' % len(chapters))
+        log.debug('Found %d section nodes' % toc.count())
         htmlpath = os.path.splitext(hhcpath)[0] + ".html"
-        with open(htmlpath, 'wb') as f:
-            if chapters:
-                f.write('<html><head><meta http-equiv="Content-type"'
-                    ' content="text/html;charset=UTF-8" /></head><body>\n')
-                path0 = chapters[0][1]
-                subpath = os.path.dirname(path0)
-                base = os.path.dirname(f.name)
+        base = os.path.dirname(os.path.abspath(htmlpath))
 
-                for chapter in chapters:
-                    title = chapter[0]
-                    rsrcname = os.path.basename(chapter[1])
+        def unquote(x):
+            if isinstance(x, unicode):
+                x = x.encode('utf-8')
+            return _unquote(x).decode('utf-8')
+
+        def unquote_path(x):
+            y = unquote(x)
+            if (not os.path.exists(os.path.join(base, x)) and
+                os.path.exists(os.path.join(base, y))):
+                x = y
+            return x
+
+        def donode(item, parent, base, subpath):
+            for child in item:
+                title = child.title
+                if not title: continue
+                raw = unquote_path(child.href or '')
+                rsrcname = os.path.basename(raw)
                 rsrcpath = os.path.join(subpath, rsrcname)
                 if (not os.path.exists(os.path.join(base, rsrcpath)) and
-                            os.path.exists(os.path.join(base, chapter[1]))):
-                        rsrcpath = chapter[1]
+                        os.path.exists(os.path.join(base, raw))):
+                    rsrcpath = raw
 
-                    # title should already be url encoded
-                    url = "<br /><a href=" + rsrcpath + ">" + title + " </a>\n"
-                    if isinstance(url, unicode):
-                        url = url.encode('utf-8')
-                    f.write(url)
+                if '%' not in rsrcpath:
+                    rsrcpath = urlquote(rsrcpath)
+                if not raw:
+                    rsrcpath = ''
+                c = DIV(A(title, href=rsrcpath))
+                donode(child, c, base, subpath)
+                parent.append(c)
 
-                f.write("</body></html>")
+        with open(htmlpath, 'wb') as f:
+            if toc.count() > 1:
+                from lxml.html.builder import HTML, BODY, DIV, A
+                path0 = toc[0].href
+                path0 = unquote_path(path0)
+                subpath = os.path.dirname(path0)
+                base = os.path.dirname(f.name)
+                root = DIV()
+                donode(toc, root, base, subpath)
+                raw = html.tostring(HTML(BODY(root)), encoding='utf-8',
+                                   pretty_print=True)
+                f.write(raw)
             else:
                 f.write(hhcdata)
-        return htmlpath
-
+        return htmlpath, toc
 
     def _read_file(self, name):
         f = open(name, 'rb')
@@ -186,41 +168,27 @@ class CHMInput(InputFormatPlugin):
         f.close()
         return data
 
-    def _visit_node(self, node, chapters, depth):
-        # check that node is a normal node (not a comment, DOCTYPE, etc.)
-        # (normal nodes have string tags)
-        if isinstance(node.tag, basestring):
+    def add_node(self, node, toc, ancestor_map):
         from calibre.ebooks.chm.reader import match_string
-
-            chapter_path = None
-            if match_string(node.tag, 'object') and match_string(node.attrib['type'], 'text/sitemap'):
-                chapter_title = None
-                for child in node:
-                    if match_string(child.tag,'param') and match_string(child.attrib['name'], 'name'):
-                        chapter_title = child.attrib['value']
-                    if match_string(child.tag,'param') and match_string(child.attrib['name'],'local'):
-                        chapter_path = child.attrib['value']
-                if chapter_title is not None and chapter_path is not None:
-                    chapter = [chapter_title, chapter_path, depth]
-                    chapters.append(chapter)
-            if node.tag=="UL":
-                depth = depth + 1
-            if node.tag=="/UL":
-                depth = depth - 1
+        if match_string(node.attrib['type'], 'text/sitemap'):
+            p = node.xpath('ancestor::ul[1]/ancestor::li[1]/object[1]')
+            parent = p[0] if p else None
+            toc = ancestor_map.get(parent, toc)
+            title = href = u''
+            for param in node.xpath('./param'):
+                if match_string(param.attrib['name'], 'name'):
+                    title = param.attrib['value']
+                elif match_string(param.attrib['name'], 'local'):
+                    href = param.attrib['value']
+            child = toc.add(title or _('Unknown'), href)
+            ancestor_map[node] = child
 
     def _process_nodes(self, root):
-        chapters = []
-        depth = 0
-        for node in root.iter():
-            self._visit_node(node, chapters, depth)
-        return chapters
+        from calibre.ebooks.oeb.base import TOC
+        toc = TOC()
+        ancestor_map = {}
+        for node in root.xpath('//object'):
+            self.add_node(node, toc, ancestor_map)
+        return toc
 
-    def _add_item(self, oeb, title, path):
-        bname = os.path.basename(path)
-        id, href = oeb.manifest.generate(id='html',
-                href=ascii_filename(bname))
-        item = oeb.manifest.add(id, href, 'text/html')
-        item.html_input_href = bname
-        oeb.spine.add(item, True)
-        oeb.toc.add(title, item.href)
 

src/calibre/ebooks/metadata/toc.py

@@ -194,8 +194,7 @@ class TOC(list):
                 content = content_path(np)
                 if content and text:
                     content = content[0]
-                    src = get_attr(content, attr='src')
-                    if src:
+                    # if get_attr(content, attr='src'):
                     purl = urlparse(content.get('src'))
                     href, fragment = unquote(purl[2]), unquote(purl[5])
                     nd = dest.add_item(href, fragment, text)

src/calibre/ebooks/mobi/reader/headers.py

@@ -13,6 +13,7 @@ from calibre.utils.date import parse_date
 from calibre.ebooks.mobi import MobiError
 from calibre.ebooks.metadata import MetaInformation, check_isbn
 from calibre.ebooks.mobi.langcodes import main_language, sub_language, mobi2iana
+from calibre.utils.cleantext import clean_ascii_chars
 from calibre.utils.localization import canonicalize_lang
 
 NULL_INDEX = 0xffffffff
@@ -31,6 +32,8 @@ class EXTHHeader(object): # {{{
         self.kf8_header = None
         self.uuid = self.cdetype = None
 
+        self.decode = lambda x : clean_ascii_chars(x.decode(codec, 'replace'))
+
         while left > 0:
             left -= 1
             idx, size = struct.unpack('>LL', raw[pos:pos + 8])
@@ -66,7 +69,7 @@ class EXTHHeader(object): # {{{
                 # title contains non ASCII chars or non filename safe chars
                 # they are messed up in the PDB header
                 try:
-                    title = content.decode(codec)
+                    title = self.decode(content)
                 except:
                     pass
             elif idx == 524: # Lang code
@@ -80,31 +83,30 @@ class EXTHHeader(object): # {{{
             #else:
             #    print 'unknown record', idx, repr(content)
         if title:
-            self.mi.title = replace_entities(title)
+            self.mi.title = replace_entities(clean_ascii_chars(title))
 
     def process_metadata(self, idx, content, codec):
         if idx == 100:
             if self.mi.is_null('authors'):
                 self.mi.authors = []
-            au = content.decode(codec, 'ignore').strip()
+            au = self.decode(content).strip()
             self.mi.authors.append(au)
             if self.mi.is_null('author_sort') and re.match(r'\S+?\s*,\s+\S+', au.strip()):
                 self.mi.author_sort = au.strip()
         elif idx == 101:
-            self.mi.publisher = content.decode(codec, 'ignore').strip()
+            self.mi.publisher = self.decode(content).strip()
             if self.mi.publisher in {'Unknown', _('Unknown')}:
                 self.mi.publisher = None
         elif idx == 103:
-            self.mi.comments  = content.decode(codec, 'ignore')
+            self.mi.comments  = self.decode(content).strip()
         elif idx == 104:
-            raw = check_isbn(content.decode(codec, 'ignore').strip().replace('-', ''))
+            raw = check_isbn(self.decode(content).strip().replace('-', ''))
             if raw:
                 self.mi.isbn = raw
         elif idx == 105:
             if not self.mi.tags:
                 self.mi.tags = []
-            self.mi.tags.extend([x.strip() for x in content.decode(codec,
-                'ignore').split(';')])
+            self.mi.tags.extend([x.strip() for x in self.decode(content).split(';')])
             self.mi.tags = list(set(self.mi.tags))
         elif idx == 106:
             try:
@@ -112,7 +114,7 @@ class EXTHHeader(object): # {{{
             except:
                 pass
         elif idx == 108:
-            self.mi.book_producer = content.decode(codec, 'ignore').strip()
+            self.mi.book_producer = self.decode(content).strip()
         elif idx == 112: # dc:source set in some EBSP amazon samples
             try:
                 content = content.decode(codec).strip()

src/calibre/ebooks/mobi/reader/mobi6.py

@@ -249,7 +249,10 @@ class MobiReader(object):
         head.insert(0, m)
         if not title:
             title = head.makeelement('title', {})
+            try:
                 title.text = self.book_header.title
+            except ValueError:
+                title.text = clean_ascii_chars(self.book_header.title)
             title.tail = '\n\t'
             head.insert(0, title)
             head.text = '\n\t'

src/calibre/ebooks/oeb/reader.py

@@ -373,16 +373,12 @@ class OEBReader(object):
             if not title:
                 self._toc_from_navpoint(item, toc, child)
                 continue
-            if not href:
-                gc = xpath(child, 'ncx:navPoint')
-                if not gc:
+            if (not href or not href[0]) and not xpath(child, 'ncx:navPoint'):
                 # This node is useless
                 continue
-                href = 'missing.html'
-
-            href = item.abshref(urlnormalize(href[0]))
+            href = item.abshref(urlnormalize(href[0])) if href and href[0] else ''
             path, _ = urldefrag(href)
-            if path not in self.oeb.manifest.hrefs:
+            if href and path not in self.oeb.manifest.hrefs:
                 self.logger.warn('TOC reference %r not found' % href)
                 gc = xpath(child, 'ncx:navPoint')
                 if not gc:

src/calibre/gui2/metadata/single_download.py

@@ -31,7 +31,7 @@ from calibre.utils.logging import GUILog as Log
 from calibre.ebooks.metadata.sources.identify import urls_from_identifiers
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.opf2 import OPF
-from calibre.gui2 import error_dialog, NONE, rating_font
+from calibre.gui2 import error_dialog, NONE, rating_font, gprefs
 from calibre.utils.date import (utcnow, fromordinal, format_date,
         UNDEFINED_DATE, as_utc)
 from calibre.library.comments import comments_to_html
@@ -264,6 +264,15 @@ class ResultsView(QTableView): # {{{
             sm = self.selectionModel()
             sm.select(idx, sm.ClearAndSelect|sm.Rows)
 
+    def resize_delegate(self):
+        self.rt_delegate.max_width = int(self.width()/2.1)
+        self.resizeColumnsToContents()
+
+    def resizeEvent(self, ev):
+        ret = super(ResultsView, self).resizeEvent(ev)
+        self.resize_delegate()
+        return ret
+
     def currentChanged(self, current, previous):
         ret = QTableView.currentChanged(self, current, previous)
         self.show_details(current)
@@ -385,7 +394,7 @@ class IdentifyWorker(Thread): # {{{
 
     def sample_results(self):
         m1 = Metadata('The Great Gatsby', ['Francis Scott Fitzgerald'])
-        m2 = Metadata('The Great Gatsby', ['F. Scott Fitzgerald'])
+        m2 = Metadata('The Great Gatsby - An extra long title to test resizing', ['F. Scott Fitzgerald'])
         m1.has_cached_cover_url = True
         m2.has_cached_cover_url = False
         m1.comments  = 'Some comments '*10
@@ -963,12 +972,16 @@ class FullFetch(QDialog): # {{{
         self.covers_widget.chosen.connect(self.ok_clicked)
         self.stack.addWidget(self.covers_widget)
 
+        self.resize(850, 600)
+        geom = gprefs.get('metadata_single_gui_geom', None)
+        if geom is not None and geom:
+            self.restoreGeometry(geom)
+
         # Workaround for Qt 4.8.0 bug that causes the frame of the window to go
         # off the top of the screen if a max height is not set for the
         # QWebView. Seems to only happen on windows, but keep it for all
         # platforms just in case.
-        self.identify_widget.comments_view.setMaximumHeight(500)
-        self.resize(850, 600)
+        self.identify_widget.comments_view.setMaximumHeight(self.height()-100)
 
         self.finished.connect(self.cleanup)
 
@@ -995,12 +1008,14 @@ class FullFetch(QDialog): # {{{
         self.covers_widget.reset_covers()
 
     def accept(self):
+        gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry())
         if self.stack.currentIndex() == 1:
             return QDialog.accept(self)
         # Prevent the usual dialog accept mechanisms from working
         pass
 
     def reject(self):
+        gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry())
         self.identify_widget.cancel()
         self.covers_widget.cancel()
         return QDialog.reject(self)

src/calibre/gui2/store/stores/amazon_plugin.py

@@ -7,6 +7,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import random
 from contextlib import closing
 
 from lxml import html

src/calibre/gui2/store/stores/empik_plugin.py

@@ -25,17 +25,12 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class EmpikStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
-        plain_url = 'http://www.empik.com/ebooki'
-        url = 'https://ssl.afiliant.com/affskrypt,,2f9de2,,23c7f,,,?u=(' + plain_url + ')'
-        detail_url = None
-
-        if detail_item:
-            detail_url = 'https://ssl.afiliant.com/affskrypt,,2f9de2,,23c7f,,,?u=(' + detail_item + ')'
+        url = 'http://www.empik.com/ebooki'
 
         if external or self.config.get('open_external', False):
-            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
         else:
-            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d = WebStoreDialog(self.gui, url, parent, detail_item)
             d.setWindowTitle(self.name)
             d.set_tags(self.config.get('tags', ''))
             d.exec_()

src/calibre/gui2/viewer/config.py

@@ -70,6 +70,7 @@ def config(defaults=None):
     c.add_opt('bottom_margin', default=20)
     c.add_opt('text_color', default=None)
     c.add_opt('background_color', default=None)
+    c.add_opt('show_controls', default=True)
 
     fonts = c.add_group('FONTS', _('Font options'))
     fonts('serif_family', default='Times New Roman' if iswindows else 'Liberation Serif',
@@ -221,6 +222,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
         for x in ('text', 'background'):
             setattr(self, 'current_%s_color'%x, getattr(opts, '%s_color'%x))
         self.update_sample_colors()
+        self.opt_show_controls.setChecked(opts.show_controls)
 
     def change_color(self, which, reset=False):
         if reset:
@@ -292,6 +294,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
                 self.opt_override_book_margins.isChecked())
         c.set('text_color', self.current_text_color)
         c.set('background_color', self.current_background_color)
+        c.set('show_controls', self.opt_show_controls.isChecked())
         for x in ('top', 'bottom', 'side'):
             c.set(x+'_margin', int(getattr(self, 'opt_%s_margin'%x).value()))
 

src/calibre/gui2/viewer/config.ui

@@ -347,8 +347,8 @@ QToolBox::tab:hover {
            <rect>
             <x>0</x>
             <y>0</y>
-            <width>811</width>
-            <height>352</height>
+            <width>352</width>
+            <height>176</height>
            </rect>
           </property>
           <attribute name="label">
@@ -573,8 +573,8 @@ QToolBox::tab:hover {
            <rect>
             <x>0</x>
             <y>0</y>
-            <width>352</width>
-            <height>123</height>
+            <width>811</width>
+            <height>352</height>
            </rect>
           </property>
           <attribute name="label">
@@ -605,20 +605,27 @@ QToolBox::tab:hover {
              </property>
             </widget>
            </item>
-           <item row="2" column="0" colspan="2">
+           <item row="3" column="0" colspan="2">
             <widget class="QCheckBox" name="opt_remember_window_size">
              <property name="text">
               <string>Remember last used &amp;window size and layout</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="0" colspan="2">
+           <item row="4" column="0" colspan="2">
             <widget class="QCheckBox" name="opt_remember_current_page">
              <property name="text">
               <string>Remember the &amp;current page when quitting</string>
              </property>
             </widget>
            </item>
+           <item row="2" column="0" colspan="2">
+            <widget class="QCheckBox" name="opt_show_controls">
+             <property name="text">
+              <string>Show &amp;controls in the viewer window</string>
+             </property>
+            </widget>
+           </item>
           </layout>
          </widget>
         </widget>

src/calibre/gui2/viewer/documentview.py

@@ -33,6 +33,7 @@ class Document(QWebPage): # {{{
 
     page_turn = pyqtSignal(object)
     mark_element = pyqtSignal(QWebElement)
+    settings_changed = pyqtSignal()
 
     def set_font_settings(self, opts):
         settings = self.settings()
@@ -57,6 +58,7 @@ class Document(QWebPage): # {{{
             self.set_font_settings(opts)
             self.set_user_stylesheet(opts)
             self.misc_config(opts)
+            self.settings_changed.emit()
             self.after_load()
 
     def __init__(self, shortcuts, parent=None, debug_javascript=False):
@@ -153,6 +155,7 @@ class Document(QWebPage): # {{{
         self.cols_per_screen = opts.cols_per_screen
         self.side_margin = opts.side_margin
         self.top_margin, self.bottom_margin = opts.top_margin, opts.bottom_margin
+        self.show_controls = opts.show_controls
 
     def fit_images(self):
         if self.do_fit_images and not self.in_paged_mode:
@@ -676,7 +679,7 @@ class DocumentView(QWebView): # {{{
 
         if not text and img.isNull() and self.manager is not None:
             menu.addSeparator()
-            if self.document.in_fullscreen_mode and self.manager is not None:
+            if (not self.document.show_controls or self.document.in_fullscreen_mode) and self.manager is not None:
                 menu.addAction(self.manager.toggle_toolbar_action)
             menu.addAction(self.manager.action_full_screen)
 

src/calibre/gui2/viewer/main.py

@@ -303,6 +303,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.toggle_toolbar_action = QAction(_('Show/hide controls'), self)
         self.toggle_toolbar_action.setCheckable(True)
         self.toggle_toolbar_action.triggered.connect(self.toggle_toolbars)
+        self.toolbar_hidden = None
         self.addAction(self.toggle_toolbar_action)
         self.full_screen_label_anim = QPropertyAnimation(
                 self.full_screen_label, 'size')
@@ -359,7 +360,10 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                 # continue to function even when the toolbars are hidden
                 self.addAction(action)
 
+        self.view.document.settings_changed.connect(self.settings_changed)
+
         self.restore_state()
+        self.settings_changed()
         self.action_toggle_paged_mode.toggled[bool].connect(self.toggle_paged_mode)
         if (start_in_fullscreen or self.view.document.start_in_fullscreen):
             self.action_full_screen.trigger()
@@ -373,6 +377,11 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         if at_start: return
         self.reload()
 
+    def settings_changed(self):
+        for x in ('', '2'):
+            x = getattr(self, 'tool_bar'+x)
+            x.setVisible(self.view.document.show_controls)
+
     def reload(self):
         if hasattr(self, 'current_index') and self.current_index > -1:
             self.view.document.page_position.save(overwrite=False)
@@ -575,8 +584,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.vertical_scrollbar.setVisible(True)
         self.window_mode_changed = 'normal'
         self.esc_full_screen_action.setEnabled(False)
-        self.tool_bar.setVisible(True)
-        self.tool_bar2.setVisible(True)
+        self.settings_changed()
         self.full_screen_label.setVisible(False)
         if hasattr(self, '_original_frame_margins'):
             om = self._original_frame_margins

src/calibre/gui2/viewer/toc.py

@@ -56,7 +56,7 @@ class TOCItem(QStandardItem):
         self.title = text
         self.parent = parent
         QStandardItem.__init__(self, text if text else '')
-        self.abspath = toc.abspath
+        self.abspath = toc.abspath if toc.href else None
         self.fragment = toc.fragment
         all_items.append(self)
         self.bold_font = QFont(self.font())
@@ -70,11 +70,13 @@ class TOCItem(QStandardItem):
             if si == self.abspath:
                 spos = i
                 break
+        am = {}
+        if self.abspath is not None:
             try:
                 am = getattr(spine[i], 'anchor_map', {})
             except UnboundLocalError:
                 # Spine was empty?
-            am = {}
+                pass
         frag = self.fragment if (self.fragment and self.fragment in am) else None
         self.starts_at = spos
         self.start_anchor = frag

src/calibre/library/server/browse.py

@@ -372,8 +372,10 @@ class BrowseServer(object):
             if meta['is_custom'] and category not in displayed_custom_fields:
                 continue
             # get the icon files
-            if category in self.icon_map:
-                icon = '_'+quote(self.icon_map[category])
+            main_cat = (category.partition('.')[0]) if hasattr(category,
+                                                    'partition') else category
+            if main_cat in self.icon_map:
+                icon = '_'+quote(self.icon_map[main_cat])
             elif category in category_icon_map:
                 icon = category_icon_map[category]
             elif meta['is_custom']:

src/calibre/library/server/mobile.py

@@ -77,7 +77,7 @@ def build_navigation(start, num, total, url_base): # {{{
     right_buttons = TD(CLASS('button', style='text-align:right'))
 
     if start > 1:
-        for t,s in [('First', 1), ('Previous', max(start-(num+1),1))]:
+        for t,s in [('First', 1), ('Previous', max(start-num,1))]:
             left_buttons.append(A(t, href='%s;start=%d'%(url_base, s)))
 
     if total > start + num: