Sync to trunk.

Changelog.yaml

@@ -19,6 +19,62 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.8.44
+  date: 2012-03-23
+
+  new features:
+    - title: "E-book viewer: A whole new full screen mode."
+      description: "The new mode has no toolbars to distract from the text and the ability to set the width of the column of text via Preferences in the ebook viewer. Click the Fullscreen button on the toolbar in the viewer to enter fullscreen mode (or press the F11 or Ctrl+Shit+F keys)"
+      type: major
+      tickets: [959830]
+
+    - title: "Copy to Library: If books were auto merged by the copy to library process, popup a message telling the user about it, as otherwise some people forget they have turned on auto merge and accuse calibre of losing their books."
+
+    - title: "Unix driver for Ectaco JetBook color"
+      tickets: [958442] 
+
+    - title: "Add a link to the 'Adding Books Preferences' in the drop down menu of the Add Books button for easier access and more prominence"
+      tickets: [958145] 
+
+    - title: "Smarten punctuation: Add a few more cases for detecting opening and closing quotes"
+ 
+  bug fixes:
+    - title: "Get Books: Updates to various store plugins to deal with website changes: Amazon Europe, Waterstones, Foyles, B&N, Kobo, Woblink and Empik"
+
+    - title: "Catalog generation: Do not error out when generating csv/xml catalogs if the catalog title contains filename invalid characters."
+      tickets: [960154]
+
+    - title: "RTF Output: Ignore corrupted images in the input document, instead of erroring out."
+      tickets: [959600]
+
+    - title: "E-book viewer: Try to preserve page position when the window is resized"
+
+    - title: "Fix bug that caused wrong series to be shown when clicking on the first letter of a series group in the Tag Browser"
+
+    - title: "Fix calibre not supporting different http and https proxies."
+      tickets: [960173]
+
+    - title: "MOBI Input: Fix regression caused by KF8 support that broke reading of ancient non-Amazon PRC files"
+
+    - title: "Fix EPUB to EPUB conversion of an EPUB with obfuscated fonts resulting in the fonts not being readable in Adobe Digital Editions"
+      tickets: [957527]
+
+    - title: "RTF Output: Fix bug that broke conversion to RTF when the input document contains <img> tags with no src attribute."
+
+    - title: "Fix regression in 0.8.43 that broke use of general mode templates that ended in a semi-colon."
+      tickets: [957295]
+
+  improved recipes:
+    - b92
+    - Various Polish news sources
+    - Le Monde
+    - FHM UK
+
+  new recipes:
+    - title: Ivana Milakovic and Klub knjige 
+      author: Darko Miletic
+
+
 - version: 0.8.43
   date: 2012-03-16
 

recipes/b92.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2012, Darko Miletic <darko.miletic at gmail.com>'
 '''
 b92.net
 '''
@@ -20,13 +20,13 @@ class B92(BasicNewsRecipe):
     encoding              = 'cp1250'
     language              = 'sr'
     publication_type      = 'newsportal'
-    masthead_url          = 'http://www.b92.net/images/fp/logo.gif'
+    masthead_url          = 'http://b92s.net/v4/img/new-logo.png'
     extra_css             = """ 
-                                @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)}
                                 @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)}
                                 body{font-family: Arial,Helvetica,sans1,sans-serif} 
-                                .articledescription{font-family: serif1, serif}
                                 .article-info2,.article-info1{text-transform: uppercase; font-size: small}
+                                img{display: block}
+                                .sms{font-weight: bold}
                             """
     
     conversion_options = {
@@ -37,11 +37,17 @@ class B92(BasicNewsRecipe):
                         , 'linearize_tables' : True
                         }
     
-    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+    preprocess_regexps = [
+                           (re.compile(u'\u0110'), lambda match: u'\u00D0'),
+                           (re.compile(r'<html.*?<body>', re.DOTALL|re.IGNORECASE), lambda match: '<html><head><title>something</title></head><body>')
+                         ]
     
     keep_only_tags    = [dict(attrs={'class':['article-info1','article-text']})]
-    remove_attributes = ['width','height','align','hspace','vspace','border']
+    remove_attributes = ['width','height','align','hspace','vspace','border','lang','xmlns:fb']
-    remove_tags       = [dict(name=['embed','link','base','meta'])]
+    remove_tags       = [
+                           dict(name=['embed','link','base','meta','iframe'])
+                          ,dict(attrs={'id':'social'})
+                        ]
 
     feeds          = [
                         (u'Vesti'      , u'http://www.b92.net/info/rss/vesti.xml'     )

recipes/marketing_magazine.recipe

@@ -1,4 +1,6 @@
 __license__   = 'GPL v3'
+__author__    = 'faber1971'
+description   = 'Collection of Italian marketing websites - v1.04 (17, March 2012)'
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
@@ -9,12 +11,9 @@ class AdvancedUserRecipe1327062445(BasicNewsRecipe):
     auto_cleanup = True
     remove_javascript = True
     no_stylesheets = True
+    conversion_options = {'linearize_tables': True}
     remove_tags = [
                      dict(name='ul', attrs={'id':'ads0'})
                   ]
     masthead_url            = 'http://www.simrendeogun.com/wp-content/uploads/2011/06/New-Marketing-Magazine-Logo.jpg'
-    __author__    = 'faber1971'
+    feeds          = [(u'My Marketing', u'http://feed43.com/0537744466058428.xml'), (u'My Marketing_', u'http://feed43.com/8126723074604845.xml'), (u'Venturini', u'http://robertoventurini.blogspot.com/feeds/posts/default?alt=rss'), (u'Ninja Marketing', u'http://feeds.feedburner.com/NinjaMarketing'), (u'Comunitàzione', u'http://www.comunitazione.it/feed/novita.asp'), (u'Brandforum news', u'http://www.brandforum.it/rss/news'), (u'Brandforum papers', u'http://www.brandforum.it/rss/papers'), (u'MarketingArena', u'http://feeds.feedburner.com/marketingarena'), (u'minimarketing', u'http://feeds.feedburner.com/minimarketingit'), (u'Marketing Journal', u'http://feeds.feedburner.com/marketingjournal/jPwA'), (u'Disambiguando', u'http://giovannacosenza.wordpress.com/feed/')]
-    description   = 'Collection of Italian marketing websites - v1.03 (20, February 2012)'
-    language = 'it'
-
-    feeds          = [(u'My Marketing', u'http://feed43.com/0537744466058428.xml'), (u'My Marketing_', u'http://feed43.com/8126723074604845.xml'), (u'Venturini', u'http://robertoventurini.blogspot.com/feeds/posts/default?alt=rss'), (u'Ninja Marketing', u'http://feeds.feedburner.com/NinjaMarketing'), (u'Comunitàzione', u'http://www.comunitazione.it/feed/novita.asp'), (u'Brandforum news', u'http://www.brandforum.it/rss/news'), (u'Brandforum papers', u'http://www.brandforum.it/rss/papers'), (u'MarketingArena', u'http://feeds.feedburner.com/marketingarena'), (u'minimarketing', u'http://feeds.feedburner.com/minimarketingit'), (u'Disambiguando', u'http://giovannacosenza.wordpress.com/feed/')]

setup/iso_639/hi.po

@@ -10,19 +10,19 @@ msgstr ""
 "Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
 "devel@lists.alioth.debian.org>\n"
 "POT-Creation-Date: 2011-11-25 14:01+0000\n"
-"PO-Revision-Date: 2011-09-27 16:03+0000\n"
+"PO-Revision-Date: 2012-03-18 12:56+0000\n"
-"Last-Translator: Kovid Goyal <Unknown>\n"
+"Last-Translator: Vibhav Pant <vibhavp@gmail.com>\n"
 "Language-Team: Hindi\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2011-11-26 05:19+0000\n"
+"X-Launchpad-Export-Date: 2012-03-19 04:40+0000\n"
-"X-Generator: Launchpad (build 14381)\n"
+"X-Generator: Launchpad (build 14969)\n"
 "Language: \n"
 
 #. name for aaa
 msgid "Ghotuo"
-msgstr ""
+msgstr "घोटुओ"
 
 #. name for aab
 msgid "Alumu-Tesu"
@@ -30,7 +30,7 @@ msgstr ""
 
 #. name for aac
 msgid "Ari"
-msgstr ""
+msgstr "अरी"
 
 #. name for aad
 msgid "Amal"
@@ -58,11 +58,11 @@ msgstr ""
 
 #. name for aak
 msgid "Ankave"
-msgstr ""
+msgstr "अनकावे"
 
 #. name for aal
 msgid "Afade"
-msgstr ""
+msgstr "अफ़ाडे"
 
 #. name for aam
 msgid "Aramanik"
@@ -74,7 +74,7 @@ msgstr ""
 
 #. name for aao
 msgid "Arabic; Algerian Saharan"
-msgstr ""
+msgstr "अरबी भाषा; अल्जीरियाई सहारा"
 
 #. name for aap
 msgid "Arára; Pará"
@@ -94,11 +94,11 @@ msgstr ""
 
 #. name for aat
 msgid "Albanian; Arvanitika"
-msgstr ""
+msgstr "अल्बानियन भाषा; अरवनितिका"
 
 #. name for aau
 msgid "Abau"
-msgstr ""
+msgstr "अबाऊ"
 
 #. name for aaw
 msgid "Solong"
@@ -110,7 +110,7 @@ msgstr ""
 
 #. name for aaz
 msgid "Amarasi"
-msgstr ""
+msgstr "अमारासि"
 
 #. name for aba
 msgid "Abé"
@@ -142,7 +142,7 @@ msgstr ""
 
 #. name for abh
 msgid "Arabic; Tajiki"
-msgstr ""
+msgstr "अरबी; ताजिकि"
 
 #. name for abi
 msgid "Abidji"
@@ -150,7 +150,7 @@ msgstr ""
 
 #. name for abj
 msgid "Aka-Bea"
-msgstr ""
+msgstr "अका-बीआ"
 
 #. name for abk
 msgid "Abkhazian"
@@ -166,11 +166,11 @@ msgstr ""
 
 #. name for abn
 msgid "Abua"
-msgstr ""
+msgstr "अबुआ"
 
 #. name for abo
 msgid "Abon"
-msgstr ""
+msgstr "अबोन"
 
 #. name for abp
 msgid "Ayta; Abellen"
@@ -178,7 +178,7 @@ msgstr ""
 
 #. name for abq
 msgid "Abaza"
-msgstr ""
+msgstr "अबाज़ा"
 
 #. name for abr
 msgid "Abron"
@@ -186,7 +186,7 @@ msgstr ""
 
 #. name for abs
 msgid "Malay; Ambonese"
-msgstr ""
+msgstr "मलय; अम्बोनीसी"
 
 #. name for abt
 msgid "Ambulas"

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 43)
+numeric_version = (0, 8, 44)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/ebooks/mobi/debug/headers.py

@@ -328,7 +328,7 @@ class MOBIHeader(object): # {{{
             (self.sect_idx, self.skel_idx, self.datp_idx, self.oth_idx
                     ) = struct.unpack_from(b'>4L', self.raw, 248)
             self.unknown9 = self.raw[264:self.length]
-            if self.meta_orth_indx != self.sect_idx:
+            if self.meta_orth_indx not in {NULL_INDEX, self.sect_idx}:
                 raise ValueError('KF8 header has different Meta orth and '
                         'section indices')
 

src/calibre/ebooks/rtf/rtfml.py

@@ -109,6 +109,7 @@ class RTFMLizer(object):
             if item.spine_position is None:
                 stylizer = Stylizer(item.data, item.href, self.oeb_book,
                         self.opts, self.opts.output_profile)
+                self.currently_dumping_item = item
                 output += self.dump_text(item.data.find(XHTML('body')), stylizer)
                 output += '{\\page }'
         for item in self.oeb_book.spine:
@@ -118,6 +119,7 @@ class RTFMLizer(object):
             content = self.remove_tabs(content)
             content = etree.fromstring(content)
             stylizer = Stylizer(content, item.href, self.oeb_book, self.opts, self.opts.output_profile)
+            self.currently_dumping_item = item
             output += self.dump_text(content.find(XHTML('body')), stylizer)
             output += '{\\page }'
         output += self.footer()
@@ -160,9 +162,15 @@ class RTFMLizer(object):
 
         for item in self.oeb_book.manifest:
             if item.media_type in OEB_RASTER_IMAGES:
-                src = os.path.basename(item.href)
+                src = item.href
+                try:
                     data, width, height = self.image_to_hexstring(item.data)
-                text = text.replace('SPECIAL_IMAGE-%s-REPLACE_ME' % src, '\n\n{\\*\\shppict{\\pict\\picw%i\\pich%i\\jpegblip \n%s\n}}\n\n' % (width, height, data))
+                except:
+                    self.log.warn('Image %s is corrupted, ignoring'%item.href)
+                    repl = '\n\n'
+                else:
+                    repl = '\n\n{\\*\\shppict{\\pict\\jpegblip\\picw%i\\pich%i \n%s\n}}\n\n' % (width, height, data)
+                text = text.replace('SPECIAL_IMAGE-%s-REPLACE_ME' % src, repl)
         return text
 
     def image_to_hexstring(self, data):
@@ -205,7 +213,8 @@ class RTFMLizer(object):
         return text
 
     def dump_text(self, elem, stylizer, tag_stack=[]):
-        from calibre.ebooks.oeb.base import XHTML_NS, namespace, barename
+        from calibre.ebooks.oeb.base import (XHTML_NS, namespace, barename,
+                urlnormalize)
 
         if not isinstance(elem.tag, basestring) \
            or namespace(elem.tag) != XHTML_NS:
@@ -236,7 +245,7 @@ class RTFMLizer(object):
         if tag == 'img':
             src = elem.get('src')
             if src:
-                src = os.path.basename(elem.get('src'))
+                src = urlnormalize(self.currently_dumping_item.abshref(src))
                 block_start = ''
                 block_end = ''
                 if 'block' not in tag_stack:

src/calibre/gui2/actions/catalog.py

@@ -13,6 +13,7 @@ from calibre.gui2 import choose_dir, error_dialog, warning_dialog
 from calibre.gui2.tools import generate_catalog
 from calibre.utils.config import dynamic
 from calibre.gui2.actions import InterfaceAction
+from calibre import sanitize_file_name_unicode
 
 class GenerateCatalogAction(InterfaceAction):
 
@@ -89,7 +90,8 @@ class GenerateCatalogAction(InterfaceAction):
                     _('Select destination for %(title)s.%(fmt)s') % dict(
                         title=job.catalog_title, fmt=job.fmt.lower()))
             if export_dir:
-                destination = os.path.join(export_dir, '%s.%s' % (job.catalog_title, job.fmt.lower()))
+                destination = os.path.join(export_dir, '%s.%s' % (
+                    sanitize_file_name_unicode(job.catalog_title), job.fmt.lower()))
                 shutil.copyfile(job.catalog_file_path, destination)
 
 

src/calibre/gui2/actions/copy_to_library.py

@@ -13,7 +13,8 @@ from contextlib import closing
 from PyQt4.Qt import QToolButton
 
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2 import error_dialog, Dispatcher, warning_dialog, gprefs
+from calibre.gui2 import (error_dialog, Dispatcher, warning_dialog, gprefs,
+        info_dialog)
 from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.date import now
@@ -30,6 +31,7 @@ class Worker(Thread): # {{{
         self.progress = progress
         self.done = done
         self.delete_after = delete_after
+        self.auto_merged_ids = {}
 
     def run(self):
         try:
@@ -79,6 +81,8 @@ class Worker(Thread): # {{{
             if prefs['add_formats_to_existing']:
                 identical_book_list = newdb.find_identical_books(mi)
                 if identical_book_list: # books with same author and nearly same title exist in newdb
+                    self.auto_merged_ids[x] = _('%(title)s by %(author)s')%\
+                    dict(title=mi.title, author=mi.format_field('authors')[1])
                     automerged = True
                     seen_fmts = set()
                     for identical_book in identical_book_list:
@@ -196,6 +200,15 @@ class CopyToLibraryAction(InterfaceAction):
             self.gui.status_bar.show_message(
                     _('Copied %(num)d books to %(loc)s') %
                     dict(num=len(ids), loc=loc), 2000)
+            if self.worker.auto_merged_ids:
+                books = '\n'.join(self.worker.auto_merged_ids.itervalues())
+                info_dialog(self.gui, _('Auto merged'),
+                        _('Some books were automatically merged into existing '
+                            'records in the target library. Click Show '
+                            'details to see which ones. This behavior is '
+                            'controlled by the Auto merge option in '
+                            'Preferences->Adding books.'), det_msg=books,
+                        show=True)
             if delete_after and self.worker.processed:
                 v = self.gui.library_view
                 ci = v.currentIndex()

src/calibre/gui2/comments_editor.py

@@ -9,10 +9,10 @@ import re, os
 
 from lxml import html
 
-from PyQt4.Qt import QApplication, QFontInfo, QSize, QWidget, QPlainTextEdit, \
+from PyQt4.Qt import (QApplication, QFontInfo, QSize, QWidget, QPlainTextEdit,
-    QToolBar, QVBoxLayout, QAction, QIcon, Qt, QTabWidget, QUrl, \
+    QToolBar, QVBoxLayout, QAction, QIcon, Qt, QTabWidget, QUrl,
-    QSyntaxHighlighter, QColor, QChar, QColorDialog, QMenu, QInputDialog, \
+    QSyntaxHighlighter, QColor, QChar, QColorDialog, QMenu, QInputDialog,
-    QHBoxLayout
+    QHBoxLayout, QKeySequence)
 from PyQt4.QtWebKit import QWebView, QWebPage
 
 from calibre.ebooks.chardet import xml_to_unicode
@@ -32,6 +32,7 @@ class PageAction(QAction): # {{{
                 type=Qt.QueuedConnection)
         self.page_action.changed.connect(self.update_state,
                 type=Qt.QueuedConnection)
+        self.update_state()
 
     @property
     def page_action(self):
@@ -66,6 +67,12 @@ class EditorWidget(QWebView): # {{{
 
         self.comments_pat = re.compile(r'<!--.*?-->', re.DOTALL)
 
+        extra_shortcuts = {
+                'ToggleBold': 'Bold',
+                'ToggleItalic': 'Italic',
+                'ToggleUnderline': 'Underline',
+        }
+
         for wac, name, icon, text, checkable in [
                 ('ToggleBold', 'bold', 'format-text-bold', _('Bold'), True),
                 ('ToggleItalic', 'italic', 'format-text-italic', _('Italic'),
@@ -106,6 +113,9 @@ class EditorWidget(QWebView): # {{{
             ]:
             ac = PageAction(wac, icon, text, checkable, self)
             setattr(self, 'action_'+name, ac)
+            ss = extra_shortcuts.get(wac, None)
+            if ss:
+                ac.setShortcut(QKeySequence(getattr(QKeySequence, ss)))
 
         self.action_color = QAction(QIcon(I('format-text-color')), _('Foreground color'),
                 self)

src/calibre/gui2/complete.py

@@ -6,8 +6,8 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 
-from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
+from PyQt4.Qt import (QLineEdit, QAbstractListModel, Qt,
-        QApplication, QCompleter
+        QApplication, QCompleter, QMetaObject)
 
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
@@ -182,14 +182,27 @@ class MultiCompleteComboBox(EnComboBox):
     def set_add_separator(self, what):
         self.lineEdit().set_add_separator(what)
 
-
+    def show_initial_value(self, what):
+        '''
+        Show an initial value. Handle the case of the initial value being blank
+        correctly (on Qt 4.8.0 having a blank value causes the first value from
+        the completer to be shown, when the event loop runs).
+        '''
+        what = unicode(what)
+        le = self.lineEdit()
+        if not what.strip():
+            QMetaObject.invokeMethod(self, 'clearEditText',
+                    Qt.QueuedConnection)
+        else:
+            self.setEditText(what)
+            le.selectAll()
 
 if __name__ == '__main__':
     from PyQt4.Qt import QDialog, QVBoxLayout
     app = QApplication([])
     d = QDialog()
     d.setLayout(QVBoxLayout())
-    le = MultiCompleteLineEdit(d)
+    le = MultiCompleteComboBox(d)
     d.layout().addWidget(le)
     le.all_items = ['one', 'otwo', 'othree', 'ooone', 'ootwo', 'oothree']
     d.exec_()

src/calibre/gui2/library/delegates.py

@@ -128,8 +128,7 @@ class TextDelegate(QStyledItemDelegate): # {{{
             for item in sorted(complete_items, key=sort_key):
                 editor.addItem(item)
             ct = index.data(Qt.DisplayRole).toString()
-            editor.setEditText(ct)
+            editor.show_initial_value(ct)
-            editor.lineEdit().selectAll()
         else:
             editor = EnLineEdit(parent)
         return editor
@@ -170,8 +169,7 @@ class CompleteDelegate(QStyledItemDelegate): # {{{
             for item in sorted(all_items, key=sort_key):
                 editor.addItem(item)
             ct = index.data(Qt.DisplayRole).toString()
-            editor.setEditText(ct)
+            editor.show_initial_value(ct)
-            editor.lineEdit().selectAll()
         else:
             editor = EnLineEdit(parent)
         return editor
@@ -190,8 +188,7 @@ class LanguagesDelegate(QStyledItemDelegate): # {{{
         editor = LanguagesEdit(parent=parent)
         editor.init_langs(index.model().db)
         ct = index.data(Qt.DisplayRole).toString()
-        editor.setEditText(ct)
+        editor.show_initial_value(ct)
-        editor.lineEdit().selectAll()
         return editor
 
     def setModelData(self, editor, model, index):

src/calibre/gui2/metadata/single_download.py

@@ -882,6 +882,11 @@ class FullFetch(QDialog): # {{{
         self.covers_widget.chosen.connect(self.ok_clicked)
         self.stack.addWidget(self.covers_widget)
 
+        # Workaround for Qt 4.8.0 bug that causes the frame of the window to go
+        # off the top of the screen if a max height is not set for the
+        # QWebView. Seems to only happen on windows, but keep it for all
+        # platforms just in case.
+        self.identify_widget.comments_view.setMaximumHeight(500)
         self.resize(850, 550)
 
         self.finished.connect(self.cleanup)

src/calibre/gui2/tag_browser/model.py

@@ -1170,6 +1170,8 @@ class TagsModel(QAbstractItemModel): # {{{
                         charclass = ''.join(letters_seen)
                         if k == 'author_sort':
                             expr = r'%s:"~(^[%s])|(&\s*[%s])"'%(k, charclass, charclass)
+                        elif k == 'series':
+                            expr = r'series_sort:"~^[%s]"'%(charclass)
                         else:
                             expr = r'%s:"~^[%s]"'%(k, charclass)
                         if node_searches[tag_item.tag.state] == 'true':

src/calibre/gui2/viewer/documentview.py

@@ -8,7 +8,7 @@ import os, math, re, glob, sys, zipfile
 from base64 import b64encode
 from functools import partial
 
-from PyQt4.Qt import (QSize, QSizePolicy, QUrl, SIGNAL, Qt, QTimer,
+from PyQt4.Qt import (QSize, QSizePolicy, QUrl, SIGNAL, Qt,
                      QPainter, QPalette, QBrush, QFontDatabase, QDialog,
                      QColor, QPoint, QImage, QRegion, QVariant, QIcon,
                      QFont, pyqtSignature, QAction, QByteArray, QMenu,
@@ -184,12 +184,10 @@ class Document(QWebPage): # {{{
                 self.misc_config()
                 self.after_load()
 
-    def __init__(self, shortcuts, parent=None, resize_callback=lambda: None,
+    def __init__(self, shortcuts, parent=None, debug_javascript=False):
-            debug_javascript=False):
         QWebPage.__init__(self, parent)
         self.setObjectName("py_bridge")
         self.debug_javascript = debug_javascript
-        self.resize_callback = resize_callback
         self.current_language = None
         self.loaded_javascript = False
         self.js_loader = JavaScriptLoader(
@@ -259,12 +257,6 @@ class Document(QWebPage): # {{{
         if self.loaded_javascript:
             return
         self.loaded_javascript = True
-        self.javascript(
-            '''
-            window.onresize = function(event) {
-                window.py_bridge.window_resized();
-            }
-            ''')
         self.loaded_lang = self.js_loader(self.mainFrame().evaluateJavaScript,
                 self.current_language, self.hyphenate_default_lang)
 
@@ -310,10 +302,6 @@ class Document(QWebPage): # {{{
     def debug(self, msg):
         prints(msg)
 
-    @pyqtSignature('')
-    def window_resized(self):
-        self.resize_callback()
-
     def reference_mode(self, enable):
         self.javascript(('enter' if enable else 'leave')+'_reference_mode()')
 
@@ -444,7 +432,7 @@ class Document(QWebPage): # {{{
     def scroll_fraction(self):
         def fget(self):
             try:
-                return float(self.ypos)/(self.height-self.window_height)
+                return abs(float(self.ypos)/(self.height-self.window_height))
             except ZeroDivisionError:
                 return 0.
         def fset(self, val):
@@ -516,7 +504,6 @@ class DocumentView(QWebView): # {{{
         self.initial_pos = 0.0
         self.to_bottom = False
         self.document = Document(self.shortcuts, parent=self,
-                resize_callback=self.viewport_resized,
                 debug_javascript=debug_javascript)
         self.setPage(self.document)
         self.manager = None
@@ -722,6 +709,7 @@ class DocumentView(QWebView): # {{{
         if self.manager is not None:
             self.manager.load_started()
         self.loading_url = QUrl.fromLocalFile(path)
+        html = re.sub(ur'<\s*title\s*/\s*>', u'', html, flags=re.IGNORECASE)
         if has_svg:
             self.setContent(QByteArray(html.encode(path.encoding)), mt, QUrl.fromLocalFile(path))
         else:
@@ -1035,13 +1023,9 @@ class DocumentView(QWebView): # {{{
         return handled
 
     def resizeEvent(self, event):
-        ret = QWebView.resizeEvent(self, event)
-        QTimer.singleShot(10, self.initialize_scrollbar)
-        return ret
-
-    def viewport_resized(self):
         if self.manager is not None:
-            self.manager.viewport_resized(self.scroll_fraction)
+            self.manager.viewport_resize_started(event)
+        return QWebView.resizeEvent(self, event)
 
     def event(self, ev):
         if ev.type() == ev.Gesture:

src/calibre/gui2/viewer/main.py

@@ -224,6 +224,10 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.toc.setVisible(False)
         self.action_quit = QAction(self)
         self.addAction(self.action_quit)
+        self.view_resized_timer = QTimer(self)
+        self.view_resized_timer.timeout.connect(self.viewport_resize_finished)
+        self.view_resized_timer.setSingleShot(True)
+        self.resize_in_progress = False
         qs = [Qt.CTRL+Qt.Key_Q]
         if isosx:
             qs += [Qt.CTRL+Qt.Key_W]
@@ -264,6 +268,9 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.connect(self.action_full_screen, SIGNAL('triggered(bool)'),
                      self.toggle_fullscreen)
         self.action_full_screen.setShortcuts([Qt.Key_F11, Qt.CTRL+Qt.SHIFT+Qt.Key_F])
+        self.action_full_screen.setToolTip(_('Toggle full screen (%s)') %
+                _(' or ').join([unicode(x.toString(x.NativeText)) for x in
+                    self.action_full_screen.shortcuts()]))
         self.connect(self.action_back, SIGNAL('triggered(bool)'), self.back)
         self.connect(self.action_bookmark, SIGNAL('triggered(bool)'), self.bookmark)
         self.connect(self.action_forward, SIGNAL('triggered(bool)'), self.forward)
@@ -311,6 +318,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             border-radius: 20px;
         }
         ''')
+        self.window_mode_changed = None
         self.toggle_toolbar_action = QAction(_('Show/hide controls'), self)
         self.toggle_toolbar_action.triggered.connect(self.toggle_toolbars)
         self.addAction(self.toggle_toolbar_action)
@@ -441,6 +449,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             self.showFullScreen()
 
     def showFullScreen(self):
+        self.view.document.page_position.save()
+        self.window_mode_changed = 'fullscreen'
         self.tool_bar.setVisible(False)
         self.tool_bar2.setVisible(False)
         self._original_frame_margins = (
@@ -450,7 +460,6 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.centralwidget.layout().setContentsMargins(0, 0, 0, 0)
 
         super(EbookViewer, self).showFullScreen()
-        QTimer.singleShot(10, self.show_full_screen_label)
 
     def show_full_screen_label(self):
         f = self.full_screen_label
@@ -469,6 +478,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.view.document.switch_to_fullscreen_mode()
 
     def showNormal(self):
+        self.view.document.page_position.save()
+        self.window_mode_changed = 'normal'
         self.esc_full_screen_action.setEnabled(False)
         self.tool_bar.setVisible(True)
         self.tool_bar2.setVisible(True)
@@ -478,7 +489,16 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             self.centralwidget.layout().setContentsMargins(om[0])
             self.frame.layout().setContentsMargins(om[1])
         super(EbookViewer, self).showNormal()
+
+    def handle_window_mode_toggle(self):
+        if self.window_mode_changed:
+            fs = self.window_mode_changed == 'fullscreen'
+            self.window_mode_changed = None
+            if fs:
+                self.show_full_screen_label()
+            else:
                 self.view.document.switch_to_window_mode()
+            self.view.document.page_position.restore()
 
     def goto(self, ref):
         if ref:
@@ -507,6 +527,10 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
     def toc_clicked(self, index):
         item = self.toc_model.itemFromIndex(index)
         if item.abspath is not None:
+            if not os.path.exists(item.abspath):
+                return error_dialog(self, _('No such location'),
+                        _('The location pointed to by this item'
+                            ' does not exist.'), show=True)
             url = QUrl.fromLocalFile(item.abspath)
             if item.fragment:
                 url.setFragment(item.fragment)
@@ -674,16 +698,28 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.open_progress_indicator(_('Laying out %s')%self.current_title)
         self.view.load_path(path, pos=pos)
 
-    def viewport_resized(self, frac):
+    def viewport_resize_started(self, event):
-        new_page = self.pos.value()
+        if not self.resize_in_progress:
-        if self.current_page is not None:
+            # First resize, so save the current page position
-            try:
+            self.resize_in_progress = True
-                frac = float(new_page-self.current_page.start_page)/(self.current_page.pages-1)
+            if not self.window_mode_changed:
-            except ZeroDivisionError:
+                # The special handling for window mode changed will already
-                frac = 0
+                # have saved page position, so only save it if this is not a
-            self.view.scroll_to(frac, notify=False)
+                # mode change
+                self.view.document.page_position.save()
+
+        if self.resize_in_progress:
+            self.view_resized_timer.start(75)
+
+    def viewport_resize_finished(self):
+        # There hasn't been a resize event for some time
+        # restore the current page position.
+        self.resize_in_progress = False
+        if self.window_mode_changed:
+            # This resize is part of a window mode change, special case it
+            self.handle_window_mode_toggle()
         else:
-            self.set_page_number(frac)
+            self.view.document.page_position.restore()
 
     def close_progress_indicator(self):
         self.pi.stop()

src/calibre/gui2/viewer/main.ui

@@ -284,9 +284,6 @@
    <property name="text">
     <string>Toggle full screen</string>
    </property>
-   <property name="toolTip">
-    <string>Toggle full screen (F11)</string>
-   </property>
   </action>
   <action name="action_print">
    <property name="icon">

src/calibre/gui2/viewer/position.py

@@ -57,12 +57,20 @@ class PagePosition(object):
         return ans
 
     def __enter__(self):
-        self._cpos = self.current_pos
+        self.save()
 
     def __exit__(self, *args):
+        self.restore()
+
+    def save(self):
+        self._cpos = self.current_pos
+
+    def restore(self):
+        if self._cpos is None: return
         if isinstance(self._cpos, (int, float)):
             self.document.scroll_fraction = self._cpos
         else:
             self.scroll_to_cfi(self._cpos)
         self._cpos = None
 
+

src/calibre/library/caches.py

@@ -172,11 +172,14 @@ def force_to_bool(val):
 
 class CacheRow(list): # {{{
 
-    def __init__(self, db, composites, val):
+    def __init__(self, db, composites, val, series_col, series_sort_col):
         self.db = db
         self._composites = composites
         list.__init__(self, val)
         self._must_do = len(composites) > 0
+        self._series_col = series_col
+        self._series_sort_col = series_sort_col
+        self._series_sort = None
 
     def __getitem__(self, col):
         if self._must_do:
@@ -191,12 +194,19 @@ class CacheRow(list): # {{{
             elif col in self._composites:
                 is_comp = True
             if is_comp:
-                id = list.__getitem__(self, 0)
+                id_ = list.__getitem__(self, 0)
                 self._must_do = False
-                mi = self.db.get_metadata(id, index_is_id=True,
+                mi = self.db.get_metadata(id_, index_is_id=True,
                                           get_user_categories=False)
                 for c in self._composites:
                     self[c] =  mi.get(self._composites[c])
+        if col == self._series_sort_col and self._series_sort is None:
+            if self[self._series_col]:
+                self._series_sort = title_sort(self[self._series_col])
+                self[self._series_sort_col] = self._series_sort
+            else:
+                self._series_sort = ''
+                self[self._series_sort_col] = ''
         return list.__getitem__(self, col)
 
     def __getslice__(self, i, j):
@@ -226,6 +236,8 @@ class ResultCache(SearchQueryParser): # {{{
         for key in field_metadata:
             if field_metadata[key]['datatype'] == 'composite':
                 self.composites[field_metadata[key]['rec_index']] = key
+        self.series_col = field_metadata['series']['rec_index']
+        self.series_sort_col = field_metadata['series_sort']['rec_index']
         self._data = []
         self._map = self._map_filtered = []
         self.first_sort = True
@@ -918,9 +930,11 @@ class ResultCache(SearchQueryParser): # {{{
         for id in ids:
             try:
                 self._data[id] = CacheRow(db, self.composites,
-                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
+                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0],
+                        self.series_col, self.series_sort_col)
                 self._data[id].append(db.book_on_device_string(id))
                 self._data[id].append(self.marked_ids_dict.get(id, None))
+                self._data[id].append(None)
             except IndexError:
                 return None
         try:
@@ -935,9 +949,11 @@ class ResultCache(SearchQueryParser): # {{{
         self._data.extend(repeat(None, max(ids)-len(self._data)+2))
         for id in ids:
             self._data[id] = CacheRow(db, self.composites,
-                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
+                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0],
+                        self.series_col, self.series_sort_col)
             self._data[id].append(db.book_on_device_string(id))
             self._data[id].append(self.marked_ids_dict.get(id, None))
+            self._data[id].append(None) # Series sort column
         self._map[0:0] = ids
         self._map_filtered[0:0] = ids
 
@@ -962,11 +978,13 @@ class ResultCache(SearchQueryParser): # {{{
         temp = db.conn.get('SELECT * FROM meta2')
         self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else []
         for r in temp:
-            self._data[r[0]] = CacheRow(db, self.composites, r)
+            self._data[r[0]] = CacheRow(db, self.composites, r,
+                                        self.series_col, self.series_sort_col)
         for item in self._data:
             if item is not None:
                 item.append(db.book_on_device_string(item[0]))
-                item.append(None)
+                # Temp mark and series_sort columns
+                item.extend((None, None))
 
         marked_col = self.FIELD_MAP['marked']
         for id_,val in self.marked_ids_dict.iteritems():

src/calibre/library/check_library.py

@@ -47,8 +47,8 @@ class CheckLibrary(object):
         self.is_case_sensitive = db.is_case_sensitive
 
         self.all_authors = frozenset([x[1] for x in db.all_authors()])
-        self.all_ids = frozenset([id for id in db.all_ids()])
+        self.all_ids = frozenset([id_ for id_ in db.all_ids()])
-        self.all_dbpaths = frozenset(self.dbpath(id) for id in self.all_ids)
+        self.all_dbpaths = frozenset(self.dbpath(id_) for id_ in self.all_ids)
         self.all_lc_dbpaths = frozenset([f.lower() for f in self.all_dbpaths])
 
         self.db_id_regexp = re.compile(r'^.* \((\d+)\)$')
@@ -73,8 +73,8 @@ class CheckLibrary(object):
 
         self.failed_folders = []
 
-    def dbpath(self, id):
+    def dbpath(self, id_):
-        return self.db.path(id, index_is_id=True)
+        return self.db.path(id_, index_is_id=True)
 
     @property
     def errors_occurred(self):
@@ -116,21 +116,21 @@ class CheckLibrary(object):
                     self.invalid_titles.append((auth_dir, db_path, 0))
                     continue
 
-                id = m.group(1)
+                id_ = m.group(1)
-                # Third check: the id must be in the DB and the paths must match
+                # Third check: the id_ must be in the DB and the paths must match
                 if self.is_case_sensitive:
-                    if int(id) not in self.all_ids or \
+                    if int(id_) not in self.all_ids or \
                             db_path not in self.all_dbpaths:
                         self.extra_titles.append((title_dir, db_path, 0))
                         continue
                 else:
-                    if int(id) not in self.all_ids or \
+                    if int(id_) not in self.all_ids or \
                             db_path.lower() not in self.all_lc_dbpaths:
                         self.extra_titles.append((title_dir, db_path, 0))
                         continue
 
                 # Record the book to check its formats
-                self.book_dirs.append((db_path, title_dir, id))
+                self.book_dirs.append((db_path, title_dir, id_))
                 found_titles = True
 
             # Fourth check: author directories that contain no titles
@@ -145,6 +145,21 @@ class CheckLibrary(object):
                 # Sort-of check: exception processing directory
                 self.failed_folders.append((title_path, traceback.format_exc(), []))
 
+        # Check for formats and covers in db for book dirs that are gone
+        for id_ in self.all_ids:
+            path = self.dbpath(id_)
+            if not os.path.exists(os.path.join(lib, path)):
+                title_dir = os.path.basename(path)
+                book_formats = frozenset([x for x in
+                            self.db.format_files(id_, index_is_id=True)])
+                for fmt in book_formats:
+                    self.missing_formats.append((title_dir,
+                            os.path.join(path, fmt[0]+'.'+fmt[1].lower()), id_))
+                if self.db.has_cover(id_):
+                    self.missing_covers.append((title_dir,
+                            os.path.join(path, 'cover.jpg'), id_))
+
+
     def is_ebook_file(self, filename):
         ext = os.path.splitext(filename)[1]
         if not ext:
@@ -226,8 +241,8 @@ class CheckLibrary(object):
         if self.db.has_cover(book_id):
             if 'cover.jpg' not in filenames:
                 self.missing_covers.append((title_dir,
-                        os.path.join(db_path, title_dir, 'cover.jpg'), book_id))
+                        os.path.join(db_path, 'cover.jpg'), book_id))
         else:
             if 'cover.jpg' in filenames:
                 self.extra_covers.append((title_dir,
-                        os.path.join(db_path, title_dir, 'cover.jpg'), book_id))
+                        os.path.join(db_path, 'cover.jpg'), book_id))

src/calibre/library/cli.py

@@ -204,7 +204,8 @@ class DevNull(object):
         pass
 NULL = DevNull()
 
-def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
+def do_add(db, paths, one_book_per_directory, recurse, add_duplicates, otitle,
+        oauthors, oisbn, otags, oseries, oseries_index):
     orig = sys.stdout
     #sys.stdout = NULL
     try:
@@ -231,6 +232,11 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
                 mi.title = os.path.splitext(os.path.basename(book))[0]
             if not mi.authors:
                 mi.authors = [_('Unknown')]
+            for x in ('title', 'authors', 'isbn', 'tags', 'series'):
+                val = locals()[x]
+                if val: setattr(mi, x[1:], val)
+            if oseries:
+                mi.series_index = oseries_index
 
             formats.append(format)
             metadata.append(mi)
@@ -302,39 +308,56 @@ the directory related options below.
     parser.add_option('-e', '--empty', action='store_true', default=False,
                     help=_('Add an empty book (a book with no formats)'))
     parser.add_option('-t', '--title', default=None,
-            help=_('Set the title of the added empty book'))
+            help=_('Set the title of the added book(s)'))
     parser.add_option('-a', '--authors', default=None,
-            help=_('Set the authors of the added empty book'))
+            help=_('Set the authors of the added book(s)'))
     parser.add_option('-i', '--isbn', default=None,
-            help=_('Set the ISBN of the added empty book'))
+            help=_('Set the ISBN of the added book(s)'))
+    parser.add_option('-T', '--tags', default=None,
+            help=_('Set the tags of the added book(s)'))
+    parser.add_option('-s', '--series', default=None,
+            help=_('Set the series of the added book(s)'))
+    parser.add_option('-S', '--series-index', default=1.0, type=float,
+            help=_('Set the series number of the added book(s)'))
+
 
     return parser
 
-def do_add_empty(db, title, authors, isbn):
+def do_add_empty(db, title, authors, isbn, tags, series, series_index):
-    from calibre.ebooks.metadata import MetaInformation, string_to_authors
+    from calibre.ebooks.metadata import MetaInformation
     mi = MetaInformation(None)
     if title is not None:
         mi.title = title
     if authors:
-        mi.authors = string_to_authors(authors)
+        mi.authors = authors
     if isbn:
         mi.isbn = isbn
+    if tags:
+        mi.tags = tags
+    if series:
+        mi.series, mi.series_index = series, series_index
     db.import_book(mi, [])
     write_dirtied(db)
     send_message()
 
 def command_add(args, dbpath):
+    from calibre.ebooks.metadata import string_to_authors
     parser = add_option_parser()
     opts, args = parser.parse_args(sys.argv[:1] + args)
+    aut = string_to_authors(opts.authors) if opts.authors else []
+    tags = [x.strip() for x in opts.tags.split(',')] if opts.tags else []
     if opts.empty:
-        do_add_empty(get_db(dbpath, opts), opts.title, opts.authors, opts.isbn)
+        do_add_empty(get_db(dbpath, opts), opts.title, aut, opts.isbn, tags,
+                opts.series, opts.series_index)
         return 0
     if len(args) < 2:
         parser.print_help()
         print
         print >>sys.stderr, _('You must specify at least one file to add')
         return 1
-    do_add(get_db(dbpath, opts), args[1:], opts.one_book_per_directory, opts.recurse, opts.duplicates)
+    do_add(get_db(dbpath, opts), args[1:], opts.one_book_per_directory,
+            opts.recurse, opts.duplicates, opts.title, opts.author, opts.isbn,
+            tags, opts.series, opts.series_index)
     return 0
 
 def do_remove(db, ids):

src/calibre/library/database2.py

@@ -434,6 +434,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.field_metadata.set_field_record_index('ondevice', base, prefer_custom=False)
         self.FIELD_MAP['marked'] = base = base+1
         self.field_metadata.set_field_record_index('marked', base, prefer_custom=False)
+        self.FIELD_MAP['series_sort'] = base = base+1
+        self.field_metadata.set_field_record_index('series_sort', base, prefer_custom=False)
 
         script = '''
         DROP VIEW IF EXISTS meta2;

src/calibre/library/field_metadata.py

@@ -327,6 +327,16 @@ class FieldMetadata(dict):
                              'is_custom':False,
                              'is_category':False,
                            'is_csp': False}),
+            ('series_sort',  {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':{},
+                           'kind':'field',
+                           'name':_('Series Sort'),
+                           'search_terms':['series_sort'],
+                           'is_custom':False,
+                           'is_category':False,
+                           'is_csp': False}),
             ('sort',      {'table':None,
                            'column':None,
                            'datatype':'text',

src/calibre/manual/template_lang.rst

@@ -298,6 +298,7 @@ The following functions are available in addition to those described in single-f
     * ``or(value, value, ...)`` -- returns the string "1" if any value is not empty, otherwise returns the empty string. This function works well with test or first_non_empty. You can have as many values as you want.
     * ``print(a, b, ...)`` -- prints the arguments to standard output. Unless you start calibre from the command line (``calibre-debug -g``), the output will go to a black hole.
     * ``raw_field(name)`` -- returns the metadata field named by name without applying any formatting.
+    * ``series_sort()`` -- returns the series sort value.
     * ``strcat(a, b, ...)`` -- can take any number of arguments. Returns a string formed by concatenating all the arguments.
     * ``strcat_max(max, string1, prefix2, string2, ...)`` -- Returns a string formed by concatenating the arguments. The returned value is initialized to string1. `Prefix, string` pairs are added to the end of the value as long as the resulting string length is less than `max`. String1 is returned even if string1 is longer than max. You can pass as many `prefix, string` pairs as you wish.
     * ``strcmp(x, y, lt, eq, gt)`` -- does a case-insensitive comparison x and y as strings. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.