Merge from trunk

Changelog.yaml

@@ -20,6 +20,54 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.9.29
+  date: 2013-05-03
+
+  new features:
+    - title: "Bulk metadata download: Allow reviewing of the downloaded metadata before it is applied"
+
+    - title: "FB2 Output: Write ISBN, pubdate, tags and publisher metadata when creating fb2 files"
+      tickets: [1174047]
+ 
+  bug fixes:
+    - title: "When reading metadata from EPUB 3 files, use the first <dc:title> element rather than the last."
+      tickets: [1175184]
+
+    - title: "Fix regression causing the search query parser to not parse search string containing newlines/tabs instead of spaces correctly"
+      tickets: [1174629]
+
+    - title: "Kobo driver: Fix covers written to wrong place on OS X/linux when books sent to SD card. Fix covers not sent to SD card is images directory missing."
+      tickets: [1174147,1174126]
+
+    - title: "Fix 'Preferences->Behavior->Virtual library to use when this library is opened' being applied only on calibre startup and not when switching to the library"
+
+    - title: "PDF metadata: When rendering the first page as the cover, respect the PDF CropBox."
+      tickets: [1173795]
+
+    - title: "PDF Output: Fix link generation broken on windows when converting epubs if the filenames contained uppercase letters."
+      tickets: [1169795]
+
+    - title: "Tolino driver: Fix card and main memory swapped on windows"
+      tickets: [1173544]
+
+    - title: "FB2 Output: Fix images being ignored when converting a EPUB with image filenames that contain URL unsafe characters."
+      tickets: [1173351]
+
+    - title: "EPUB Input: Fix page margins specified in Adobe page template files with incorrect mime-types not being removed."
+
+  improved recipes:
+    - The New Republic
+    - io9
+    - What if
+    - Orlando Sentinel
+    - Read It Later recipe
+    - Smithsonian
+    - Business Week Magazine
+
+  new recipes:
+    - title: Diario Extra
+      author: Douglas Delgado
+
 - version: 0.9.28
   date: 2013-04-26
 

manual/gui.rst

@@ -582,6 +582,10 @@ Calibre has several keyboard shortcuts to save you time and mouse movement. Thes
       - Open the advanced search dialog
     * - :kbd:`Esc`
       - Clear the current search
+    * - :kbd:`Shift+Esc`
+      - Focus the book list
+    * - :kbd:`Ctrl+Esc`
+      - Clear the virtual library
     * - :kbd:`N or F3`
       - Find the next book that matches the current search (only works if the highlight checkbox next to the search bar is checked)
     * - :kbd:`Shift+N or Shift+F3`

recipes/diario_extra.recipe

@@ -0,0 +1,47 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class goonews(BasicNewsRecipe):
+    __author__     = 'Douglas Delgado'
+    title          = u'Diario Extra'
+    publisher      = 'Sociedad Periodistica Extra Limitada'
+    description    = 'Diario de circulacion nacional de Costa Rica.'
+    category       = 'Spanish, Entertainment'
+    masthead_url   = 'http://www.diarioextra.com/img/apariencia/logo.png'
+
+    oldest_article = 7
+    delay          = 1
+    max_articles_per_feed = 100
+    auto_cleanup = True
+    encoding       = 'utf-8'
+    language       = 'es_CR'
+    use_embedded_content  = False
+    remove_empty_feeds    = True
+    remove_javascript = True
+    no_stylesheets = True
+
+    feeds          = [(u'Nacionales',
+                       u'http://www.diarioextra.com/includes/rss_text.php?id=1'),
+                      (u'Internacionales',
+                       u'http://www.diarioextra.com/includes/rss_text.php?id=2'),
+                      (u'Sucesos',
+                       u'http://www.diarioextra.com/includes/rss_text.php?id=3'),
+                      (u'Deportes',
+                       u'http://www.diarioextra.com/includes/rss_text.php?id=6'),
+                      (u'Espectaculos',
+                       u'http://www.diarioextra.com/includes/rss_text.php?id=7'),
+                      (u'Opinion',
+                       u'http://www.diarioextra.com/includes/rss_text.php?id=4')]
+
+    def get_cover_url(self):
+        index = 'http://kiosko.net/cr/np/cr_extra.html'
+        soup = self.index_to_soup(index)
+        for image in soup.findAll('img', src=True):
+            if image['src'].endswith('cr_extra.750.jpg'):
+                return image['src']
+        return None
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:30px;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal; font-style:italic; font-size:18px;}
+                '''

setup.cfg

@@ -1,4 +1,4 @@
 [flake8]
 max-line-length = 160
 builtins = _,dynamic_property,__,P,I,lopen,icu_lower,icu_upper,icu_title,ngettext
-ignore = E12,E22,E231,E301,E302,E304,E401,W391
+ignore = E12,E203,E22,E231,E241,E301,E302,E304,E401,W391

setup.py

@@ -63,7 +63,7 @@ def main(args=sys.argv):
 
     parser = option_parser()
     command.add_all_options(parser)
-    parser.set_usage('Usage: python setup.py %s [options]\n\n'%args[1]+\
+    parser.set_usage('Usage: python setup.py %s [options]\n\n'%args[1]+
             command.description)
 
     opts, args = parser.parse_args(args)

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 9, 28)
+numeric_version = (0, 9, 29)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -1763,7 +1763,7 @@ if __name__ == '__main__':
             'calibre.utils.terminal', 'calibre.utils.magick', 'PIL', 'Image',
             'sqlite3', 'mechanize', 'httplib', 'xml', 'inspect', 'urllib',
             'calibre.utils.date', 'calibre.utils.config', 'platform',
-            'calibre.utils.zipfile',
+            'calibre.utils.zipfile', 'calibre.utils.formatter',
         ):
             if x in sys.modules:
                 ret = 1

src/calibre/ebooks/conversion/plugins/txt_input.py

@@ -63,7 +63,6 @@ class TXTInput(InputFormatPlugin):
                 normalize_line_endings, convert_textile, remove_indents,
                 block_to_single_line, separate_hard_scene_breaks)
 
-
         self.log = log
         txt = ''
         log.debug('Reading text from file...')
@@ -92,6 +91,12 @@ class TXTInput(InputFormatPlugin):
             log.debug('Using user specified input encoding of %s' % ienc)
         else:
             det_encoding = detect(txt)
+            if det_encoding and det_encoding.lower().replace('_', '-').strip() in (
+                    'gb2312', 'chinese', 'csiso58gb231280', 'euc-cn', 'euccn',
+                    'eucgb2312-cn', 'gb2312-1980', 'gb2312-80', 'iso-ir-58'):
+                # Microsoft Word exports to HTML with encoding incorrectly set to
+                # gb2312 instead of gbk. gbk is a superset of gb2312, anyway.
+                det_encoding = 'gbk'
             ienc = det_encoding['encoding']
             log.debug('Detected input encoding as %s with a confidence of %s%%' % (ienc, det_encoding['confidence'] * 100))
         if not ienc:

src/calibre/ebooks/mobi/reader/headers.py

@@ -13,12 +13,12 @@ from calibre.utils.date import parse_date
 from calibre.ebooks.mobi import MobiError
 from calibre.ebooks.metadata import MetaInformation, check_isbn
 from calibre.ebooks.mobi.langcodes import main_language, sub_language, mobi2iana
-from calibre.utils.cleantext import clean_ascii_chars
+from calibre.utils.cleantext import clean_ascii_chars, clean_xml_chars
 from calibre.utils.localization import canonicalize_lang
 
 NULL_INDEX = 0xffffffff
 
-class EXTHHeader(object): # {{{
+class EXTHHeader(object):  # {{{
 
     def __init__(self, raw, codec, title):
         self.doctype = raw[:4]
@@ -62,7 +62,7 @@ class EXTHHeader(object): # {{{
             elif idx == 502:
                 # last update time
                 pass
-            elif idx == 503: # Long title
+            elif idx == 503:  # Long title
                 # Amazon seems to regard this as the definitive book title
                 # rather than the title from the PDB header. In fact when
                 # sending MOBI files through Amazon's email service if the
@@ -72,7 +72,7 @@ class EXTHHeader(object): # {{{
                     title = self.decode(content)
                 except:
                     pass
-            elif idx == 524: # Lang code
+            elif idx == 524:  # Lang code
                 try:
                     lang = content.decode(codec)
                     lang = canonicalize_lang(lang)
@@ -83,22 +83,22 @@ class EXTHHeader(object): # {{{
             #else:
             #    print 'unknown record', idx, repr(content)
         if title:
-            self.mi.title = replace_entities(clean_ascii_chars(title))
+            self.mi.title = replace_entities(clean_xml_chars(clean_ascii_chars(title)))
 
     def process_metadata(self, idx, content, codec):
         if idx == 100:
             if self.mi.is_null('authors'):
                 self.mi.authors = []
-            au = self.decode(content).strip()
+            au = clean_xml_chars(self.decode(content).strip())
             self.mi.authors.append(au)
             if self.mi.is_null('author_sort') and re.match(r'\S+?\s*,\s+\S+', au.strip()):
                 self.mi.author_sort = au.strip()
         elif idx == 101:
-            self.mi.publisher = self.decode(content).strip()
+            self.mi.publisher = clean_xml_chars(self.decode(content).strip())
             if self.mi.publisher in {'Unknown', _('Unknown')}:
                 self.mi.publisher = None
         elif idx == 103:
-            self.mi.comments  = self.decode(content).strip()
+            self.mi.comments  = clean_xml_chars(self.decode(content).strip())
         elif idx == 104:
             raw = check_isbn(self.decode(content).strip().replace('-', ''))
             if raw:
@@ -106,7 +106,7 @@ class EXTHHeader(object): # {{{
         elif idx == 105:
             if not self.mi.tags:
                 self.mi.tags = []
-            self.mi.tags.extend([x.strip() for x in self.decode(content).split(';')])
+            self.mi.tags.extend([x.strip() for x in clean_xml_chars(self.decode(content)).split(';')])
             self.mi.tags = list(set(self.mi.tags))
         elif idx == 106:
             try:
@@ -114,8 +114,8 @@ class EXTHHeader(object): # {{{
             except:
                 pass
         elif idx == 108:
-            self.mi.book_producer = self.decode(content).strip()
+            self.mi.book_producer = clean_xml_chars(self.decode(content).strip())
-        elif idx == 112: # dc:source set in some EBSP amazon samples
+        elif idx == 112:  # dc:source set in some EBSP amazon samples
             try:
                 content = content.decode(codec).strip()
                 isig = 'urn:isbn:'
@@ -131,7 +131,7 @@ class EXTHHeader(object): # {{{
                         self.mi.application_id = self.mi.uuid = cid
             except:
                 pass
-        elif idx == 113: # ASIN or other id
+        elif idx == 113:  # ASIN or other id
             try:
                 self.uuid = content.decode('ascii')
                 self.mi.set_identifier('mobi-asin', self.uuid)
@@ -242,7 +242,7 @@ class BookHeader(object):
                 # if cnt is 1 or less, fdst section number can be garbage
                 if self.fdstcnt <= 1:
                     self.fdstidx = NULL_INDEX
-            else: # Null values
+            else:  # Null values
                 self.skelidx = self.dividx = self.othidx = self.fdstidx = \
                         NULL_INDEX
 

src/calibre/gui2/library/views.py

@@ -807,6 +807,14 @@ class BooksView(QTableView): # {{{
                 sm = self.selectionModel()
                 sm.select(index, sm.ClearAndSelect|sm.Rows)
 
+    def keyPressEvent(self, ev):
+        val = self.horizontalScrollBar().value()
+        ret = super(BooksView, self).keyPressEvent(ev)
+        if ev.isAccepted() and ev.key() in (Qt.Key_Home, Qt.Key_End
+                                            ) and ev.modifiers() & Qt.ControlModifier:
+            self.horizontalScrollBar().setValue(val)
+        return ret
+
     def ids_to_rows(self, ids):
         row_map = OrderedDict()
         ids = frozenset(ids)

src/calibre/gui2/ui.py

@@ -265,6 +265,20 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin,  # {{{
                 action=self.esc_action)
         self.esc_action.triggered.connect(self.esc)
 
+        self.shift_esc_action = QAction(self)
+        self.addAction(self.shift_esc_action)
+        self.keyboard.register_shortcut('focus book list',
+                _('Focus the book list'), default_keys=('Shift+Esc',),
+                action=self.shift_esc_action)
+        self.shift_esc_action.triggered.connect(self.shift_esc)
+
+        self.ctrl_esc_action = QAction(self)
+        self.addAction(self.ctrl_esc_action)
+        self.keyboard.register_shortcut('clear virtual library',
+                _('Clear the virtual library'), default_keys=('Ctrl+Esc',),
+                action=self.ctrl_esc_action)
+        self.ctrl_esc_action.triggered.connect(self.ctrl_esc)
+
         ####################### Start spare job server ########################
         QTimer.singleShot(1000, self.add_spare_server)
 
@@ -377,6 +391,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin,  # {{{
     def esc(self, *args):
         self.clear_button.click()
 
+    def shift_esc(self):
+        self.current_view().setFocus(Qt.OtherFocusReason)
+
+    def ctrl_esc(self):
+        self.apply_virtual_library()
+        self.current_view().setFocus(Qt.OtherFocusReason)
+
     def start_smartdevice(self):
         message = None
         if self.device_manager.get_option('smartdevice', 'autostart'):

src/calibre/translations/uk.po

@@ -5574,7 +5574,7 @@ msgstr ""
 "<p>Цим інструментом слід користуватися на останньому кроці створення вашої \n"
 "електронної книги.</p>\n"
 "{0}\n"
-"<p>Зауважте, що покращення працює лише для файлів у форматах %s.</p>\n"
+"<p>Зауважте, що покращення працює лише для файлів у форматах %s.</p>"
 
 #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/polish/main.py:48
 #: /home/kovid/work/calibre/src/calibre/gui2/actions/polish.py:446

src/calibre/utils/cleantext.py

@@ -1,9 +1,9 @@
-from __future__ import with_statement
 __license__ = 'GPL 3'
 __copyright__ = '2010, sengian <sengian1@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import re, htmlentitydefs
+from future_builtins import map
 
 _ascii_pat = None
 
@@ -28,6 +28,12 @@ def clean_ascii_chars(txt, charlist=None):
         pat = re.compile(u'|'.join(map(unichr, charlist)))
     return pat.sub('', txt)
 
+def clean_xml_chars(unicode_string):
+    def allowed(x):
+        x = ord(x)
+        return (0x0001 < x < 0xd7ff) or (0xe000 < x < 0xfffd) or (0x10000 < x < 0x10ffff)
+    return u''.join(filter(allowed, unicode_string))
+
 ##
 # Fredrik Lundh: http://effbot.org/zone/re-sub.htm#unescape-html
 # Removes HTML or XML character references and entities from a text string.