Sync to trunk.

recipes/ming_pao.recipe

@@ -16,6 +16,7 @@ __UseLife__ = True
 
 '''
 Change Log:
+2011/09/07: disable "column" section as it is no longer offered free.
 2011/06/26: add fetching Vancouver and Toronto versions of the paper, also provide captions for images using life.mingpao fetch source
             provide options to remove all images in the file
 2011/05/12: switch the main parse source to life.mingpao.com, which has more photos on the article pages
@@ -230,8 +231,9 @@ class MPRecipe(BasicNewsRecipe):
                                            (u'\u570b\u969b World', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalta', 'nal'),
                                            (u'\u7d93\u6fdf Finance', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea', 'nal'),
                                            (u'\u9ad4\u80b2 Sport', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalsp', 'nal'),
-                                           (u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal'),
+                                           (u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal')
-                                           (u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')]:
+                                           #(u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')
+                                          ]:
                     articles = self.parse_section2(url, keystr)
                     if articles:
                         feeds.append((title, articles))
@@ -591,4 +593,3 @@ class MPRecipe(BasicNewsRecipe):
 
         with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file):
             opf.render(opf_file, ncx_file)
-

resources/default_tweaks.py

@@ -70,9 +70,18 @@ author_sort_copy_method = 'comma'
 author_name_suffixes = ('Jr', 'Sr', 'Inc', 'Ph.D', 'Phd',
                         'MD', 'M.D', 'I', 'II', 'III', 'IV',
                         'Junior', 'Senior')
+author_name_prefixes = ('Mr', 'Mrs', 'Ms', 'Dr', 'Prof')
 author_name_copywords = ('Corporation', 'Company', 'Co.', 'Agency', 'Council',
         'Committee', 'Inc.', 'Institute', 'Society', 'Club', 'Team')
 
+#: Splitting multiple author names
+# By default, calibre splits a string containing multiple author names on
+# ampersands and the words "and" and "with". You can customize the splitting
+# by changing the regular expression below. Strings are split on whatever the
+# specified regular expression matches.
+# Default: r'(?i),?\s+(and|with)\s+'
+authors_split_regex = r'(?i),?\s+(and|with)\s+'
+
 #: Use author sort in Tag Browser
 # Set which author field to display in the tags pane (the list of authors,
 # series, publishers etc on the left hand side). The choices are author and

src/calibre/customize/builtins.py

@@ -571,7 +571,7 @@ from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS,
 from calibre.devices.sne.driver import SNE
 from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL,
         GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR,
-        TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY)
+        TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY, EX124G)
 from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 from calibre.devices.kobo.driver import KOBO
 from calibre.devices.bambook.driver import BAMBOOK
@@ -707,7 +707,7 @@ plugins += [
     EEEREADER,
     NEXTBOOK,
     ADAM,
-    MOOVYBOOK, COBY,
+    MOOVYBOOK, COBY, EX124G,
     ITUNES,
     BOEYE_BEX,
     BOEYE_BDX,

src/calibre/db/fields.py

@@ -164,7 +164,7 @@ class ManyToOneField(Field):
     def for_book(self, book_id, default_value=None):
         ids = self.table.book_col_map.get(book_id, None)
         if ids is not None:
-            ans = self.id_map[ids]
+            ans = self.table.id_map[ids]
         else:
             ans = default_value
         return ans
@@ -182,7 +182,7 @@ class ManyToOneField(Field):
         return self.table.id_map.iterkeys()
 
     def sort_keys_for_books(self, get_metadata, all_book_ids):
-        keys = {id_ : self._sort_key(self.id_map.get(id_, '')) for id_ in
+        keys = {id_ : self._sort_key(self.table.id_map.get(id_, '')) for id_ in
                 all_book_ids}
         return {id_ : keys.get(
             self.book_col_map.get(id_, None), '') for id_ in all_book_ids}
@@ -196,7 +196,7 @@ class ManyToManyField(Field):
     def for_book(self, book_id, default_value=None):
         ids = self.table.book_col_map.get(book_id, ())
         if ids:
-            ans = tuple(self.id_map[i] for i in ids)
+            ans = tuple(self.table.id_map[i] for i in ids)
         else:
             ans = default_value
         return ans
@@ -211,7 +211,7 @@ class ManyToManyField(Field):
         return self.table.id_map.iterkeys()
 
     def sort_keys_for_books(self, get_metadata, all_book_ids):
-        keys = {id_ : self._sort_key(self.id_map.get(id_, '')) for id_ in
+        keys = {id_ : self._sort_key(self.table.id_map.get(id_, '')) for id_ in
                 all_book_ids}
 
         def sort_key_for_book(book_id):
@@ -222,6 +222,13 @@ class ManyToManyField(Field):
 
         return {id_ : sort_key_for_book(id_) for id_ in all_book_ids}
 
+class IdentifiersField(ManyToManyField):
+
+    def for_book(self, book_id, default_value=None):
+        ids = self.table.book_col_map.get(book_id, ())
+        if not ids:
+            ids = default_value
+        return ids
 
 class AuthorsField(ManyToManyField):
 
@@ -249,6 +256,8 @@ def create_field(name, table):
         cls = OnDeviceField
     elif name == 'formats':
         cls = FormatsField
+    elif name == 'identifiers':
+        cls = IdentifiersField
     elif table.metadata['datatype'] == 'composite':
         cls = CompositeField
     return cls(name, table)

src/calibre/db/tables.py

@@ -66,8 +66,6 @@ class VirtualTable(Table):
         self.table_type = table_type
         Table.__init__(self, name, metadata)
 
-
-
 class OneToOneTable(Table):
 
     '''

src/calibre/db/tests/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/db/tests/reading.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+import os, shutil, unittest, tempfile, datetime
+
+from calibre.utils.date import local_tz
+
+def create_db(library_path):
+    from calibre.library.database2 import LibraryDatabase2
+    if LibraryDatabase2.exists_at(library_path):
+        raise ValueError('A library already exists at %r'%library_path)
+    src = os.path.join(os.path.dirname(__file__), 'metadata.db')
+    db = os.path.join(library_path, 'metadata.db')
+    shutil.copyfile(src, db)
+    return db
+
+def init_cache(library_path):
+    from calibre.db.backend import DB
+    from calibre.db.cache import Cache
+    backend = DB(library_path)
+    cache = Cache(backend)
+    cache.init()
+    return cache
+
+class ReadingTest(unittest.TestCase):
+
+    def setUp(self):
+        self.library_path = tempfile.mkdtemp()
+        create_db(self.library_path)
+
+    def tearDown(self):
+        shutil.rmtree(self.library_path)
+
+    def test_read(self): # {{{
+        cache = init_cache(self.library_path)
+        tests = {
+                2 : {
+                    'title': 'Title One',
+                    'sort': 'One',
+                    'authors': ('Author One',),
+                    'author_sort': 'One, Author',
+                    'series' : 'Series One',
+                    'series_index': 1.0,
+                    'tags':('Tag Two', 'Tag One'),
+                    'rating': 4.0,
+                    'identifiers': {'test':'one'},
+                    'timestamp': datetime.datetime(2011, 9, 5, 15, 6,
+                        tzinfo=local_tz),
+                    'pubdate': datetime.datetime(2011, 9, 5, 15, 6,
+                        tzinfo=local_tz),
+                    'publisher': 'Publisher One',
+                    'languages': ('eng',),
+                    'comments': '<p>Comments One</p>',
+                    '#enum':'One',
+                    '#authors':('Custom One', 'Custom Two'),
+                    '#date':datetime.datetime(2011, 9, 5, 0, 0,
+                        tzinfo=local_tz),
+                    '#rating':2.0,
+                    '#series':'My Series One',
+                    '#series_index': 1.0,
+                    '#tags':('My Tag One', 'My Tag Two'),
+                    '#yesno':True,
+                    '#comments': '<div>My Comments One<p></p></div>',
+                },
+                1  : {
+                    'title': 'Title Two',
+                    'sort': 'Title Two',
+                    'authors': ('Author Two', 'Author One'),
+                    'author_sort': 'Two, Author & One, Author',
+                    'series' : 'Series Two',
+                    'series_index': 2.0,
+                    'rating': 6.0,
+                    'tags': ('Tag Two',),
+                    'identifiers': {'test':'two'},
+                    'timestamp': datetime.datetime(2011, 9, 6, 0, 0,
+                        tzinfo=local_tz),
+                    'pubdate': datetime.datetime(2011, 8, 5, 0, 0,
+                        tzinfo=local_tz),
+                    'publisher': 'Publisher Two',
+                    'languages': ('deu',),
+                    'comments': '<p>Comments Two</p>',
+                    '#enum':'Two',
+                    '#authors':('My Author Two',),
+                    '#date':datetime.datetime(2011, 9, 1, 0, 0,
+                        tzinfo=local_tz),
+                    '#rating':4.0,
+                    '#series':'My Series Two',
+                    '#series_index': 3.0,
+                    '#tags':('My Tag Two',),
+                    '#yesno':False,
+                    '#comments': '<div>My Comments Two<p></p></div>',
+
+                },
+        }
+        for book_id, test in tests.iteritems():
+            for field, expected_val in test.iteritems():
+                self.assertEqual(expected_val,
+                        cache.field_for(field, book_id))
+        # }}}
+
+def tests():
+    return unittest.TestLoader().loadTestsFromTestCase(ReadingTest)
+
+def run():
+    unittest.TextTestRunner(verbosity=2).run(tests())
+
+if __name__ == '__main__':
+    run()
+

src/calibre/debug.py

@@ -180,6 +180,7 @@ def main(args=sys.argv):
         sys.path.insert(0, base)
         g = globals()
         g['__name__'] = '__main__'
+        g['__file__'] = ef
         execfile(ef, g)
         return
 

src/calibre/devices/android/driver.py

@@ -127,7 +127,7 @@ class ANDROID(USBMS):
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
             'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA',
-            'GENERIC-', 'ZTE', 'MID', 'QUALCOMM']
+            'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -137,11 +137,11 @@ class ANDROID(USBMS):
             '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2',
             'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK',
             'MB525', 'ANDROID2.3', 'SGH-I997', 'GT-I5800_CARD', 'MB612',
-            'GT-S5830_CARD', 'GT-S5570_CARD', 'MB870', 'MID7015A']
+            'GT-S5830_CARD', 'GT-S5570_CARD', 'MB870', 'MID7015A', 'ALPANDIGITAL']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
-            '__UMS_COMPOSITE', 'SGH-I997_CARD', 'MB870']
+            '__UMS_COMPOSITE', 'SGH-I997_CARD', 'MB870', 'ALPANDIGITAL']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/devices/misc.py

@@ -377,3 +377,31 @@ class COBY(USBMS):
             return 'eBooks'
         return self.EBOOK_DIR_CARD_A
 
+class EX124G(USBMS):
+
+    name = 'Motorola Ex124G device interface'
+    gui_name = 'Ex124G'
+    description = _('Communicate with the Ex124G')
+
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['mobi', 'prc', 'azw']
+
+    VENDOR_ID   = [0x0e8d]
+    PRODUCT_ID  = [0x0002]
+    BCD         = [0x0100]
+    VENDOR_NAME = 'MOTOROLA'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '_PHONE'
+
+    EBOOK_DIR_MAIN = 'eBooks'
+
+    SUPPORTS_SUB_DIRS = False
+
+    def get_carda_ebook_dir(self, for_upload=False):
+        if for_upload:
+            return 'eBooks'
+        return self.EBOOK_DIR_CARD_A
+
+

src/calibre/devices/prs505/driver.py

@@ -291,6 +291,8 @@ class PRS505(USBMS):
             thumbnail_dir = os.path.join(thumbnail_dir, relpath)
             if not os.path.exists(thumbnail_dir):
                 os.makedirs(thumbnail_dir)
-            with open(os.path.join(thumbnail_dir, 'main_thumbnail.jpg'), 'wb') as f:
+            cpath = os.path.join(thumbnail_dir, 'main_thumbnail.jpg')
+            with open(cpath, 'wb') as f:
                 f.write(metadata.thumbnail[-1])
+            debug_print('Cover uploaded to: %r'%cpath)
 

src/calibre/ebooks/conversion/preprocess.py

@@ -636,12 +636,3 @@ class HTMLPreProcessor(object):
         html = re.sub(r'\s--\s', u'\u2014', html)
         return substitute_entites(html)
 
-    def unsmarten_punctuation(self, html):
-        from calibre.utils.unsmarten import unsmarten_html
-        from calibre.ebooks.chardet import substitute_entites
-        from calibre.ebooks.conversion.utils import HeuristicProcessor
-        preprocessor = HeuristicProcessor(self.extra_opts, self.log)
-        html = preprocessor.fix_nbsp_indents(html)
-        html = unsmarten_html(html)
-        return substitute_entites(html)
-

src/calibre/ebooks/metadata/__init__.py

@@ -10,11 +10,17 @@ import os, sys, re
 from urllib import unquote, quote
 from urlparse import urlparse
 
-from calibre import relpath, guess_type, remove_bracketed_text
+from calibre import relpath, guess_type, remove_bracketed_text, prints
 
 from calibre.utils.config import tweaks
 
-_author_pat = re.compile(',?\s+(and|with)\s+', re.IGNORECASE)
+try:
+    _author_pat = re.compile(tweaks['authors_split_regex'])
+except:
+    prints ('Author split regexp:', tweaks['authors_split_regex'],
+            'is invalid, using default')
+    _author_pat = re.compile(r'(?i),?\s+(and|with)\s+')
+
 def string_to_authors(raw):
     raw = raw.replace('&&', u'\uffff')
     raw = _author_pat.sub('&', raw)
@@ -45,6 +51,17 @@ def author_to_author_sort(author, method=None):
     if method == u'copy':
         return author
 
+    prefixes = set([x.lower() for x in tweaks['author_name_prefixes']])
+    prefixes |= set([x+u'.' for x in prefixes])
+    while True:
+        if not tokens:
+            return author
+        tok = tokens[0].lower()
+        if tok in prefixes:
+            tokens = tokens[1:]
+        else:
+            break
+
     suffixes = set([x.lower() for x in tweaks['author_name_suffixes']])
     suffixes |= set([x+u'.' for x in suffixes])
 

src/calibre/ebooks/metadata/epub.py

@@ -16,6 +16,7 @@ from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ptempfile import TemporaryDirectory, PersistentTemporaryFile
 from calibre import CurrentDir, walk
 from calibre.constants import isosx
+from calibre.utils.localization import lang_as_iso639_1
 
 class EPubException(Exception):
     pass
@@ -231,6 +232,15 @@ def set_metadata(stream, mi, apply_null=False, update_timestamp=False):
 
     for x in ('guide', 'toc', 'manifest', 'spine'):
         setattr(mi, x, None)
+    if mi.languages:
+        langs = []
+        for lc in mi.languages:
+            lc2 = lang_as_iso639_1(lc)
+            if lc2: lc = lc2
+            langs.append(lc)
+        mi.languages = langs
+
+
     reader.opf.smart_update(mi)
     if apply_null:
         if not getattr(mi, 'series', None):

src/calibre/ebooks/metadata/sources/amazon.py

@@ -74,6 +74,20 @@ class Worker(Thread): # Get details {{{
             9: ['sept'],
             12: ['déc'],
             },
+                'jp': {
+            1: [u'1月'],
+            2: [u'2月'],
+            3: [u'3月'],
+            4: [u'4月'],
+            5: [u'5月'],
+            6: [u'6月'],
+            7: [u'7月'],
+            8: [u'8月'],
+            9: [u'9月'],
+            10: [u'10月'],
+            11: [u'11月'],
+            12: [u'12月'],
+            },
 
         }
 
@@ -86,13 +100,15 @@ class Worker(Thread): # Get details {{{
                  text()="Produktinformation" or \
                  text()="Dettagli prodotto" or \
                  text()="Product details" or \
-                 text()="Détails sur le produit"]/../div[@class="content"]
+                 text()="Détails sur le produit" or \
+                 text()="登録情報"]/../div[@class="content"]
             '''
         self.publisher_xpath = '''
             descendant::*[starts-with(text(), "Publisher:") or \
                     starts-with(text(), "Verlag:") or \
                     starts-with(text(), "Editore:") or \
-                    starts-with(text(), "Editeur")]
+                    starts-with(text(), "Editeur") or \
+                    starts-with(text(), "出版社:")]
             '''
         self.language_xpath =    '''
             descendant::*[
@@ -101,10 +117,11 @@ class Worker(Thread): # Get details {{{
                 or text() = "Sprache:" \
                 or text() = "Lingua:" \
                 or starts-with(text(), "Langue") \
+                or starts-with(text(), "言語") \
                 ]
             '''
         self.ratings_pat = re.compile(
-            r'([0-9.]+) (out of|von|su|étoiles sur) (\d+)( (stars|Sternen|stelle)){0,1}')
+            r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち) ([\d\.]+)( (stars|Sternen|stelle)){0,1}')
 
         lm = {
                 'eng': ('English', 'Englisch'),
@@ -112,6 +129,7 @@ class Worker(Thread): # Get details {{{
                 'ita': ('Italian', 'Italiano'),
                 'deu': ('German', 'Deutsch'),
                 'spa': ('Spanish', 'Espa\xf1ol', 'Espaniol'),
+                'jpn': ('Japanese', u'日本語'),
                 }
         self.lang_map = {}
         for code, names in lm.iteritems():
@@ -403,6 +421,7 @@ class Amazon(Source):
             'de' : _('Germany'),
             'uk' : _('UK'),
             'it' : _('Italy'),
+            'jp' : _('Japan'),
     }
 
     options = (
@@ -411,6 +430,22 @@ class Amazon(Source):
                     'country\'s Amazon website.'), choices=AMAZON_DOMAINS),
             )
 
+    def __init__(self, *args, **kwargs):
+        Source.__init__(self, *args, **kwargs)
+        self.set_amazon_id_touched_fields()
+
+    def save_settings(self, *args, **kwargs):
+        Source.save_settings(self, *args, **kwargs)
+        self.set_amazon_id_touched_fields()
+
+    def set_amazon_id_touched_fields(self):
+        ident_name = "identifier:amazon"
+        if self.domain != 'com':
+            ident_name += '_' + self.domain
+        tf = [x for x in self.touched_fields if not
+                x.startswith('identifier:amazon')] + [ident_name]
+        self.touched_fields = frozenset(tf)
+
     def get_domain_and_asin(self, identifiers):
         for key, val in identifiers.iteritems():
             key = key.lower()
@@ -488,13 +523,23 @@ class Amazon(Source):
             # Insufficient metadata to make an identify query
             return None, None
 
-        latin1q = dict([(x.encode('latin1', 'ignore'), y.encode('latin1',
+        # magic parameter to enable Japanese Shift_JIS encoding.
+        if domain == 'jp':
+            q['__mk_ja_JP'] = u'カタカナ'
+
+        if domain == 'jp':
+            encode_to = 'Shift_JIS'
+        else:
+            encode_to = 'latin1'
+        encoded_q = dict([(x.encode(encode_to, 'ignore'), y.encode(encode_to,
             'ignore')) for x, y in
             q.iteritems()])
         udomain = domain
         if domain == 'uk':
             udomain = 'co.uk'
-        url = 'http://www.amazon.%s/s/?'%udomain + urlencode(latin1q)
+        elif domain == 'jp':
+            udomain = 'co.jp'
+        url = 'http://www.amazon.%s/s/?'%udomain + urlencode(encoded_q)
         return url, domain
 
     # }}}
@@ -663,7 +708,7 @@ if __name__ == '__main__': # tests {{{
     # To run these test use: calibre-debug -e
     # src/calibre/ebooks/metadata/sources/amazon.py
     from calibre.ebooks.metadata.sources.test import (test_identify_plugin,
-            title_test, authors_test)
+            isbn_test, title_test, authors_test)
     com_tests = [ # {{{
 
             (  # Description has links
@@ -744,6 +789,21 @@ if __name__ == '__main__': # tests {{{
             ),
     ] # }}}
 
+    jp_tests = [ # {{{
+            ( # isbn -> title, authors
+                {'identifiers':{'isbn': '9784101302720' }},
+                [title_test(u'精霊の守り人',
+                    exact=True), authors_test([u'上橋 菜穂子'])
+                 ]
+            ),
+            ( # title, authors -> isbn (will use Shift_JIS encoding in query.)
+                {'title': u'考えない練習',
+                 'authors': [u'小池 龍之介']},
+                [isbn_test('9784093881067'), ]
+            ),
+    ] # }}}
+
     test_identify_plugin(Amazon.name, com_tests)
+    #test_identify_plugin(Amazon.name, jp_tests)
 # }}}
 

src/calibre/ebooks/oeb/output.py

@@ -43,6 +43,7 @@ class OEBOutput(OutputFormatPlugin):
                         except:
                             self.log.exception('Something went wrong while trying to'
                                     ' workaround Pocketbook cover bug, ignoring')
+                        self.migrate_lang_code(root)
                     raw = etree.tostring(root, pretty_print=True,
                             encoding='utf-8', xml_declaration=True)
                     if key == OPF_MIME:
@@ -104,3 +105,12 @@ class OEBOutput(OutputFormatPlugin):
             p.remove(m)
             p.insert(0, m)
     # }}}
+
+    def migrate_lang_code(self, root): # {{{
+        from calibre.utils.localization import lang_as_iso639_1
+        for lang in root.xpath('//*[local-name() = "language"]'):
+            clc = lang_as_iso639_1(lang.text)
+            if clc:
+                lang.text = clc
+    # }}}
+

src/calibre/ebooks/oeb/transforms/unsmarten.py

@@ -10,16 +10,22 @@ from calibre.ebooks.oeb.base import OEB_DOCS, XPath, barename
 from calibre.utils.unsmarten import unsmarten_text
 
 class UnsmartenPunctuation(object):
-    
+
+    def __init__(self):
+        self.html_tags = XPath('descendant::h:*')
+
     def unsmarten(self, root):
-        for x in XPath('//h:*')(root):
+        for x in self.html_tags(root):
             if not barename(x) == 'pre':
-                if hasattr(x, 'text') and x.text:
+                if getattr(x, 'text', None):
                     x.text = unsmarten_text(x.text)
-                if hasattr(x, 'tail') and x.tail:
+                if getattr(x, 'tail', None) and x.tail:
                     x.tail = unsmarten_text(x.tail)
 
     def __call__(self, oeb, context):
+        bx = XPath('//h:body')
         for x in oeb.manifest.items:
             if x.media_type in OEB_DOCS:
-                self.unsmarten(x.data)
+                for body in bx(x.data):
+                    self.unsmarten(body)
+

src/calibre/ebooks/txt/textileml.py

@@ -83,7 +83,7 @@ class TextileMLizer(OEB2HTML):
             for i in self.our_ids:
                 if i not in self.our_links:
                     text = re.sub(r'%?\('+i+'\)\xa0?%?', r'', text)
-                    
+
         # Remove obvious non-needed escaping, add sub/sup-script ones
         text = check_escaping(text, ['\*', '_', '\*'])
         # escape the super/sub-scripts if needed
@@ -189,7 +189,7 @@ class TextileMLizer(OEB2HTML):
         emright = int(round(right / stylizer.profile.fbase))
         if emright >= 1:
             txt += ')' * emright
-            
+
         return txt
 
     def check_id_tag(self, attribs):
@@ -235,7 +235,7 @@ class TextileMLizer(OEB2HTML):
         tags = []
         tag = barename(elem.tag)
         attribs = elem.attrib
-        
+
         # Ignore anything that is set to not be displayed.
         if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \
            or style['visibility'] == 'hidden':
@@ -246,7 +246,7 @@ class TextileMLizer(OEB2HTML):
             ems = int(round(float(style.marginTop) / style.fontSize) - 1)
             if ems >= 1:
                 text.append(u'\n\n\xa0' * ems)
-            
+
         if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'div'):
             if tag == 'div':
                 tag = 'p'
@@ -432,7 +432,7 @@ class TextileMLizer(OEB2HTML):
                 'span', 'table', 'tr', 'td'):
             if not self.in_a_link:
                 text.append(self.check_styles(style))
-        
+
         # Process tags that contain text.
         if hasattr(elem, 'text') and elem.text:
             txt = elem.text

src/calibre/gui2/actions/__init__.py

@@ -15,6 +15,8 @@ from calibre import prints
 from calibre.gui2 import Dispatcher
 from calibre.gui2.keyboard import NameConflict
 
+def menu_action_unique_name(plugin, unique_name):
+    return u'%s : menu action : %s'%(plugin.unique_name, unique_name)
 
 class InterfaceAction(QObject):
 
@@ -178,30 +180,30 @@ class InterfaceAction(QObject):
             description=None, triggered=None, shortcut_name=None):
         '''
         Convenience method to easily add actions to a QMenu.
+        Returns the created QAction, This action has one extra attribute
+        calibre_shortcut_unique_name which if not None refers to the unique
+        name under which this action is registered with the keyboard manager.
 
         :param menu: The QMenu the newly created action will be added to
         :param unique_name: A unique name for this action, this must be
-        globally unique, so make it as descriptive as possible. If in doubt add
+            globally unique, so make it as descriptive as possible. If in doubt add
-        a uuid to it.
+            a uuid to it.
         :param text: The text of the action.
         :param icon: Either a QIcon or a file name. The file name is passed to
-        the I() builtin, so you do not need to pass the full path to the images
+            the I() builtin, so you do not need to pass the full path to the images
-        directory.
+            directory.
         :param shortcut: A string, a list of strings, None or False. If False,
-        no keyboard shortcut is registered for this action. If None, a keyboard
+            no keyboard shortcut is registered for this action. If None, a keyboard
-        shortcut with no default keybinding is registered. String and list of
+            shortcut with no default keybinding is registered. String and list of
-        strings register a shortcut with default keybinding as specified.
+            strings register a shortcut with default keybinding as specified.
         :param description: A description for this action. Used to set
-        tooltips.
+            tooltips.
         :param triggered: A callable which is connected to the triggered signal
-        of the created action.
+            of the created action.
         :param shortcut_name: The test displayed to the user when customizing
-        the keyboard shortcuts for this action. By default it is set to the
+            the keyboard shortcuts for this action. By default it is set to the
-        value of ``text``.
+            value of ``text``.
 
-        :return: The created QAction, This action has one extra attribute
-        calibre_shortcut_unique_name which if not None refers to the unique
-        name under which this action is registered with the keyboard manager.
         '''
         if shortcut_name is None:
             shortcut_name = unicode(text)
@@ -214,7 +216,7 @@ class InterfaceAction(QObject):
         if shortcut is not None and shortcut is not False:
             keys = ((shortcut,) if isinstance(shortcut, basestring) else
                     tuple(shortcut))
-        unique_name = '%s : menu action : %s'%(self.unique_name, unique_name)
+        unique_name = menu_action_unique_name(self, unique_name)
         if description is not None:
             ac.setToolTip(description)
             ac.setStatusTip(description)

src/calibre/gui2/convert/look_and_feel.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import SIGNAL, QVariant
+from PyQt4.Qt import SIGNAL, QVariant, Qt
 
 from calibre.gui2.convert.look_and_feel_ui import Ui_Form
 from calibre.gui2.convert import Widget
@@ -45,6 +45,12 @@ class LookAndFeelWidget(Widget, Ui_Form):
                 self.font_key_wizard)
         self.opt_remove_paragraph_spacing.toggle()
         self.opt_remove_paragraph_spacing.toggle()
+        self.opt_smarten_punctuation.stateChanged.connect(
+                lambda state: state != Qt.Unchecked and
+                self.opt_unsmarten_punctuation.setCheckState(Qt.Unchecked))
+        self.opt_unsmarten_punctuation.stateChanged.connect(
+                lambda state: state != Qt.Unchecked and
+                self.opt_smarten_punctuation.setCheckState(Qt.Unchecked))
 
     def get_value_handler(self, g):
         if g is self.opt_change_justification:

src/calibre/gui2/jobs.py

@@ -21,7 +21,7 @@ from calibre.utils.ipc.job import ParallelJob
 from calibre.gui2 import Dispatcher, error_dialog, question_dialog, NONE, config, gprefs
 from calibre.gui2.device import DeviceJob
 from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog
-from calibre import __appname__
+from calibre import __appname__, as_unicode
 from calibre.gui2.dialogs.job_view_ui import Ui_Dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob
@@ -264,6 +264,26 @@ class JobManager(QAbstractTableModel): # {{{
                     _('This job cannot be stopped'), show=True)
         self._kill_job(job)
 
+    def kill_multiple_jobs(self, rows, view):
+        jobs = [self.jobs[row] for row in rows]
+        devjobs = [j for j in jobs is isinstance(j, DeviceJob)]
+        if devjobs:
+            error_dialog(view, _('Cannot kill job'),
+                         _('Cannot kill jobs that communicate with the device')).exec_()
+            jobs = [j for j in jobs if not isinstance(j, DeviceJob)]
+        jobs = [j for j in jobs if j.duration is None]
+        unkillable = [j for j in jobs if not getattr(j, 'killable', True)]
+        if unkillable:
+            names = u'\n'.join(as_unicode(j.description) for j in unkillable)
+            error_dialog(view, _('Cannot kill job'),
+                    _('Some of the jobs cannot be stopped. Click Show details'
+                        ' to see the list of unstoppable jobs.'), det_msg=names,
+                    show=True)
+            jobs = [j for j in jobs if getattr(j, 'killable', True)]
+        jobs = [j for j in jobs if j.duration is None]
+        for j in jobs:
+            self._kill_job(j)
+
     def kill_all_jobs(self):
         for job in self.jobs:
             if (isinstance(job, DeviceJob) or job.duration is not None or
@@ -484,8 +504,10 @@ class JobsDialog(QDialog, Ui_JobsDialog):
                 ngettext('Do you really want to stop the selected job?',
                     'Do you really want to stop all the selected jobs?',
                     len(rows))):
-            for row in rows:
+            if len(rows) > 1:
-                self.model.kill_job(row, self)
+                self.model.kill_multiple_jobs(rows, self)
+            else:
+                self.model.kill_job(rows[0], self)
 
     def kill_all_jobs(self, *args):
         if question_dialog(self, _('Are you sure?'),

src/calibre/gui2/keyboard.py

@@ -116,6 +116,11 @@ class Manager(QObject): # {{{
         done unregistering.
         '''
         self.shortcuts.pop(unique_name, None)
+        for group in self.groups.itervalues():
+            try:
+                group.remove(unique_name)
+            except ValueError:
+                pass
 
     def finalize(self):
         custom_keys_map = {un:tuple(keys) for un, keys in self.config.get(

src/calibre/gui2/preferences/plugins.py

@@ -235,11 +235,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.highlight_index(idx)
 
     def highlight_index(self, idx):
-        self.plugin_view.scrollTo(idx)
         self.plugin_view.selectionModel().select(idx,
                 self.plugin_view.selectionModel().ClearAndSelect)
         self.plugin_view.setCurrentIndex(idx)
         self.plugin_view.setFocus(Qt.OtherFocusReason)
+        self.plugin_view.scrollTo(idx, self.plugin_view.EnsureVisible)
 
     def find_next(self, *args):
         idx = self.plugin_view.currentIndex()

src/calibre/library/catalog.py

@@ -5113,6 +5113,8 @@ Author '{0}':
 
         if catalog_source_built:
             recommendations = []
+            recommendations.append(('remove_fake_margins', False,
+                OptionRecommendation.HIGH))
             if DEBUG:
                 recommendations.append(('comments', '\n'.join(line for line in build_log),
                     OptionRecommendation.HIGH))

src/calibre/manual/epub.py

@@ -6,7 +6,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, time
+import os, time, glob
+
+from lxml import etree
 
 from sphinx.builders.epub import EpubBuilder
 
@@ -55,4 +57,38 @@ class EPUBHelpBuilder(EpubBuilder):
     def build_epub(self, outdir, *args, **kwargs):
         if self.config.epub_cover:
             self.add_cover(outdir, self.config.epub_cover)
+        self.fix_duplication_bugs(outdir)
         EpubBuilder.build_epub(self, outdir, *args, **kwargs)
+
+    def fix_duplication_bugs(self, outdir):
+        opf = glob.glob(outdir+os.sep+'*.opf')[0]
+        root = etree.fromstring(open(opf, 'rb').read())
+        seen = set()
+        for x in root.xpath(
+                '//*[local-name()="spine"]/*[local-name()="itemref"]'):
+            idref = x.get('idref')
+            if idref in seen:
+                x.getparent().remove(x)
+            else:
+                seen.add(idref)
+
+        with open(opf, 'wb') as f:
+            f.write(etree.tostring(root, encoding='utf-8', xml_declaration=True))
+
+
+        ncx = glob.glob(outdir+os.sep+'*.ncx')[0]
+        root = etree.fromstring(open(ncx, 'rb').read())
+        seen = set()
+        for x in root.xpath(
+                '//*[local-name()="navMap"]/*[local-name()="navPoint"]'):
+            text = x.xpath('descendant::*[local-name()="text"]')[0]
+            text = text.text
+            if text in seen:
+                x.getparent().remove(x)
+            else:
+                seen.add(text)
+
+        with open(ncx, 'wb') as f:
+            f.write(etree.tostring(root, encoding='utf-8', xml_declaration=True))
+
+

src/calibre/utils/mreplace.py

@@ -7,26 +7,32 @@ import re
 from UserDict import UserDict
 
 class MReplace(UserDict):
-    def __init__(self, dict = None):
+
-        UserDict.__init__(self, dict)
+    def __init__(self, data=None, case_sensitive=True):
+        UserDict.__init__(self, data)
         self.re = None
         self.regex = None
+        self.case_sensitive = case_sensitive
         self.compile_regex()
 
-    def compile_regex(self): 
+    def compile_regex(self):
         if len(self.data) > 0:
             keys = sorted(self.data.keys(), key=len)
             keys.reverse()
             tmp = "(%s)" % "|".join(map(re.escape, keys))
             if self.re != tmp:
                 self.re = tmp
-                self.regex = re.compile(self.re)
+                if self.case_sensitive:
+                    self.regex = re.compile(self.re)
+                else:
+                    self.regex = re.compile(self.re, re.I)
 
-    def __call__(self, mo): 
+    def __call__(self, mo):
         return self[mo.string[mo.start():mo.end()]]
 
-    def mreplace(self, text): 
+    def mreplace(self, text):
         #Replace without regex compile
         if len(self.data) < 1 or self.re is None:
             return text
-        return self.regex.sub(self, text)
+        return self.regex.sub(self, text)
+

src/calibre/utils/unsmarten.py

@@ -6,15 +6,38 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import re
+from calibre.utils.mreplace import MReplace
 
-def unsmarten_text(txt):
+_mreplace = MReplace({
-    txt = re.sub(u'&#8211;|&ndash;|–', r'--', txt) # en-dash
+        '&#8211;': '--',
-    txt = re.sub(u'&#8212;|&mdash;|—', r'---', txt) # em-dash
+        '&ndash;': '--',
-    txt = re.sub(u'&#8230;|&hellip;|…', r'...', txt) # ellipsis
+        '–': '--',
+        '&#8212;': '---',
+        '&mdash;': '---',
+        '—': '---',
+        '&#8230;': '...',
+        '&hellip;': '...',
+        '…': '...',
+        '&#8220;': '"',
+        '&#8221;': '"',
+        '&#8243;': '"',
+        '&ldquo;': '"',
+        '&rdquo;': '"',
+        '&Prime;': '"',
+        '“':'"',
+        '”':'"',
+        '″':'"',
+        '&#8216;':"'",
+        '&#8217;':"'",
+        '&#8242;':"'",
+        '&lsquo;':"'",
+        '&rsquo;':"'",
+        '&prime;':"'",
+        '‘':"'",
+        '’':"'",
+        '′':"'",
+})
 
-    txt = re.sub(u'&#8220;|&#8221;|&#8243;|&ldquo;|&rdquo;|&Prime;|“|”|″', r'"', txt)  # double quote
+unsmarten_text = _mreplace.mreplace
-    txt = re.sub(u'&#8216;|&#8217;|&#8242;|&lsquo;|&rsquo;|&prime;|‘|’|′', r"'", txt)  # single quote
 
-    return txt