diff --git a/recipes/ming_pao.recipe b/recipes/ming_pao.recipe index 947d85692f..7060a7cd3e 100644 --- a/recipes/ming_pao.recipe +++ b/recipes/ming_pao.recipe @@ -16,6 +16,7 @@ __UseLife__ = True ''' Change Log: +2011/09/07: disable "column" section as it is no longer offered free. 2011/06/26: add fetching Vancouver and Toronto versions of the paper, also provide captions for images using life.mingpao fetch source provide options to remove all images in the file 2011/05/12: switch the main parse source to life.mingpao.com, which has more photos on the article pages @@ -230,8 +231,9 @@ class MPRecipe(BasicNewsRecipe): (u'\u570b\u969b World', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalta', 'nal'), (u'\u7d93\u6fdf Finance', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea', 'nal'), (u'\u9ad4\u80b2 Sport', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalsp', 'nal'), - (u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal'), - (u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')]: + (u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal') + #(u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl') + ]: articles = self.parse_section2(url, keystr) if articles: feeds.append((title, articles)) @@ -591,4 +593,3 @@ class MPRecipe(BasicNewsRecipe): with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): opf.render(opf_file, ncx_file) - diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index b385511d56..ead9995eb3 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -70,9 +70,18 @@ author_sort_copy_method = 'comma' author_name_suffixes = ('Jr', 'Sr', 'Inc', 'Ph.D', 'Phd', 'MD', 'M.D', 'I', 'II', 'III', 'IV', 'Junior', 'Senior') +author_name_prefixes = ('Mr', 'Mrs', 'Ms', 'Dr', 'Prof') author_name_copywords = ('Corporation', 'Company', 'Co.', 'Agency', 'Council', 'Committee', 'Inc.', 'Institute', 'Society', 'Club', 'Team') +#: Splitting multiple author names +# By default, calibre splits a string containing multiple author names on +# ampersands and the words "and" and "with". You can customize the splitting +# by changing the regular expression below. Strings are split on whatever the +# specified regular expression matches. +# Default: r'(?i),?\s+(and|with)\s+' +authors_split_regex = r'(?i),?\s+(and|with)\s+' + #: Use author sort in Tag Browser # Set which author field to display in the tags pane (the list of authors, # series, publishers etc on the left hand side). The choices are author and diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 9d9cbd1d09..1474b540ee 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -571,7 +571,7 @@ from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS, from calibre.devices.sne.driver import SNE from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, - TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY) + TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY, EX124G) from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG from calibre.devices.kobo.driver import KOBO from calibre.devices.bambook.driver import BAMBOOK @@ -707,7 +707,7 @@ plugins += [ EEEREADER, NEXTBOOK, ADAM, - MOOVYBOOK, COBY, + MOOVYBOOK, COBY, EX124G, ITUNES, BOEYE_BEX, BOEYE_BDX, diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 0a497555c4..5c0dffd383 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -164,7 +164,7 @@ class ManyToOneField(Field): def for_book(self, book_id, default_value=None): ids = self.table.book_col_map.get(book_id, None) if ids is not None: - ans = self.id_map[ids] + ans = self.table.id_map[ids] else: ans = default_value return ans @@ -182,7 +182,7 @@ class ManyToOneField(Field): return self.table.id_map.iterkeys() def sort_keys_for_books(self, get_metadata, all_book_ids): - keys = {id_ : self._sort_key(self.id_map.get(id_, '')) for id_ in + keys = {id_ : self._sort_key(self.table.id_map.get(id_, '')) for id_ in all_book_ids} return {id_ : keys.get( self.book_col_map.get(id_, None), '') for id_ in all_book_ids} @@ -196,7 +196,7 @@ class ManyToManyField(Field): def for_book(self, book_id, default_value=None): ids = self.table.book_col_map.get(book_id, ()) if ids: - ans = tuple(self.id_map[i] for i in ids) + ans = tuple(self.table.id_map[i] for i in ids) else: ans = default_value return ans @@ -211,7 +211,7 @@ class ManyToManyField(Field): return self.table.id_map.iterkeys() def sort_keys_for_books(self, get_metadata, all_book_ids): - keys = {id_ : self._sort_key(self.id_map.get(id_, '')) for id_ in + keys = {id_ : self._sort_key(self.table.id_map.get(id_, '')) for id_ in all_book_ids} def sort_key_for_book(book_id): @@ -222,6 +222,13 @@ class ManyToManyField(Field): return {id_ : sort_key_for_book(id_) for id_ in all_book_ids} +class IdentifiersField(ManyToManyField): + + def for_book(self, book_id, default_value=None): + ids = self.table.book_col_map.get(book_id, ()) + if not ids: + ids = default_value + return ids class AuthorsField(ManyToManyField): @@ -249,6 +256,8 @@ def create_field(name, table): cls = OnDeviceField elif name == 'formats': cls = FormatsField + elif name == 'identifiers': + cls = IdentifiersField elif table.metadata['datatype'] == 'composite': cls = CompositeField return cls(name, table) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 185d15d86b..6a6730c2b4 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -66,8 +66,6 @@ class VirtualTable(Table): self.table_type = table_type Table.__init__(self, name, metadata) - - class OneToOneTable(Table): ''' diff --git a/src/calibre/db/tests/__init__.py b/src/calibre/db/tests/__init__.py new file mode 100644 index 0000000000..cc6da1e995 --- /dev/null +++ b/src/calibre/db/tests/__init__.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/db/tests/metadata.db b/src/calibre/db/tests/metadata.db new file mode 100644 index 0000000000..812bf296ba Binary files /dev/null and b/src/calibre/db/tests/metadata.db differ diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py new file mode 100644 index 0000000000..f25b308f79 --- /dev/null +++ b/src/calibre/db/tests/reading.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +import os, shutil, unittest, tempfile, datetime + +from calibre.utils.date import local_tz + +def create_db(library_path): + from calibre.library.database2 import LibraryDatabase2 + if LibraryDatabase2.exists_at(library_path): + raise ValueError('A library already exists at %r'%library_path) + src = os.path.join(os.path.dirname(__file__), 'metadata.db') + db = os.path.join(library_path, 'metadata.db') + shutil.copyfile(src, db) + return db + +def init_cache(library_path): + from calibre.db.backend import DB + from calibre.db.cache import Cache + backend = DB(library_path) + cache = Cache(backend) + cache.init() + return cache + +class ReadingTest(unittest.TestCase): + + def setUp(self): + self.library_path = tempfile.mkdtemp() + create_db(self.library_path) + + def tearDown(self): + shutil.rmtree(self.library_path) + + def test_read(self): # {{{ + cache = init_cache(self.library_path) + tests = { + 2 : { + 'title': 'Title One', + 'sort': 'One', + 'authors': ('Author One',), + 'author_sort': 'One, Author', + 'series' : 'Series One', + 'series_index': 1.0, + 'tags':('Tag Two', 'Tag One'), + 'rating': 4.0, + 'identifiers': {'test':'one'}, + 'timestamp': datetime.datetime(2011, 9, 5, 15, 6, + tzinfo=local_tz), + 'pubdate': datetime.datetime(2011, 9, 5, 15, 6, + tzinfo=local_tz), + 'publisher': 'Publisher One', + 'languages': ('eng',), + 'comments': '

Comments One

', + '#enum':'One', + '#authors':('Custom One', 'Custom Two'), + '#date':datetime.datetime(2011, 9, 5, 0, 0, + tzinfo=local_tz), + '#rating':2.0, + '#series':'My Series One', + '#series_index': 1.0, + '#tags':('My Tag One', 'My Tag Two'), + '#yesno':True, + '#comments': '
My Comments One

', + }, + 1 : { + 'title': 'Title Two', + 'sort': 'Title Two', + 'authors': ('Author Two', 'Author One'), + 'author_sort': 'Two, Author & One, Author', + 'series' : 'Series Two', + 'series_index': 2.0, + 'rating': 6.0, + 'tags': ('Tag Two',), + 'identifiers': {'test':'two'}, + 'timestamp': datetime.datetime(2011, 9, 6, 0, 0, + tzinfo=local_tz), + 'pubdate': datetime.datetime(2011, 8, 5, 0, 0, + tzinfo=local_tz), + 'publisher': 'Publisher Two', + 'languages': ('deu',), + 'comments': '

Comments Two

', + '#enum':'Two', + '#authors':('My Author Two',), + '#date':datetime.datetime(2011, 9, 1, 0, 0, + tzinfo=local_tz), + '#rating':4.0, + '#series':'My Series Two', + '#series_index': 3.0, + '#tags':('My Tag Two',), + '#yesno':False, + '#comments': '
My Comments Two

', + + }, + } + for book_id, test in tests.iteritems(): + for field, expected_val in test.iteritems(): + self.assertEqual(expected_val, + cache.field_for(field, book_id)) + # }}} + +def tests(): + return unittest.TestLoader().loadTestsFromTestCase(ReadingTest) + +def run(): + unittest.TextTestRunner(verbosity=2).run(tests()) + +if __name__ == '__main__': + run() + diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 79110d9585..20f8617396 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -180,6 +180,7 @@ def main(args=sys.argv): sys.path.insert(0, base) g = globals() g['__name__'] = '__main__' + g['__file__'] = ef execfile(ef, g) return diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 5785649739..a068590df4 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -127,7 +127,7 @@ class ANDROID(USBMS): VENDOR_NAME = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER', 'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS', 'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA', - 'GENERIC-', 'ZTE', 'MID', 'QUALCOMM'] + 'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -137,11 +137,11 @@ class ANDROID(USBMS): '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2', 'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK', 'MB525', 'ANDROID2.3', 'SGH-I997', 'GT-I5800_CARD', 'MB612', - 'GT-S5830_CARD', 'GT-S5570_CARD', 'MB870', 'MID7015A'] + 'GT-S5830_CARD', 'GT-S5570_CARD', 'MB870', 'MID7015A', 'ALPANDIGITAL'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', - '__UMS_COMPOSITE', 'SGH-I997_CARD', 'MB870'] + '__UMS_COMPOSITE', 'SGH-I997_CARD', 'MB870', 'ALPANDIGITAL'] OSX_MAIN_MEM = 'Android Device Main Memory' diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 90d03f073a..f25c41d073 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -377,3 +377,31 @@ class COBY(USBMS): return 'eBooks' return self.EBOOK_DIR_CARD_A +class EX124G(USBMS): + + name = 'Motorola Ex124G device interface' + gui_name = 'Ex124G' + description = _('Communicate with the Ex124G') + + author = 'Kovid Goyal' + supported_platforms = ['windows', 'osx', 'linux'] + + # Ordered list of supported formats + FORMATS = ['mobi', 'prc', 'azw'] + + VENDOR_ID = [0x0e8d] + PRODUCT_ID = [0x0002] + BCD = [0x0100] + VENDOR_NAME = 'MOTOROLA' + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '_PHONE' + + EBOOK_DIR_MAIN = 'eBooks' + + SUPPORTS_SUB_DIRS = False + + def get_carda_ebook_dir(self, for_upload=False): + if for_upload: + return 'eBooks' + return self.EBOOK_DIR_CARD_A + + diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 402ca65658..9c1dc930b8 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -291,6 +291,8 @@ class PRS505(USBMS): thumbnail_dir = os.path.join(thumbnail_dir, relpath) if not os.path.exists(thumbnail_dir): os.makedirs(thumbnail_dir) - with open(os.path.join(thumbnail_dir, 'main_thumbnail.jpg'), 'wb') as f: + cpath = os.path.join(thumbnail_dir, 'main_thumbnail.jpg') + with open(cpath, 'wb') as f: f.write(metadata.thumbnail[-1]) + debug_print('Cover uploaded to: %r'%cpath) diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py index 7ba4217f7d..0f804cc208 100644 --- a/src/calibre/ebooks/conversion/preprocess.py +++ b/src/calibre/ebooks/conversion/preprocess.py @@ -636,12 +636,3 @@ class HTMLPreProcessor(object): html = re.sub(r'\s--\s', u'\u2014', html) return substitute_entites(html) - def unsmarten_punctuation(self, html): - from calibre.utils.unsmarten import unsmarten_html - from calibre.ebooks.chardet import substitute_entites - from calibre.ebooks.conversion.utils import HeuristicProcessor - preprocessor = HeuristicProcessor(self.extra_opts, self.log) - html = preprocessor.fix_nbsp_indents(html) - html = unsmarten_html(html) - return substitute_entites(html) - diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index a9816db5ae..c3a229fe3c 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -10,11 +10,17 @@ import os, sys, re from urllib import unquote, quote from urlparse import urlparse -from calibre import relpath, guess_type, remove_bracketed_text +from calibre import relpath, guess_type, remove_bracketed_text, prints from calibre.utils.config import tweaks -_author_pat = re.compile(',?\s+(and|with)\s+', re.IGNORECASE) +try: + _author_pat = re.compile(tweaks['authors_split_regex']) +except: + prints ('Author split regexp:', tweaks['authors_split_regex'], + 'is invalid, using default') + _author_pat = re.compile(r'(?i),?\s+(and|with)\s+') + def string_to_authors(raw): raw = raw.replace('&&', u'\uffff') raw = _author_pat.sub('&', raw) @@ -45,6 +51,17 @@ def author_to_author_sort(author, method=None): if method == u'copy': return author + prefixes = set([x.lower() for x in tweaks['author_name_prefixes']]) + prefixes |= set([x+u'.' for x in prefixes]) + while True: + if not tokens: + return author + tok = tokens[0].lower() + if tok in prefixes: + tokens = tokens[1:] + else: + break + suffixes = set([x.lower() for x in tweaks['author_name_suffixes']]) suffixes |= set([x+u'.' for x in suffixes]) diff --git a/src/calibre/ebooks/metadata/epub.py b/src/calibre/ebooks/metadata/epub.py index 27fa94e217..30fe53f1a2 100644 --- a/src/calibre/ebooks/metadata/epub.py +++ b/src/calibre/ebooks/metadata/epub.py @@ -16,6 +16,7 @@ from calibre.ebooks.metadata.opf2 import OPF from calibre.ptempfile import TemporaryDirectory, PersistentTemporaryFile from calibre import CurrentDir, walk from calibre.constants import isosx +from calibre.utils.localization import lang_as_iso639_1 class EPubException(Exception): pass @@ -231,6 +232,15 @@ def set_metadata(stream, mi, apply_null=False, update_timestamp=False): for x in ('guide', 'toc', 'manifest', 'spine'): setattr(mi, x, None) + if mi.languages: + langs = [] + for lc in mi.languages: + lc2 = lang_as_iso639_1(lc) + if lc2: lc = lc2 + langs.append(lc) + mi.languages = langs + + reader.opf.smart_update(mi) if apply_null: if not getattr(mi, 'series', None): diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index aaa13d5769..fa72766a0a 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -74,6 +74,20 @@ class Worker(Thread): # Get details {{{ 9: ['sept'], 12: ['déc'], }, + 'jp': { + 1: [u'1月'], + 2: [u'2月'], + 3: [u'3月'], + 4: [u'4月'], + 5: [u'5月'], + 6: [u'6月'], + 7: [u'7月'], + 8: [u'8月'], + 9: [u'9月'], + 10: [u'10月'], + 11: [u'11月'], + 12: [u'12月'], + }, } @@ -86,13 +100,15 @@ class Worker(Thread): # Get details {{{ text()="Produktinformation" or \ text()="Dettagli prodotto" or \ text()="Product details" or \ - text()="Détails sur le produit"]/../div[@class="content"] + text()="Détails sur le produit" or \ + text()="登録情報"]/../div[@class="content"] ''' self.publisher_xpath = ''' descendant::*[starts-with(text(), "Publisher:") or \ starts-with(text(), "Verlag:") or \ starts-with(text(), "Editore:") or \ - starts-with(text(), "Editeur")] + starts-with(text(), "Editeur") or \ + starts-with(text(), "出版社:")] ''' self.language_xpath = ''' descendant::*[ @@ -101,10 +117,11 @@ class Worker(Thread): # Get details {{{ or text() = "Sprache:" \ or text() = "Lingua:" \ or starts-with(text(), "Langue") \ + or starts-with(text(), "言語") \ ] ''' self.ratings_pat = re.compile( - r'([0-9.]+) (out of|von|su|étoiles sur) (\d+)( (stars|Sternen|stelle)){0,1}') + r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち) ([\d\.]+)( (stars|Sternen|stelle)){0,1}') lm = { 'eng': ('English', 'Englisch'), @@ -112,6 +129,7 @@ class Worker(Thread): # Get details {{{ 'ita': ('Italian', 'Italiano'), 'deu': ('German', 'Deutsch'), 'spa': ('Spanish', 'Espa\xf1ol', 'Espaniol'), + 'jpn': ('Japanese', u'日本語'), } self.lang_map = {} for code, names in lm.iteritems(): @@ -403,6 +421,7 @@ class Amazon(Source): 'de' : _('Germany'), 'uk' : _('UK'), 'it' : _('Italy'), + 'jp' : _('Japan'), } options = ( @@ -411,6 +430,22 @@ class Amazon(Source): 'country\'s Amazon website.'), choices=AMAZON_DOMAINS), ) + def __init__(self, *args, **kwargs): + Source.__init__(self, *args, **kwargs) + self.set_amazon_id_touched_fields() + + def save_settings(self, *args, **kwargs): + Source.save_settings(self, *args, **kwargs) + self.set_amazon_id_touched_fields() + + def set_amazon_id_touched_fields(self): + ident_name = "identifier:amazon" + if self.domain != 'com': + ident_name += '_' + self.domain + tf = [x for x in self.touched_fields if not + x.startswith('identifier:amazon')] + [ident_name] + self.touched_fields = frozenset(tf) + def get_domain_and_asin(self, identifiers): for key, val in identifiers.iteritems(): key = key.lower() @@ -488,13 +523,23 @@ class Amazon(Source): # Insufficient metadata to make an identify query return None, None - latin1q = dict([(x.encode('latin1', 'ignore'), y.encode('latin1', + # magic parameter to enable Japanese Shift_JIS encoding. + if domain == 'jp': + q['__mk_ja_JP'] = u'カタカナ' + + if domain == 'jp': + encode_to = 'Shift_JIS' + else: + encode_to = 'latin1' + encoded_q = dict([(x.encode(encode_to, 'ignore'), y.encode(encode_to, 'ignore')) for x, y in q.iteritems()]) udomain = domain if domain == 'uk': udomain = 'co.uk' - url = 'http://www.amazon.%s/s/?'%udomain + urlencode(latin1q) + elif domain == 'jp': + udomain = 'co.jp' + url = 'http://www.amazon.%s/s/?'%udomain + urlencode(encoded_q) return url, domain # }}} @@ -663,7 +708,7 @@ if __name__ == '__main__': # tests {{{ # To run these test use: calibre-debug -e # src/calibre/ebooks/metadata/sources/amazon.py from calibre.ebooks.metadata.sources.test import (test_identify_plugin, - title_test, authors_test) + isbn_test, title_test, authors_test) com_tests = [ # {{{ ( # Description has links @@ -744,6 +789,21 @@ if __name__ == '__main__': # tests {{{ ), ] # }}} + jp_tests = [ # {{{ + ( # isbn -> title, authors + {'identifiers':{'isbn': '9784101302720' }}, + [title_test(u'精霊の守り人', + exact=True), authors_test([u'上橋 菜穂子']) + ] + ), + ( # title, authors -> isbn (will use Shift_JIS encoding in query.) + {'title': u'考えない練習', + 'authors': [u'小池 龍之介']}, + [isbn_test('9784093881067'), ] + ), + ] # }}} + test_identify_plugin(Amazon.name, com_tests) + #test_identify_plugin(Amazon.name, jp_tests) # }}} diff --git a/src/calibre/ebooks/oeb/output.py b/src/calibre/ebooks/oeb/output.py index 816fe71abb..38ac2495fd 100644 --- a/src/calibre/ebooks/oeb/output.py +++ b/src/calibre/ebooks/oeb/output.py @@ -43,6 +43,7 @@ class OEBOutput(OutputFormatPlugin): except: self.log.exception('Something went wrong while trying to' ' workaround Pocketbook cover bug, ignoring') + self.migrate_lang_code(root) raw = etree.tostring(root, pretty_print=True, encoding='utf-8', xml_declaration=True) if key == OPF_MIME: @@ -104,3 +105,12 @@ class OEBOutput(OutputFormatPlugin): p.remove(m) p.insert(0, m) # }}} + + def migrate_lang_code(self, root): # {{{ + from calibre.utils.localization import lang_as_iso639_1 + for lang in root.xpath('//*[local-name() = "language"]'): + clc = lang_as_iso639_1(lang.text) + if clc: + lang.text = clc + # }}} + diff --git a/src/calibre/ebooks/oeb/transforms/unsmarten.py b/src/calibre/ebooks/oeb/transforms/unsmarten.py index a83fa6f39f..c01094681f 100644 --- a/src/calibre/ebooks/oeb/transforms/unsmarten.py +++ b/src/calibre/ebooks/oeb/transforms/unsmarten.py @@ -10,16 +10,22 @@ from calibre.ebooks.oeb.base import OEB_DOCS, XPath, barename from calibre.utils.unsmarten import unsmarten_text class UnsmartenPunctuation(object): - + + def __init__(self): + self.html_tags = XPath('descendant::h:*') + def unsmarten(self, root): - for x in XPath('//h:*')(root): + for x in self.html_tags(root): if not barename(x) == 'pre': - if hasattr(x, 'text') and x.text: + if getattr(x, 'text', None): x.text = unsmarten_text(x.text) - if hasattr(x, 'tail') and x.tail: + if getattr(x, 'tail', None) and x.tail: x.tail = unsmarten_text(x.tail) def __call__(self, oeb, context): + bx = XPath('//h:body') for x in oeb.manifest.items: if x.media_type in OEB_DOCS: - self.unsmarten(x.data) + for body in bx(x.data): + self.unsmarten(body) + diff --git a/src/calibre/ebooks/txt/textileml.py b/src/calibre/ebooks/txt/textileml.py index 500ce1d9c7..de712abd07 100644 --- a/src/calibre/ebooks/txt/textileml.py +++ b/src/calibre/ebooks/txt/textileml.py @@ -83,7 +83,7 @@ class TextileMLizer(OEB2HTML): for i in self.our_ids: if i not in self.our_links: text = re.sub(r'%?\('+i+'\)\xa0?%?', r'', text) - + # Remove obvious non-needed escaping, add sub/sup-script ones text = check_escaping(text, ['\*', '_', '\*']) # escape the super/sub-scripts if needed @@ -189,7 +189,7 @@ class TextileMLizer(OEB2HTML): emright = int(round(right / stylizer.profile.fbase)) if emright >= 1: txt += ')' * emright - + return txt def check_id_tag(self, attribs): @@ -235,7 +235,7 @@ class TextileMLizer(OEB2HTML): tags = [] tag = barename(elem.tag) attribs = elem.attrib - + # Ignore anything that is set to not be displayed. if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \ or style['visibility'] == 'hidden': @@ -246,7 +246,7 @@ class TextileMLizer(OEB2HTML): ems = int(round(float(style.marginTop) / style.fontSize) - 1) if ems >= 1: text.append(u'\n\n\xa0' * ems) - + if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'div'): if tag == 'div': tag = 'p' @@ -432,7 +432,7 @@ class TextileMLizer(OEB2HTML): 'span', 'table', 'tr', 'td'): if not self.in_a_link: text.append(self.check_styles(style)) - + # Process tags that contain text. if hasattr(elem, 'text') and elem.text: txt = elem.text diff --git a/src/calibre/gui2/actions/__init__.py b/src/calibre/gui2/actions/__init__.py index 4f42a1d2bc..fa67130a1c 100644 --- a/src/calibre/gui2/actions/__init__.py +++ b/src/calibre/gui2/actions/__init__.py @@ -15,6 +15,8 @@ from calibre import prints from calibre.gui2 import Dispatcher from calibre.gui2.keyboard import NameConflict +def menu_action_unique_name(plugin, unique_name): + return u'%s : menu action : %s'%(plugin.unique_name, unique_name) class InterfaceAction(QObject): @@ -178,30 +180,30 @@ class InterfaceAction(QObject): description=None, triggered=None, shortcut_name=None): ''' Convenience method to easily add actions to a QMenu. + Returns the created QAction, This action has one extra attribute + calibre_shortcut_unique_name which if not None refers to the unique + name under which this action is registered with the keyboard manager. :param menu: The QMenu the newly created action will be added to :param unique_name: A unique name for this action, this must be - globally unique, so make it as descriptive as possible. If in doubt add - a uuid to it. + globally unique, so make it as descriptive as possible. If in doubt add + a uuid to it. :param text: The text of the action. :param icon: Either a QIcon or a file name. The file name is passed to - the I() builtin, so you do not need to pass the full path to the images - directory. + the I() builtin, so you do not need to pass the full path to the images + directory. :param shortcut: A string, a list of strings, None or False. If False, - no keyboard shortcut is registered for this action. If None, a keyboard - shortcut with no default keybinding is registered. String and list of - strings register a shortcut with default keybinding as specified. + no keyboard shortcut is registered for this action. If None, a keyboard + shortcut with no default keybinding is registered. String and list of + strings register a shortcut with default keybinding as specified. :param description: A description for this action. Used to set - tooltips. + tooltips. :param triggered: A callable which is connected to the triggered signal - of the created action. + of the created action. :param shortcut_name: The test displayed to the user when customizing - the keyboard shortcuts for this action. By default it is set to the - value of ``text``. + the keyboard shortcuts for this action. By default it is set to the + value of ``text``. - :return: The created QAction, This action has one extra attribute - calibre_shortcut_unique_name which if not None refers to the unique - name under which this action is registered with the keyboard manager. ''' if shortcut_name is None: shortcut_name = unicode(text) @@ -214,7 +216,7 @@ class InterfaceAction(QObject): if shortcut is not None and shortcut is not False: keys = ((shortcut,) if isinstance(shortcut, basestring) else tuple(shortcut)) - unique_name = '%s : menu action : %s'%(self.unique_name, unique_name) + unique_name = menu_action_unique_name(self, unique_name) if description is not None: ac.setToolTip(description) ac.setStatusTip(description) diff --git a/src/calibre/gui2/convert/look_and_feel.py b/src/calibre/gui2/convert/look_and_feel.py index 5ca7e1ea02..4785e222fc 100644 --- a/src/calibre/gui2/convert/look_and_feel.py +++ b/src/calibre/gui2/convert/look_and_feel.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import SIGNAL, QVariant +from PyQt4.Qt import SIGNAL, QVariant, Qt from calibre.gui2.convert.look_and_feel_ui import Ui_Form from calibre.gui2.convert import Widget @@ -45,6 +45,12 @@ class LookAndFeelWidget(Widget, Ui_Form): self.font_key_wizard) self.opt_remove_paragraph_spacing.toggle() self.opt_remove_paragraph_spacing.toggle() + self.opt_smarten_punctuation.stateChanged.connect( + lambda state: state != Qt.Unchecked and + self.opt_unsmarten_punctuation.setCheckState(Qt.Unchecked)) + self.opt_unsmarten_punctuation.stateChanged.connect( + lambda state: state != Qt.Unchecked and + self.opt_smarten_punctuation.setCheckState(Qt.Unchecked)) def get_value_handler(self, g): if g is self.opt_change_justification: diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index 076a9720a3..b7992eb319 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -21,7 +21,7 @@ from calibre.utils.ipc.job import ParallelJob from calibre.gui2 import Dispatcher, error_dialog, question_dialog, NONE, config, gprefs from calibre.gui2.device import DeviceJob from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog -from calibre import __appname__ +from calibre import __appname__, as_unicode from calibre.gui2.dialogs.job_view_ui import Ui_Dialog from calibre.gui2.progress_indicator import ProgressIndicator from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob @@ -264,6 +264,26 @@ class JobManager(QAbstractTableModel): # {{{ _('This job cannot be stopped'), show=True) self._kill_job(job) + def kill_multiple_jobs(self, rows, view): + jobs = [self.jobs[row] for row in rows] + devjobs = [j for j in jobs is isinstance(j, DeviceJob)] + if devjobs: + error_dialog(view, _('Cannot kill job'), + _('Cannot kill jobs that communicate with the device')).exec_() + jobs = [j for j in jobs if not isinstance(j, DeviceJob)] + jobs = [j for j in jobs if j.duration is None] + unkillable = [j for j in jobs if not getattr(j, 'killable', True)] + if unkillable: + names = u'\n'.join(as_unicode(j.description) for j in unkillable) + error_dialog(view, _('Cannot kill job'), + _('Some of the jobs cannot be stopped. Click Show details' + ' to see the list of unstoppable jobs.'), det_msg=names, + show=True) + jobs = [j for j in jobs if getattr(j, 'killable', True)] + jobs = [j for j in jobs if j.duration is None] + for j in jobs: + self._kill_job(j) + def kill_all_jobs(self): for job in self.jobs: if (isinstance(job, DeviceJob) or job.duration is not None or @@ -484,8 +504,10 @@ class JobsDialog(QDialog, Ui_JobsDialog): ngettext('Do you really want to stop the selected job?', 'Do you really want to stop all the selected jobs?', len(rows))): - for row in rows: - self.model.kill_job(row, self) + if len(rows) > 1: + self.model.kill_multiple_jobs(rows, self) + else: + self.model.kill_job(rows[0], self) def kill_all_jobs(self, *args): if question_dialog(self, _('Are you sure?'), diff --git a/src/calibre/gui2/keyboard.py b/src/calibre/gui2/keyboard.py index 6413b216ce..9b0b1d8f69 100644 --- a/src/calibre/gui2/keyboard.py +++ b/src/calibre/gui2/keyboard.py @@ -116,6 +116,11 @@ class Manager(QObject): # {{{ done unregistering. ''' self.shortcuts.pop(unique_name, None) + for group in self.groups.itervalues(): + try: + group.remove(unique_name) + except ValueError: + pass def finalize(self): custom_keys_map = {un:tuple(keys) for un, keys in self.config.get( diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index 06a503f855..20507b4ce1 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -235,11 +235,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.highlight_index(idx) def highlight_index(self, idx): - self.plugin_view.scrollTo(idx) self.plugin_view.selectionModel().select(idx, self.plugin_view.selectionModel().ClearAndSelect) self.plugin_view.setCurrentIndex(idx) self.plugin_view.setFocus(Qt.OtherFocusReason) + self.plugin_view.scrollTo(idx, self.plugin_view.EnsureVisible) def find_next(self, *args): idx = self.plugin_view.currentIndex() diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 5ea8b4b49f..51aefe214b 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -5113,6 +5113,8 @@ Author '{0}': if catalog_source_built: recommendations = [] + recommendations.append(('remove_fake_margins', False, + OptionRecommendation.HIGH)) if DEBUG: recommendations.append(('comments', '\n'.join(line for line in build_log), OptionRecommendation.HIGH)) diff --git a/src/calibre/manual/epub.py b/src/calibre/manual/epub.py index a162303b09..1aadbe9f91 100644 --- a/src/calibre/manual/epub.py +++ b/src/calibre/manual/epub.py @@ -6,7 +6,9 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, time +import os, time, glob + +from lxml import etree from sphinx.builders.epub import EpubBuilder @@ -55,4 +57,38 @@ class EPUBHelpBuilder(EpubBuilder): def build_epub(self, outdir, *args, **kwargs): if self.config.epub_cover: self.add_cover(outdir, self.config.epub_cover) + self.fix_duplication_bugs(outdir) EpubBuilder.build_epub(self, outdir, *args, **kwargs) + + def fix_duplication_bugs(self, outdir): + opf = glob.glob(outdir+os.sep+'*.opf')[0] + root = etree.fromstring(open(opf, 'rb').read()) + seen = set() + for x in root.xpath( + '//*[local-name()="spine"]/*[local-name()="itemref"]'): + idref = x.get('idref') + if idref in seen: + x.getparent().remove(x) + else: + seen.add(idref) + + with open(opf, 'wb') as f: + f.write(etree.tostring(root, encoding='utf-8', xml_declaration=True)) + + + ncx = glob.glob(outdir+os.sep+'*.ncx')[0] + root = etree.fromstring(open(ncx, 'rb').read()) + seen = set() + for x in root.xpath( + '//*[local-name()="navMap"]/*[local-name()="navPoint"]'): + text = x.xpath('descendant::*[local-name()="text"]')[0] + text = text.text + if text in seen: + x.getparent().remove(x) + else: + seen.add(text) + + with open(ncx, 'wb') as f: + f.write(etree.tostring(root, encoding='utf-8', xml_declaration=True)) + + diff --git a/src/calibre/utils/mreplace.py b/src/calibre/utils/mreplace.py index b9fbc0bded..70591d6ca7 100644 --- a/src/calibre/utils/mreplace.py +++ b/src/calibre/utils/mreplace.py @@ -7,26 +7,32 @@ import re from UserDict import UserDict class MReplace(UserDict): - def __init__(self, dict = None): - UserDict.__init__(self, dict) + + def __init__(self, data=None, case_sensitive=True): + UserDict.__init__(self, data) self.re = None self.regex = None + self.case_sensitive = case_sensitive self.compile_regex() - def compile_regex(self): + def compile_regex(self): if len(self.data) > 0: keys = sorted(self.data.keys(), key=len) keys.reverse() tmp = "(%s)" % "|".join(map(re.escape, keys)) if self.re != tmp: self.re = tmp - self.regex = re.compile(self.re) + if self.case_sensitive: + self.regex = re.compile(self.re) + else: + self.regex = re.compile(self.re, re.I) - def __call__(self, mo): + def __call__(self, mo): return self[mo.string[mo.start():mo.end()]] - def mreplace(self, text): + def mreplace(self, text): #Replace without regex compile if len(self.data) < 1 or self.re is None: return text - return self.regex.sub(self, text) \ No newline at end of file + return self.regex.sub(self, text) + diff --git a/src/calibre/utils/unsmarten.py b/src/calibre/utils/unsmarten.py index b9f9175599..6f0c2a19e1 100644 --- a/src/calibre/utils/unsmarten.py +++ b/src/calibre/utils/unsmarten.py @@ -6,15 +6,38 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import re +from calibre.utils.mreplace import MReplace -def unsmarten_text(txt): - txt = re.sub(u'–|–|–', r'--', txt) # en-dash - txt = re.sub(u'—|—|—', r'---', txt) # em-dash - txt = re.sub(u'…|…|…', r'...', txt) # ellipsis +_mreplace = MReplace({ + '–': '--', + '–': '--', + '–': '--', + '—': '---', + '—': '---', + '—': '---', + '…': '...', + '…': '...', + '…': '...', + '“': '"', + '”': '"', + '″': '"', + '“': '"', + '”': '"', + '″': '"', + '“':'"', + '”':'"', + '″':'"', + '‘':"'", + '’':"'", + '′':"'", + '‘':"'", + '’':"'", + '′':"'", + '‘':"'", + '’':"'", + '′':"'", +}) - txt = re.sub(u'“|”|″|“|”|″|“|”|″', r'"', txt) # double quote - txt = re.sub(u'‘|’|′|‘|’|′|‘|’|′', r"'", txt) # single quote +unsmarten_text = _mreplace.mreplace - return txt