From cdd3c6bc48b51ea573e0be495b8cc40e50b21c55 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Sep 2011 19:11:41 -0600 Subject: [PATCH 01/10] Fix #840020 (String typo please update (no replace)) --- resources/default_tweaks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index f11a0b7bc0..b385511d56 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -63,7 +63,7 @@ authors_completer_append_separator = False # end of an author name. The case of the suffix is ignored and trailing # periods are automatically handled. # The author name copy words are a set of words which if they occur in an -# author name cause the automatically geenrated author sort string to be +# author name cause the automatically generated author sort string to be # identical to the author name. This means that the sort for a string like Acme # Inc. will be Acme Inc. instead of Inc., Acme author_sort_copy_method = 'comma' From c756509cb32afd173580204f37c182ee5257012c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 2 Sep 2011 20:29:35 -0600 Subject: [PATCH 02/10] ... --- src/calibre/db/backend.py | 6 ++++-- src/calibre/db/cache.py | 6 ++++-- src/calibre/db/tables.py | 20 ++++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index afafa3a18a..c29e0d5c12 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -25,7 +25,8 @@ from calibre.utils.config import to_json, from_json, prefs, tweaks from calibre.utils.date import utcfromtimestamp, parse_date from calibre.utils.filenames import is_case_sensitive from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, - SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, CompositeTable) + SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, + CompositeTable, LanguagesTable) # }}} ''' @@ -604,11 +605,12 @@ class DB(object): for col in ('series', 'publisher', 'rating'): tables[col] = ManyToOneTable(col, self.field_metadata[col].copy()) - for col in ('authors', 'tags', 'formats', 'identifiers'): + for col in ('authors', 'tags', 'formats', 'identifiers', 'languages'): cls = { 'authors':AuthorsTable, 'formats':FormatsTable, 'identifiers':IdentifiersTable, + 'languages':LanguagesTable, }.get(col, ManyToManyTable) tables[col] = cls(col, self.field_metadata[col].copy()) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index ea3766ad87..7158fc0267 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -13,7 +13,8 @@ from functools import wraps, partial from calibre.db.locking import create_locks, RecordLock from calibre.db.fields import create_field -from calibre.ebooks.book.base import Metadata +from calibre.db.tables import VirtualTable +from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.date import now def api(f): @@ -189,7 +190,8 @@ class Cache(object): if table.metadata['datatype'] == 'composite': self.composites.add(field) - self.fields['ondevice'] = create_field('ondevice', None) + self.fields['ondevice'] = create_field('ondevice', + VirtualTable('ondevice')) @read_api def field_for(self, name, book_id, default_value=None): diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index fa7b001851..185d15d86b 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -13,6 +13,7 @@ from dateutil.tz import tzoffset from calibre.constants import plugins from calibre.utils.date import parse_date, local_tz, UNDEFINED_DATE +from calibre.utils.localization import lang_map from calibre.ebooks.metadata import author_to_author_sort _c_speedup = plugins['speedup'][0] @@ -54,6 +55,19 @@ class Table(object): self.link_table = (link_table if link_table else 'books_%s_link'%self.metadata['table']) +class VirtualTable(Table): + + ''' + A dummy table used for fields that only exist in memory like ondevice + ''' + + def __init__(self, name, table_type=ONE_ONE, datatype='text'): + metadata = {'datatype':datatype, 'table':name} + self.table_type = table_type + Table.__init__(self, name, metadata) + + + class OneToOneTable(Table): ''' @@ -210,3 +224,9 @@ class IdentifiersTable(ManyToManyTable): for key in tuple(self.col_book_map.iterkeys()): self.col_book_map[key] = tuple(self.col_book_map[key]) +class LanguagesTable(ManyToManyTable): + + def read_id_maps(self, db): + ManyToManyTable.read_id_maps(self, db) + lm = lang_map() + self.lang_name_map = {x:lm.get(x, x) for x in self.id_map.itervalues()} From 2425e2c09bb5283241e92a425e9798548e2ac454 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Sep 2011 08:24:46 -0600 Subject: [PATCH 03/10] Content server: Fix --url-prefix not used for links in the book details view. --- src/calibre/library/server/base.py | 2 ++ src/calibre/library/server/browse.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/server/base.py b/src/calibre/library/server/base.py index 2f345eb670..26e4d3469e 100644 --- a/src/calibre/library/server/base.py +++ b/src/calibre/library/server/base.py @@ -49,6 +49,8 @@ class DispatchController(object): # {{{ elif self.prefix: self.dispatcher.connect(name+'prefix_extra', self.prefix, self, **kwargs) + self.dispatcher.connect(name+'prefix_extra_trailing', + self.prefix+'/', self, **kwargs) self.dispatcher.connect(name, route, self, **kwargs) self.funcs.append(expose(func)) diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index e4c21e4ed9..afea6baa32 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -695,8 +695,8 @@ class BrowseServer(object): for tag in dbtags: tval = ('{2}') - href='/browse/matches/%s/%s' % \ - (quote(tag.category), quote(str(tag.id))) + href='%s/browse/matches/%s/%s' % \ + (self.opts.url_prefix, quote(tag.category), quote(str(tag.id))) vals.append(tval.format(xml(tag.name, True), xml(href, True), xml(val if len(dbtags) == 1 else tag.name), From 384e24a06aea0a66405577cbb40cc80a75c059f4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Sep 2011 08:49:59 -0600 Subject: [PATCH 04/10] Fix Economist not downloading all aticles from Technology Quarterly --- recipes/economist.recipe | 43 ++++++++++++++--------------------- recipes/economist_free.recipe | 43 ++++++++++++++--------------------- 2 files changed, 34 insertions(+), 52 deletions(-) diff --git a/recipes/economist.recipe b/recipes/economist.recipe index 92dafeaf6f..7dc869bf74 100644 --- a/recipes/economist.recipe +++ b/recipes/economist.recipe @@ -77,32 +77,23 @@ class Economist(BasicNewsRecipe): continue self.log('Found section: %s'%section_title) articles = [] - for h5 in section.findAll('h5'): - article_title = self.tag_to_string(h5).strip() - if not article_title: - continue - data = h5.findNextSibling(attrs={'class':'article'}) - if data is None: continue - a = data.find('a', href=True) - if a is None: continue - url = a['href'] - if url.startswith('/'): url = 'http://www.economist.com'+url - url += '/print' - article_title += ': %s'%self.tag_to_string(a).strip() - articles.append({'title':article_title, 'url':url, - 'description':'', 'date':''}) - if not articles: - # We have last or first section - for art in section.findAll(attrs={'class':'article'}): - a = art.find('a', href=True) - if a is not None: - url = a['href'] - if url.startswith('/'): url = 'http://www.economist.com'+url - url += '/print' - title = self.tag_to_string(a) - if title: - articles.append({'title':title, 'url':url, - 'description':'', 'date':''}) + subsection = '' + for node in section.findAll(attrs={'class':'article'}): + subsec = node.findPreviousSibling('h5') + if subsec is not None: + subsection = self.tag_to_string(subsec) + prefix = (subsection+': ') if subsection else '' + a = node.find('a', href=True) + if a is not None: + url = a['href'] + if url.startswith('/'): url = 'http://www.economist.com'+url + url += '/print' + title = self.tag_to_string(a) + if title: + title = prefix + title + self.log('\tFound article:', title) + articles.append({'title':title, 'url':url, + 'description':'', 'date':''}) if articles: if section_title not in feeds: diff --git a/recipes/economist_free.recipe b/recipes/economist_free.recipe index cc3f48805d..5f45a6ab8f 100644 --- a/recipes/economist_free.recipe +++ b/recipes/economist_free.recipe @@ -69,32 +69,23 @@ class Economist(BasicNewsRecipe): continue self.log('Found section: %s'%section_title) articles = [] - for h5 in section.findAll('h5'): - article_title = self.tag_to_string(h5).strip() - if not article_title: - continue - data = h5.findNextSibling(attrs={'class':'article'}) - if data is None: continue - a = data.find('a', href=True) - if a is None: continue - url = a['href'] - if url.startswith('/'): url = 'http://www.economist.com'+url - url += '/print' - article_title += ': %s'%self.tag_to_string(a).strip() - articles.append({'title':article_title, 'url':url, - 'description':'', 'date':''}) - if not articles: - # We have last or first section - for art in section.findAll(attrs={'class':'article'}): - a = art.find('a', href=True) - if a is not None: - url = a['href'] - if url.startswith('/'): url = 'http://www.economist.com'+url - url += '/print' - title = self.tag_to_string(a) - if title: - articles.append({'title':title, 'url':url, - 'description':'', 'date':''}) + subsection = '' + for node in section.findAll(attrs={'class':'article'}): + subsec = node.findPreviousSibling('h5') + if subsec is not None: + subsection = self.tag_to_string(subsec) + prefix = (subsection+': ') if subsection else '' + a = node.find('a', href=True) + if a is not None: + url = a['href'] + if url.startswith('/'): url = 'http://www.economist.com'+url + url += '/print' + title = self.tag_to_string(a) + if title: + title = prefix + title + self.log('\tFound article:', title) + articles.append({'title':title, 'url':url, + 'description':'', 'date':''}) if articles: if section_title not in feeds: From 3f50c415c403383f088ecb49385e5f9586955ebd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Sep 2011 08:52:30 -0600 Subject: [PATCH 05/10] ... --- src/calibre/db/cache.py | 5 +++-- src/calibre/db/fields.py | 14 ++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 7158fc0267..01e97bb4c3 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -347,8 +347,9 @@ class Cache(object): as_path=as_path) @read_api - def multisort(self, fields): - all_book_ids = frozenset(self._all_book_ids()) + def multisort(self, fields, ids_to_sort=None): + all_book_ids = frozenset(self._all_book_ids() if ids_to_sort is None + else ids_to_sort) get_metadata = partial(self._get_metadata, get_user_categories=False) sort_keys = tuple(self.fields[field[0]].sort_keys_for_books(get_metadata, diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 7fd19501ce..0a497555c4 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -51,9 +51,13 @@ class Field(object): def __iter__(self): ''' - Iterate over the ids for all values in this field + Iterate over the ids for all values in this field. + + WARNING: Some fields such as composite fields and virtual + fields like ondevice do not have ids for their values, in such + cases this is an empty iterator. ''' - raise NotImplementedError() + return iter(()) def sort_keys_for_books(self, get_metadata, all_book_ids): ''' @@ -78,9 +82,6 @@ class OneToOneField(Field): def __iter__(self): return self.table.book_col_map.iterkeys() - def iter_book_ids(self): - return self.table.book_col_map.iterkeys() - def sort_keys_for_books(self, get_metadata, all_book_ids): return {id_ : self._sort_key(self.book_col_map.get(id_, '')) for id_ in all_book_ids} @@ -154,9 +155,6 @@ class OnDeviceField(OneToOneField): def __iter__(self): return iter(()) - def iter_book_ids(self): - return iter(()) - def sort_keys_for_books(self, get_metadata, all_book_ids): return {id_ : self.for_book(id_) for id_ in all_book_ids} From 3b65dde5645caf39eea6f618296aa6065d2e97fe Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Sep 2011 08:57:02 -0600 Subject: [PATCH 06/10] Cyanogen mod on LG android phone. Fixes #840372 (LG-P509 (Optimus T) not being detected.) --- src/calibre/devices/android/driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 63ec658b30..5785649739 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -81,6 +81,7 @@ class ANDROID(USBMS): # LG 0x1004 : { + 0x61c5 : [0x100, 0x226, 0x9999], 0x61cc : [0x100], 0x61ce : [0x100], 0x618e : [0x226, 0x9999, 0x100] From 7af3b6e882cb2f4f8a4a6f3502e459f458419bb2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Sep 2011 10:14:39 -0600 Subject: [PATCH 07/10] Content server AJAX interface: Add a dictionary that maps authors/tags/series/publishers/etc to URLs that list the books by that author/tag/publisher/etc. --- src/calibre/library/server/ajax.py | 50 +++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/src/calibre/library/server/ajax.py b/src/calibre/library/server/ajax.py index a8cf7c2672..98b4be1fb0 100644 --- a/src/calibre/library/server/ajax.py +++ b/src/calibre/library/server/ajax.py @@ -83,6 +83,10 @@ def category_url(prefix, cid): def icon_url(prefix, name): return absurl(prefix, '/browse/icon/'+name) + +def books_in_url(prefix, category, cid): + return absurl(prefix, '/ajax/books_in/%s/%s'%( + encode_name(category), encode_name(cid))) # }}} class AjaxServer(object): @@ -114,7 +118,7 @@ class AjaxServer(object): # Get book metadata {{{ - def ajax_book_to_json(self, book_id): + def ajax_book_to_json(self, book_id, get_category_urls=True): mi = self.db.get_metadata(book_id, index_is_id=True) try: mi.rating = mi.rating/2. @@ -151,18 +155,46 @@ class AjaxServer(object): data['other_formats'] = {fmt: absurl(self.opts.url_prefix, u'/get/%s/%d'%(fmt, book_id)) for fmt in other_fmts} + if get_category_urls: + category_urls = data['category_urls'] = {} + ccache = self.categories_cache() + for key in mi.all_field_keys(): + fm = mi.metadata_for_field(key) + if (fm and fm['is_category'] and not fm['is_csp'] and + key != 'formats' and fm['datatype'] not in ['rating']): + categories = mi.get(key) + if isinstance(categories, basestring): + categories = [categories] + if categories is None: + categories = [] + dbtags = {} + for category in categories: + for tag in ccache.get(key, []): + if tag.original_name == category: + dbtags[category] = books_in_url(self.opts.url_prefix, + tag.category if tag.category else key, + tag.original_name if tag.id is None else + unicode(tag.id)) + break + category_urls[key] = dbtags + return data, mi.last_modified @Endpoint(set_last_modified=False) - def ajax_book(self, book_id): + def ajax_book(self, book_id, category_urls='true'): ''' Return the metadata of the book as a JSON dictionary. + + If category_urls == 'true' the returned dictionary also contains a + mapping of category names to URLs that return the list of books in the + given category. ''' cherrypy.response.timeout = 3600 try: book_id = int(book_id) - data, last_modified = self.ajax_book_to_json(book_id) + data, last_modified = self.ajax_book_to_json(book_id, + get_category_urls=category_urls.lower()=='true') except: raise cherrypy.HTTPError(404, 'No book with id: %r'%book_id) @@ -172,7 +204,7 @@ class AjaxServer(object): return data @Endpoint(set_last_modified=False) - def ajax_books(self, ids=None): + def ajax_books(self, ids=None, category_urls='true'): ''' Return the metadata for a list of books specified as a comma separated list of ids. The metadata is returned as a dictionary mapping ids to @@ -192,9 +224,11 @@ class AjaxServer(object): ' of integers') ans = {} lm = None + gcu = category_urls.lower()=='true' for book_id in ids: try: - data, last_modified = self.ajax_book_to_json(book_id) + data, last_modified = self.ajax_book_to_json(book_id, + get_category_urls=gcu) except: ans[book_id] = None else: @@ -431,9 +465,9 @@ class AjaxServer(object): 'name':item_names.get(x, x.original_name), 'average_rating': x.avg_rating, 'count': x.count, - 'url': absurl(self.opts.url_prefix, '/ajax/books_in/%s/%s'%( - encode_name(x.category if x.category else toplevel), - encode_name(x.original_name if x.id is None else unicode(x.id)))), + 'url': books_in_url(self.opts.url_prefix, + x.category if x.category else toplevel, + x.original_name if x.id is None else unicode(x.id)), 'has_children': x.original_name in children, } for x in items] From 3470004cc46ec40ee4b03522d376bebbda39ce7c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Sep 2011 12:38:03 -0600 Subject: [PATCH 08/10] ... --- src/calibre/gui2/keyboard.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/keyboard.py b/src/calibre/gui2/keyboard.py index b40845d61a..806f1d2f17 100644 --- a/src/calibre/gui2/keyboard.py +++ b/src/calibre/gui2/keyboard.py @@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en' from collections import OrderedDict from functools import partial +import sip from PyQt4.Qt import (QObject, QKeySequence, QAbstractItemModel, QModelIndex, Qt, QStyledItemDelegate, QTextDocument, QStyle, pyqtSignal, QFrame, QApplication, QSize, QRectF, QWidget, QTreeView, @@ -125,11 +126,18 @@ class Manager(QObject): # {{{ #pprint.pprint(self.keys_map) def replace_action(self, unique_name, new_action): + ''' + Replace the action associated with a shortcut. If this method returns + False you must call finalize() to have shortcuts set on the new action + correctly. + ''' sc = self.shortcuts[unique_name] ac = sc['action'] - if ac is not None: + if ac is not None and not sip.isdeleted(ac): new_action.setShortcuts(ac.shortcuts()) ac.setShortcuts([]) + return True + return False # }}} From 384e8ff3204b5576a40f2c008ec7197ebaa5d8fa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Sep 2011 14:13:48 -0600 Subject: [PATCH 09/10] ... --- src/calibre/gui2/actions/__init__.py | 3 +-- src/calibre/gui2/keyboard.py | 15 +++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/actions/__init__.py b/src/calibre/gui2/actions/__init__.py index 4f2d5fe559..4f42a1d2bc 100644 --- a/src/calibre/gui2/actions/__init__.py +++ b/src/calibre/gui2/actions/__init__.py @@ -220,12 +220,11 @@ class InterfaceAction(QObject): ac.setStatusTip(description) ac.setWhatsThis(description) - ac.calibre_shortcut_unique_name = None + ac.calibre_shortcut_unique_name = unique_name if shortcut is not False: self.gui.keyboard.register_shortcut(unique_name, shortcut_name, default_keys=keys, action=ac, description=description, group=self.action_spec[0]) - ac.calibre_shortcut_unique_name = unique_name if triggered is not None: ac.triggered.connect(triggered) return ac diff --git a/src/calibre/gui2/keyboard.py b/src/calibre/gui2/keyboard.py index 806f1d2f17..6413b216ce 100644 --- a/src/calibre/gui2/keyboard.py +++ b/src/calibre/gui2/keyboard.py @@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en' from collections import OrderedDict from functools import partial -import sip from PyQt4.Qt import (QObject, QKeySequence, QAbstractItemModel, QModelIndex, Qt, QStyledItemDelegate, QTextDocument, QStyle, pyqtSignal, QFrame, QApplication, QSize, QRectF, QWidget, QTreeView, @@ -127,17 +126,13 @@ class Manager(QObject): # {{{ def replace_action(self, unique_name, new_action): ''' - Replace the action associated with a shortcut. If this method returns - False you must call finalize() to have shortcuts set on the new action - correctly. + Replace the action associated with a shortcut. + Once you're done calling replace_action() for all shortcuts you want + replaced, call finalize() to have the shortcuts assigned to the replaced + actions. ''' sc = self.shortcuts[unique_name] - ac = sc['action'] - if ac is not None and not sip.isdeleted(ac): - new_action.setShortcuts(ac.shortcuts()) - ac.setShortcuts([]) - return True - return False + sc['action'] = new_action # }}} From c7bf67e6c3cc430e3d7a9b26befdcaaca216c8f0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 3 Sep 2011 14:42:04 -0600 Subject: [PATCH 10/10] Various fixes to markdown found by the automatic python checker --- setup/check.py | 8 ++-- .../ebooks/markdown/blockprocessors.py | 44 +++++++++---------- src/calibre/ebooks/markdown/commandline.py | 4 +- src/calibre/ebooks/markdown/etree_loader.py | 2 + .../ebooks/markdown/extensions/def_list.py | 8 ++-- .../ebooks/markdown/extensions/footnotes.py | 16 +++---- .../ebooks/markdown/extensions/headerid.py | 9 ++-- src/calibre/ebooks/markdown/inlinepatterns.py | 2 +- src/calibre/ebooks/markdown/odict.py | 6 +-- src/calibre/ebooks/markdown/treeprocessors.py | 8 ++-- 10 files changed, 53 insertions(+), 54 deletions(-) diff --git a/setup/check.py b/setup/check.py index 6c0acd78ae..9142de27c1 100644 --- a/setup/check.py +++ b/setup/check.py @@ -63,10 +63,10 @@ class Check(Command): for f in x[-1]: y = self.j(x[0], f) mtime = os.stat(y).st_mtime - if f.endswith('.py') and f not in ('ptempfile.py', 'feedparser.py', - 'pyparsing.py', 'markdown.py') and \ - 'genshi' not in y and cache.get(y, 0) != mtime and \ - 'prs500/driver.py' not in y: + if (f.endswith('.py') and f not in ('ptempfile.py', 'feedparser.py', + 'pyparsing.py', 'markdown.py') and + 'genshi' not in y and cache.get(y, 0) != mtime and + 'prs500/driver.py' not in y): yield y, mtime for x in os.walk(self.j(self.d(self.SRC), 'recipes')): diff --git a/src/calibre/ebooks/markdown/blockprocessors.py b/src/calibre/ebooks/markdown/blockprocessors.py index 79f4db93bc..e42d0f55e6 100644 --- a/src/calibre/ebooks/markdown/blockprocessors.py +++ b/src/calibre/ebooks/markdown/blockprocessors.py @@ -3,10 +3,10 @@ CORE MARKDOWN BLOCKPARSER ============================================================================= This parser handles basic parsing of Markdown blocks. It doesn't concern itself -with inline elements such as **bold** or *italics*, but rather just catches +with inline elements such as **bold** or *italics*, but rather just catches blocks, lists, quotes, etc. -The BlockParser is made up of a bunch of BlockProssors, each handling a +The BlockParser is made up of a bunch of BlockProssors, each handling a different type of block. Extensions may add/replace/remove BlockProcessors as they need to alter how markdown blocks are parsed. @@ -16,8 +16,8 @@ import re import markdown class BlockProcessor: - """ Base class for block processors. - + """ Base class for block processors. + Each subclass will provide the methods below to work with the source and tree. Each processor will need to define it's own ``test`` and ``run`` methods. The ``test`` method should return True or False, to indicate @@ -58,32 +58,32 @@ class BlockProcessor: return '\n'.join(lines) def test(self, parent, block): - """ Test for block type. Must be overridden by subclasses. - + """ Test for block type. Must be overridden by subclasses. + As the parser loops through processors, it will call the ``test`` method on each to determine if the given block of text is of that type. This method must return a boolean ``True`` or ``False``. The actual method of - testing is left to the needs of that particular block type. It could + testing is left to the needs of that particular block type. It could be as simple as ``block.startswith(some_string)`` or a complex regular expression. As the block type may be different depending on the parent - of the block (i.e. inside a list), the parent etree element is also + of the block (i.e. inside a list), the parent etree element is also provided and may be used as part of the test. Keywords: - + * ``parent``: A etree element which will be the parent of the block. - * ``block``: A block of text from the source which has been split at + * ``block``: A block of text from the source which has been split at blank lines. """ pass def run(self, parent, blocks): - """ Run processor. Must be overridden by subclasses. - + """ Run processor. Must be overridden by subclasses. + When the parser determines the appropriate type of a block, the parser will call the corresponding processor's ``run`` method. This method should parse the individual lines of the block and append them to - the etree. + the etree. Note that both the ``parent`` and ``etree`` keywords are pointers to instances of the objects which should be edited in place. Each @@ -103,8 +103,8 @@ class BlockProcessor: class ListIndentProcessor(BlockProcessor): - """ Process children of list items. - + """ Process children of list items. + Example: * a list item process this part @@ -154,7 +154,7 @@ class ListIndentProcessor(BlockProcessor): """ Create a new li and parse the block with it as the parent. """ li = markdown.etree.SubElement(parent, 'li') self.parser.parseBlocks(li, [block]) - + def get_level(self, parent, block): """ Get level of indent based on list level. """ # Get indent level @@ -188,7 +188,7 @@ class CodeBlockProcessor(BlockProcessor): def test(self, parent, block): return block.startswith(' '*markdown.TAB_LENGTH) - + def run(self, parent, blocks): sibling = self.lastChild(parent) block = blocks.pop(0) @@ -208,7 +208,7 @@ class CodeBlockProcessor(BlockProcessor): block, theRest = self.detab(block) code.text = markdown.AtomicString('%s\n' % block.rstrip()) if theRest: - # This block contained unindented line(s) after the first indented + # This block contained unindented line(s) after the first indented # line. Insert these lines as the first block of the master blocks # list for future processing. blocks.insert(0, theRest) @@ -229,7 +229,7 @@ class BlockQuoteProcessor(BlockProcessor): # Pass lines before blockquote in recursively for parsing forst. self.parser.parseBlocks(parent, [before]) # Remove ``> `` from begining of each line. - block = '\n'.join([self.clean(line) for line in + block = '\n'.join([self.clean(line) for line in block[m.start():].split('\n')]) sibling = self.lastChild(parent) if sibling and sibling.tag == "blockquote": @@ -355,7 +355,7 @@ class HashHeaderProcessor(BlockProcessor): blocks.insert(0, after) else: # This should never happen, but just in case... - message(CRITICAL, "We've got a problem header!") + print("We've got a problem header!") class SetextHeaderProcessor(BlockProcessor): @@ -407,7 +407,7 @@ class HRProcessor(BlockProcessor): # Recursively parse lines before hr so they get parsed first. self.parser.parseBlocks(parent, ['\n'.join(prelines)]) # create hr - hr = markdown.etree.SubElement(parent, 'hr') + markdown.etree.SubElement(parent, 'hr') # check for lines in block after hr. lines = lines[len(prelines)+1:] if len(lines): @@ -418,7 +418,7 @@ class HRProcessor(BlockProcessor): class EmptyBlockProcessor(BlockProcessor): """ Process blocks and start with an empty line. """ - # Detect a block that only contains whitespace + # Detect a block that only contains whitespace # or only whitespace on the first line. RE = re.compile(r'^\s*\n') diff --git a/src/calibre/ebooks/markdown/commandline.py b/src/calibre/ebooks/markdown/commandline.py index 1eedc6dbb1..9be4026444 100644 --- a/src/calibre/ebooks/markdown/commandline.py +++ b/src/calibre/ebooks/markdown/commandline.py @@ -9,7 +9,7 @@ Markdown is called from the command line. import markdown import sys import logging -from logging import DEBUG, INFO, WARN, ERROR, CRITICAL +from logging import DEBUG, INFO, CRITICAL EXECUTABLE_NAME_FOR_USAGE = "python markdown.py" """ The name used in the usage statement displayed for python versions < 2.3. @@ -57,7 +57,7 @@ def parse_options(): parser.add_option("-s", "--safe", dest="safe", default=False, metavar="SAFE_MODE", help="safe mode ('replace', 'remove' or 'escape' user's HTML tag)") - parser.add_option("-o", "--output_format", dest="output_format", + parser.add_option("-o", "--output_format", dest="output_format", default='xhtml1', metavar="OUTPUT_FORMAT", help="Format of output. One of 'xhtml1' (default) or 'html4'.") parser.add_option("--noisy", diff --git a/src/calibre/ebooks/markdown/etree_loader.py b/src/calibre/ebooks/markdown/etree_loader.py index e2599b2cb9..2591328084 100644 --- a/src/calibre/ebooks/markdown/etree_loader.py +++ b/src/calibre/ebooks/markdown/etree_loader.py @@ -8,9 +8,11 @@ def importETree(): etree_in_c = None try: # Is it Python 2.5+ with C implemenation of ElementTree installed? import xml.etree.cElementTree as etree_in_c + etree_in_c except ImportError: try: # Is it Python 2.5+ with Python implementation of ElementTree? import xml.etree.ElementTree as etree + etree except ImportError: try: # An earlier version of Python with cElementTree installed? import cElementTree as etree_in_c diff --git a/src/calibre/ebooks/markdown/extensions/def_list.py b/src/calibre/ebooks/markdown/extensions/def_list.py index 5bc1eb56b0..b5419bdfd2 100644 --- a/src/calibre/ebooks/markdown/extensions/def_list.py +++ b/src/calibre/ebooks/markdown/extensions/def_list.py @@ -8,7 +8,7 @@ Added parsing of Definition Lists to Python-Markdown. A simple example: Apple - : Pomaceous fruit of plants of the genus Malus in + : Pomaceous fruit of plants of the genus Malus in the family Rosaceae. : An american computer company. @@ -80,11 +80,11 @@ class DefListIndentProcessor(markdown.blockprocessors.ListIndentProcessor): ITEM_TYPES = ['dd'] LIST_TYPES = ['dl'] - def create_item(parent, block): + def create_item(self, parent, block): """ Create a new dd and parse the block with it as the parent. """ dd = markdown.etree.SubElement(parent, 'dd') self.parser.parseBlocks(dd, [block]) - + class DefListExtension(markdown.Extension): @@ -95,7 +95,7 @@ class DefListExtension(markdown.Extension): md.parser.blockprocessors.add('defindent', DefListIndentProcessor(md.parser), '>indent') - md.parser.blockprocessors.add('deflist', + md.parser.blockprocessors.add('deflist', DefListProcessor(md.parser), '>ulist') diff --git a/src/calibre/ebooks/markdown/extensions/footnotes.py b/src/calibre/ebooks/markdown/extensions/footnotes.py index 729b49b2e6..df137f5d53 100644 --- a/src/calibre/ebooks/markdown/extensions/footnotes.py +++ b/src/calibre/ebooks/markdown/extensions/footnotes.py @@ -43,7 +43,7 @@ class FootnoteExtension(markdown.Extension): for key, value in configs: self.config[key][0] = value - + self.reset() def extendMarkdown(self, md, md_globals): @@ -82,7 +82,7 @@ class FootnoteExtension(markdown.Extension): return (child, element), False finder(child) return None - + res = finder(root) return res @@ -106,7 +106,7 @@ class FootnoteExtension(markdown.Extension): div = etree.Element("div") div.set('class', 'footnote') - hr = etree.SubElement(div, "hr") + etree.SubElement(div, "hr") ol = etree.SubElement(div, "ol") for id in self.footnotes.keys(): @@ -149,9 +149,9 @@ class FootnotePreprocessor(markdown.preprocessors.Preprocessor): Keywords: * lines: A list of lines of text - + Return: A list of lines with footnote definitions removed. - + """ i, id, footnote = self._findFootnoteDefinition(lines) @@ -175,9 +175,9 @@ class FootnotePreprocessor(markdown.preprocessors.Preprocessor): * lines: A list of lines of text. Return: A three item tuple containing the index of the first line of a - footnote definition, the id of the definition and the body of the + footnote definition, the id of the definition and the body of the definition. - + """ counter = 0 for line in lines: @@ -199,7 +199,6 @@ class FootnotePreprocessor(markdown.preprocessors.Preprocessor): """ items = [] - item = -1 i = 0 # to keep track of where we are def detab(line): @@ -277,7 +276,6 @@ class FootnoteTreeprocessor(markdown.treeprocessors.Treeprocessor): ind = element.getchildren().find(child) element.getchildren().insert(ind + 1, footnotesDiv) child.tail = None - fnPlaceholder.parent.replaceChild(fnPlaceholder, footnotesDiv) else: root.append(footnotesDiv) diff --git a/src/calibre/ebooks/markdown/extensions/headerid.py b/src/calibre/ebooks/markdown/extensions/headerid.py index 7433673736..0ffd91ee93 100644 --- a/src/calibre/ebooks/markdown/extensions/headerid.py +++ b/src/calibre/ebooks/markdown/extensions/headerid.py @@ -57,7 +57,7 @@ Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/). Project website: Contact: markdown@freewisdom.org -License: BSD (see ../docs/LICENSE for details) +License: BSD (see ../docs/LICENSE for details) Dependencies: * [Python 2.3+](http://python.org) @@ -66,7 +66,6 @@ Dependencies: """ import calibre.ebooks.markdown.markdown as markdown -from calibre.ebooks.markdown.markdown import etree import re from string import ascii_lowercase, digits, punctuation @@ -106,7 +105,7 @@ class HeaderIdProcessor(markdown.blockprocessors.BlockProcessor): # Create header using named groups from RE start_level, force_id = self._get_meta() level = len(m.group('level')) + start_level - if level > 6: + if level > 6: level = 6 h = markdown.etree.SubElement(parent, 'h%d' % level) h.text = m.group('header').strip() @@ -119,7 +118,7 @@ class HeaderIdProcessor(markdown.blockprocessors.BlockProcessor): blocks.insert(0, after) else: # This should never happen, but just in case... - message(CRITICAL, "We've got a problem header!") + print ("We've got a problem header!") def _get_meta(self): """ Return meta data suported by this ext as a tuple """ @@ -128,7 +127,7 @@ class HeaderIdProcessor(markdown.blockprocessors.BlockProcessor): if hasattr(self.md, 'Meta'): if self.md.Meta.has_key('header_level'): level = int(self.md.Meta['header_level'][0]) - 1 - if self.md.Meta.has_key('header_forceid'): + if self.md.Meta.has_key('header_forceid'): force = self._str2bool(self.md.Meta['header_forceid'][0]) return level, force diff --git a/src/calibre/ebooks/markdown/inlinepatterns.py b/src/calibre/ebooks/markdown/inlinepatterns.py index 89fa3b2ef4..217697ed8d 100644 --- a/src/calibre/ebooks/markdown/inlinepatterns.py +++ b/src/calibre/ebooks/markdown/inlinepatterns.py @@ -47,6 +47,7 @@ from urlparse import urlparse, urlunparse import sys if sys.version >= "3.0": from html import entities as htmlentitydefs + htmlentitydefs else: import htmlentitydefs @@ -215,7 +216,6 @@ class HtmlPattern (Pattern): """ Store raw inline html and return a placeholder. """ def handleMatch (self, m): rawhtml = m.group(2) - inline = True place_holder = self.markdown.htmlStash.store(rawhtml) return place_holder diff --git a/src/calibre/ebooks/markdown/odict.py b/src/calibre/ebooks/markdown/odict.py index bf3ef07182..277cd4ebba 100644 --- a/src/calibre/ebooks/markdown/odict.py +++ b/src/calibre/ebooks/markdown/odict.py @@ -1,7 +1,7 @@ class OrderedDict(dict): """ A dictionary that keeps its keys in the order in which they're inserted. - + Copied from Django's SortedDict with some modifications. """ @@ -156,7 +156,7 @@ class OrderedDict(dict): self.keyOrder.insert(i, key) else: self.keyOrder.append(key) - except Error: + except Exception as e: # restore to prevent data loss and reraise self.keyOrder.insert(n, key) - raise Error + raise e diff --git a/src/calibre/ebooks/markdown/treeprocessors.py b/src/calibre/ebooks/markdown/treeprocessors.py index 1dc612a95e..0604b1970f 100644 --- a/src/calibre/ebooks/markdown/treeprocessors.py +++ b/src/calibre/ebooks/markdown/treeprocessors.py @@ -24,8 +24,8 @@ class Treeprocessor(Processor): def run(self, root): """ Subclasses of Treeprocessor should implement a `run` method, which - takes a root ElementTree. This method can return another ElementTree - object, and the existing root ElementTree will be replaced, or it can + takes a root ElementTree. This method can return another ElementTree + object, and the existing root ElementTree will be replaced, or it can modify the current tree and return None. """ pass @@ -185,7 +185,7 @@ class InlineProcessor(Treeprocessor): result.append(node) else: # wrong placeholder - end = index + len(prefix) + end = index + len(self.__placeholder_prefix) linkText(data[strartIndex:end]) strartIndex = end else: @@ -278,7 +278,7 @@ class InlineProcessor(Treeprocessor): for element, lst in insertQueue: if element.text: element.text = \ - markdown.inlinepatterns.handleAttributes(element.text, + markdown.inlinepatterns.handleAttributes(element.text, element) i = 0 for newChild in lst: