diff --git a/src/calibre/ebooks/markdown/__init__.py b/src/calibre/ebooks/markdown/__init__.py index ca99c59b7b..332752e501 100644 --- a/src/calibre/ebooks/markdown/__init__.py +++ b/src/calibre/ebooks/markdown/__init__.py @@ -10,7 +10,7 @@ called from the command line. import markdown html = markdown.markdown(your_text_string) -See for more +See for more information and instructions on how to extend the functionality of Python Markdown. Read that before you try modifying this file. @@ -32,11 +32,12 @@ License: BSD (see LICENSE for details). from __future__ import absolute_import from __future__ import unicode_literals -from .__version__ import version, version_info -import re +from .__version__ import version, version_info # noqa import codecs import sys import logging +import warnings +import importlib from . import util from .preprocessors import build_preprocessors from .blockprocessors import build_block_parser @@ -48,6 +49,7 @@ from .serializers import to_html_string, to_xhtml_string __all__ = ['Markdown', 'markdown', 'markdownFromFile'] + logger = logging.getLogger('MARKDOWN') @@ -57,24 +59,24 @@ class Markdown(object): doc_tag = "div" # Element used to wrap document - later removed option_defaults = { - 'html_replacement_text' : '[HTML_REMOVED]', - 'tab_length' : 4, - 'enable_attributes' : True, - 'smart_emphasis' : True, - 'lazy_ol' : True, + 'html_replacement_text': '[HTML_REMOVED]', + 'tab_length': 4, + 'enable_attributes': True, + 'smart_emphasis': True, + 'lazy_ol': True, } output_formats = { - 'html' : to_html_string, - 'html4' : to_html_string, - 'html5' : to_html_string, - 'xhtml' : to_xhtml_string, + 'html': to_html_string, + 'html4': to_html_string, + 'html5': to_html_string, + 'xhtml': to_xhtml_string, 'xhtml1': to_xhtml_string, 'xhtml5': to_xhtml_string, } ESCAPED_CHARS = ['\\', '`', '*', '_', '{', '}', '[', ']', - '(', ')', '>', '#', '+', '-', '.', '!'] + '(', ')', '>', '#', '+', '-', '.', '!'] def __init__(self, *args, **kwargs): """ @@ -86,36 +88,42 @@ class Markdown(object): If they are of type string, the module mdx_name.py will be loaded. If they are a subclass of markdown.Extension, they will be used as-is. - * extension_configs: Configuration settingis for extensions. + * extension_configs: Configuration settings for extensions. * output_format: Format of output. Supported formats are: * "xhtml1": Outputs XHTML 1.x. Default. * "xhtml5": Outputs XHTML style tags of HTML 5 - * "xhtml": Outputs latest supported version of XHTML (currently XHTML 1.1). + * "xhtml": Outputs latest supported version of XHTML + (currently XHTML 1.1). * "html4": Outputs HTML 4 * "html5": Outputs HTML style tags of HTML 5 - * "html": Outputs latest supported version of HTML (currently HTML 4). + * "html": Outputs latest supported version of HTML + (currently HTML 4). Note that it is suggested that the more specific formats ("xhtml1" and "html4") be used as "xhtml" or "html" may change in the future if it makes sense at that time. - * safe_mode: Disallow raw html. One of "remove", "replace" or "escape". - * html_replacement_text: Text used when safe_mode is set to "replace". + * safe_mode: Deprecated! Disallow raw html. One of "remove", "replace" + or "escape". + * html_replacement_text: Deprecated! Text used when safe_mode is set + to "replace". * tab_length: Length of tabs in the source. Default: 4 * enable_attributes: Enable the conversion of attributes. Default: True - * smart_emphasis: Treat `_connected_words_` intelegently Default: True + * smart_emphasis: Treat `_connected_words_` intelligently Default: True * lazy_ol: Ignore number of first item of ordered lists. Default: True """ # For backward compatibility, loop through old positional args pos = ['extensions', 'extension_configs', 'safe_mode', 'output_format'] - c = 0 - for arg in args: + for c, arg in enumerate(args): if pos[c] not in kwargs: kwargs[pos[c]] = arg - c += 1 - if c == len(pos): + if c+1 == len(pos): # pragma: no cover # ignore any additional args break + if len(args): + warnings.warn('Positional arguments are deprecated in Markdown. ' + 'Use keyword arguments only.', + DeprecationWarning) # Loop through kwargs and assign defaults for option, default in self.option_defaults.items(): @@ -126,6 +134,19 @@ class Markdown(object): # Disable attributes in safeMode when not explicitly set self.enable_attributes = False + if 'safe_mode' in kwargs: + warnings.warn('"safe_mode" is deprecated in Python-Markdown. ' + 'Use an HTML sanitizer (like ' + 'Bleach http://bleach.readthedocs.org/) ' + 'if you are parsing untrusted markdown text. ' + 'See the 2.6 release notes for more info', + DeprecationWarning) + + if 'html_replacement_text' in kwargs: + warnings.warn('The "html_replacement_text" keyword is ' + 'deprecated along with "safe_mode".', + DeprecationWarning) + self.registeredExtensions = [] self.docType = "" self.stripTopLevelTags = True @@ -134,9 +155,9 @@ class Markdown(object): self.references = {} self.htmlStash = util.HtmlStash() - self.set_output_format(kwargs.get('output_format', 'xhtml1')) self.registerExtensions(extensions=kwargs.get('extensions', []), configs=kwargs.get('extension_configs', {})) + self.set_output_format(kwargs.get('output_format', 'xhtml1')) self.reset() def build_parser(self): @@ -161,9 +182,13 @@ class Markdown(object): """ for ext in extensions: if isinstance(ext, util.string_type): - ext = self.build_extension(ext, configs.get(ext, [])) + ext = self.build_extension(ext, configs.get(ext, {})) if isinstance(ext, Extension): ext.extendMarkdown(self, globals()) + logger.debug( + 'Successfully loaded extension "%s.%s".' + % (ext.__class__.__module__, ext.__class__.__name__) + ) elif ext is not None: raise TypeError( 'Extension "%s.%s" must be of type: "markdown.Extension"' @@ -171,7 +196,7 @@ class Markdown(object): return self - def build_extension(self, ext_name, configs = []): + def build_extension(self, ext_name, configs): """Build extension by name, then return the module. The extension name may contain arguments as part of the string in the @@ -179,44 +204,102 @@ class Markdown(object): """ - # Parse extensions config params (ignore the order) configs = dict(configs) - pos = ext_name.find("(") # find the first "(" + + # Parse extensions config params (ignore the order) + pos = ext_name.find("(") # find the first "(" if pos > 0: ext_args = ext_name[pos+1:-1] ext_name = ext_name[:pos] pairs = [x.split("=") for x in ext_args.split(",")] configs.update([(x.strip(), y.strip()) for (x, y) in pairs]) + warnings.warn('Setting configs in the Named Extension string is ' + 'deprecated. It is recommended that you ' + 'pass an instance of the extension class to ' + 'Markdown or use the "extension_configs" keyword. ' + 'The current behavior will raise an error in version 2.7. ' + 'See the Release Notes for Python-Markdown version ' + '2.6 for more info.', DeprecationWarning) - # Setup the module name - module_name = ext_name + # Get class name (if provided): `path.to.module:ClassName` + ext_name, class_name = ext_name.split(':', 1) \ + if ':' in ext_name else (ext_name, '') if '.' not in ext_name: - module_name = '.'.join(['calibre.ebooks.markdown.extensions', ext_name]) + ext_name = 'markdown.extensions.' + ext_name + if ext_name.startswith('markdown.'): + ext_name = 'calibre.ebooks.' + ext_name # Try loading the extension first from one place, then another - try: # New style (markdown.extensons.) - module = __import__(module_name, {}, {}, [module_name.rpartition('.')[0]]) + try: + # Assume string uses dot syntax (`path.to.some.module`) + module = importlib.import_module(ext_name) + logger.debug( + 'Successfuly imported extension module "%s".' % ext_name + ) + # For backward compat (until deprecation) + # check that this is an extension. + if ('.' not in ext_name and not (hasattr(module, 'makeExtension') or + (class_name and hasattr(module, class_name)))): + # We have a name conflict + # eg: extensions=['tables'] and PyTables is installed + raise ImportError except ImportError: - module_name_old_style = '_'.join(['mdx', ext_name]) - try: # Old style (mdx_) - module = __import__(module_name_old_style) - except ImportError as e: - message = "Failed loading extension '%s' from '%s' or '%s'" \ - % (ext_name, module_name, module_name_old_style) + # Preppend `markdown.extensions.` to name + module_name = '.'.join(['markdown.extensions', ext_name]) + try: + module = importlib.import_module(module_name) + logger.debug( + 'Successfuly imported extension module "%s".' % + module_name + ) + warnings.warn('Using short names for Markdown\'s builtin ' + 'extensions is deprecated. Use the ' + 'full path to the extension with Python\'s dot ' + 'notation (eg: "%s" instead of "%s"). The ' + 'current behavior will raise an error in version ' + '2.7. See the Release Notes for ' + 'Python-Markdown version 2.6 for more info.' % + (module_name, ext_name), + DeprecationWarning) + except ImportError: + # Preppend `mdx_` to name + module_name_old_style = '_'.join(['mdx', ext_name]) + try: + module = importlib.import_module(module_name_old_style) + logger.debug( + 'Successfuly imported extension module "%s".' % + module_name_old_style) + warnings.warn('Markdown\'s behavior of prepending "mdx_" ' + 'to an extension name is deprecated. ' + 'Use the full path to the ' + 'extension with Python\'s dot notation ' + '(eg: "%s" instead of "%s"). The current ' + 'behavior will raise an error in version 2.7. ' + 'See the Release Notes for Python-Markdown ' + 'version 2.6 for more info.' % + (module_name_old_style, ext_name), + DeprecationWarning) + except ImportError as e: + message = "Failed loading extension '%s' from '%s', '%s' " \ + "or '%s'" % (ext_name, ext_name, module_name, + module_name_old_style) + e.args = (message,) + e.args[1:] + raise + + if class_name: + # Load given class name from module. + return getattr(module, class_name)(**configs) + else: + # Expect makeExtension() function to return a class. + try: + return module.makeExtension(**configs) + except AttributeError as e: + message = e.args[0] + message = "Failed to initiate extension " \ + "'%s': %s" % (ext_name, message) e.args = (message,) + e.args[1:] raise - # If the module is loaded successfully, we expect it to define a - # function called makeExtension() - try: - return module.makeExtension(configs.items()) - except AttributeError as e: - message = e.args[0] - message = "Failed to initiate extension " \ - "'%s': %s" % (ext_name, message) - e.args = (message,) + e.args[1:] - raise - def registerExtension(self, extension): """ This gets called by the extension """ self.registeredExtensions.append(extension) @@ -244,8 +327,8 @@ class Markdown(object): valid_formats = list(self.output_formats.keys()) valid_formats.sort() message = 'Invalid Output Format: "%s". Use one of %s.' \ - % (self.output_format, - '"' + '", "'.join(valid_formats) + '"') + % (self.output_format, + '"' + '", "'.join(valid_formats) + '"') e.args = (message,) + e.args[1:] raise return self @@ -294,23 +377,25 @@ class Markdown(object): # Run the tree-processors for treeprocessor in self.treeprocessors.values(): newRoot = treeprocessor.run(root) - if newRoot: + if newRoot is not None: root = newRoot # Serialize _properly_. Strip top-level tags. output = self.serializer(root) if self.stripTopLevelTags: try: - start = output.index('<%s>'%self.doc_tag)+len(self.doc_tag)+2 - end = output.rindex(''%self.doc_tag) + start = output.index( + '<%s>' % self.doc_tag) + len(self.doc_tag) + 2 + end = output.rindex('' % self.doc_tag) output = output[start:end].strip() - except ValueError: - if output.strip().endswith('<%s />'%self.doc_tag): + except ValueError: # pragma: no cover + if output.strip().endswith('<%s />' % self.doc_tag): # We have an empty document output = '' else: # We have a serious problem - raise ValueError('Markdown failed to strip top-level tags. Document=%r' % output.strip()) + raise ValueError('Markdown failed to strip top-level ' + 'tags. Document=%r' % output.strip()) # Run the text post-processors for pp in self.postprocessors.values(): @@ -319,7 +404,7 @@ class Markdown(object): return output.strip() def convertFile(self, input=None, output=None, encoding=None): - """Converts a markdown file and returns the HTML as a unicode string. + """Converts a Markdown file and returns the HTML as a Unicode string. Decodes the file using the provided encoding (defaults to utf-8), passes the file content to markdown, and outputs the html to either @@ -327,9 +412,9 @@ class Markdown(object): encoding as the source file. The 'xmlcharrefreplace' error handler is used when encoding the output. - **Note:** This is the only place that decoding and encoding of unicode - takes place in Python-Markdown. (All other code is unicode-in / - unicode-out.) + **Note:** This is the only place that decoding and encoding of Unicode + takes place in Python-Markdown. (All other code is Unicode-in / + Unicode-out.) Keyword arguments: @@ -354,7 +439,7 @@ class Markdown(object): if not isinstance(text, util.text_type): text = text.decode(encoding) - text = text.lstrip('\ufeff') # remove the byte-order mark + text = text.lstrip('\ufeff') # remove the byte-order mark # Convert html = self.convert(text) @@ -373,7 +458,7 @@ class Markdown(object): output_file.write(html) # Don't close here. User may want to write more. else: - # Encode manually and write bytes to stdout. + # Encode manually and write bytes to stdout. html = html.encode(encoding, "xmlcharrefreplace") try: # Write bytes directly to buffer (Python 3). @@ -393,8 +478,9 @@ Those are the two functions we really mean to export: markdown() and markdownFromFile(). """ + def markdown(text, *args, **kwargs): - """Convert a markdown string to HTML and return HTML as a unicode string. + """Convert a Markdown string to HTML and return HTML as a Unicode string. This is a shortcut function for `Markdown` class to cover the most basic use case. It initializes an instance of Markdown, loads the @@ -435,9 +521,13 @@ def markdownFromFile(*args, **kwargs): c += 1 if c == len(pos): break + if len(args): + warnings.warn('Positional arguments are depreacted in ' + 'Markdown and will raise an error in version 2.7. ' + 'Use keyword arguments only.', + DeprecationWarning) md = Markdown(**kwargs) md.convertFile(kwargs.get('input', None), kwargs.get('output', None), kwargs.get('encoding', None)) - diff --git a/src/calibre/ebooks/markdown/__main__.py b/src/calibre/ebooks/markdown/__main__.py index dc20ab7ea6..7c1f54a64b 100644 --- a/src/calibre/ebooks/markdown/__main__.py +++ b/src/calibre/ebooks/markdown/__main__.py @@ -4,23 +4,30 @@ COMMAND-LINE SPECIFIC STUFF """ -import calibre.ebooks.markdown as markdown import sys import optparse +import codecs +import warnings +import calibre.ebooks.markdown as markdown +try: + import yaml +except ImportError: # pragma: no cover + import json as yaml import logging -from logging import DEBUG, INFO, CRITICAL +from logging import DEBUG, WARNING, CRITICAL -logger = logging.getLogger('MARKDOWN') +logger = logging.getLogger('MARKDOWN') -def parse_options(): + +def parse_options(args=None, values=None): """ Define and parse `optparse` options for command-line usage. """ usage = """%prog [options] [INPUTFILE] (STDIN is assumed if no INPUTFILE is given)""" desc = "A Python implementation of John Gruber's Markdown. " \ - "http://packages.python.org/Markdown/" + "https://pythonhosted.org/Markdown/" ver = "%%prog %s" % markdown.version parser = optparse.OptionParser(usage=usage, description=desc, version=ver) @@ -29,28 +36,40 @@ def parse_options(): metavar="OUTPUT_FILE") parser.add_option("-e", "--encoding", dest="encoding", help="Encoding for input and output files.",) - parser.add_option("-q", "--quiet", default = CRITICAL, - action="store_const", const=CRITICAL+10, dest="verbose", - help="Suppress all warnings.") - parser.add_option("-v", "--verbose", - action="store_const", const=INFO, dest="verbose", - help="Print all warnings.") parser.add_option("-s", "--safe", dest="safe", default=False, metavar="SAFE_MODE", - help="'replace', 'remove' or 'escape' HTML tags in input") + help="Deprecated! 'replace', 'remove' or 'escape' HTML " + "tags in input") parser.add_option("-o", "--output_format", dest="output_format", default='xhtml1', metavar="OUTPUT_FORMAT", help="'xhtml1' (default), 'html4' or 'html5'.") - parser.add_option("--noisy", - action="store_const", const=DEBUG, dest="verbose", - help="Print debug messages.") - parser.add_option("-x", "--extension", action="append", dest="extensions", - help = "Load extension EXTENSION.", metavar="EXTENSION") parser.add_option("-n", "--no_lazy_ol", dest="lazy_ol", action='store_false', default=True, help="Observe number of first item of ordered lists.") + parser.add_option("-x", "--extension", action="append", dest="extensions", + help="Load extension EXTENSION.", metavar="EXTENSION") + parser.add_option("-c", "--extension_configs", + dest="configfile", default=None, + help="Read extension configurations from CONFIG_FILE. " + "CONFIG_FILE must be of JSON or YAML format. YAML" + "format requires that a python YAML library be " + "installed. The parsed JSON or YAML must result in a " + "python dictionary which would be accepted by the " + "'extension_configs' keyword on the markdown.Markdown " + "class. The extensions must also be loaded with the " + "`--extension` option.", + metavar="CONFIG_FILE") + parser.add_option("-q", "--quiet", default=CRITICAL, + action="store_const", const=CRITICAL+10, dest="verbose", + help="Suppress all warnings.") + parser.add_option("-v", "--verbose", + action="store_const", const=WARNING, dest="verbose", + help="Print all warnings.") + parser.add_option("--noisy", + action="store_const", const=DEBUG, dest="verbose", + help="Print debug messages.") - (options, args) = parser.parse_args() + (options, args) = parser.parse_args(args, values) if len(args) == 0: input_file = None @@ -60,28 +79,58 @@ def parse_options(): if not options.extensions: options.extensions = [] - return {'input': input_file, - 'output': options.filename, - 'safe_mode': options.safe, - 'extensions': options.extensions, - 'encoding': options.encoding, - 'output_format': options.output_format, - 'lazy_ol': options.lazy_ol}, options.verbose + extension_configs = {} + if options.configfile: + with codecs.open( + options.configfile, mode="r", encoding=options.encoding + ) as fp: + try: + extension_configs = yaml.load(fp) + except Exception as e: + message = "Failed parsing extension config file: %s" % \ + options.configfile + e.args = (message,) + e.args[1:] + raise -def run(): + opts = { + 'input': input_file, + 'output': options.filename, + 'extensions': options.extensions, + 'extension_configs': extension_configs, + 'encoding': options.encoding, + 'output_format': options.output_format, + 'lazy_ol': options.lazy_ol + } + + if options.safe: + # Avoid deprecation warning if user didn't set option + opts['safe_mode'] = options.safe + + return opts, options.verbose + + +def run(): # pragma: no cover """Run Markdown from the command line.""" # Parse options and adjust logging level if necessary options, logging_level = parse_options() - if not options: sys.exit(2) + if not options: + sys.exit(2) logger.setLevel(logging_level) - logger.addHandler(logging.StreamHandler()) + console_handler = logging.StreamHandler() + logger.addHandler(console_handler) + if logging_level <= WARNING: + # Ensure deprecation warnings get displayed + warnings.filterwarnings('default') + logging.captureWarnings(True) + warn_logger = logging.getLogger('py.warnings') + warn_logger.addHandler(console_handler) # Run markdown.markdownFromFile(**options) -if __name__ == '__main__': + +if __name__ == '__main__': # pragma: no cover # Support running module as a commandline command. - # Python 2.5 & 2.6 do: `python -m markdown.__main__ [options] [args]`. # Python 2.7 & 3.x do: `python -m markdown [options] [args]`. run() diff --git a/src/calibre/ebooks/markdown/__version__.py b/src/calibre/ebooks/markdown/__version__.py index a13559c4e3..f7c109988b 100644 --- a/src/calibre/ebooks/markdown/__version__.py +++ b/src/calibre/ebooks/markdown/__version__.py @@ -1,11 +1,12 @@ # # markdown/__version__.py # -# version_info should conform to PEP 386 +# version_info should conform to PEP 386 # (major, minor, micro, alpha/beta/rc/final, #) # (1, 1, 2, 'alpha', 0) => "1.1.2.dev" # (1, 2, 0, 'beta', 2) => "1.2b2" -version_info = (2, 3, 1, 'final', 0) +version_info = (2, 6, 3, 'final', 0) + def _get_version(): " Returns a PEP 386-compliant version number from version_info. " diff --git a/src/calibre/ebooks/markdown/blockparser.py b/src/calibre/ebooks/markdown/blockparser.py index 4504a16f51..32d3254cdf 100644 --- a/src/calibre/ebooks/markdown/blockparser.py +++ b/src/calibre/ebooks/markdown/blockparser.py @@ -3,16 +3,17 @@ from __future__ import absolute_import from . import util from . import odict + class State(list): - """ Track the current and nested state of the parser. - - This utility class is used to track the state of the BlockParser and + """ Track the current and nested state of the parser. + + This utility class is used to track the state of the BlockParser and support multiple levels if nesting. It's just a simple API wrapped around a list. Each time a state is set, that state is appended to the end of the list. Each time a state is reset, that state is removed from the end of the list. - Therefore, each time a state is set for a nested block, that state must be + Therefore, each time a state is set for a nested block, that state must be reset when we back out of that level of nesting or the state could be corrupted. @@ -36,9 +37,10 @@ class State(list): else: return False + class BlockParser: - """ Parse Markdown blocks into an ElementTree object. - + """ Parse Markdown blocks into an ElementTree object. + A wrapper class that stitches the various BlockProcessors together, looping through them and creating an ElementTree object. """ @@ -49,12 +51,12 @@ class BlockParser: self.markdown = markdown def parseDocument(self, lines): - """ Parse a markdown document into an ElementTree. - - Given a list of lines, an ElementTree object (not just a parent Element) - is created and the root element is passed to the parser as the parent. - The ElementTree object is returned. - + """ Parse a markdown document into an ElementTree. + + Given a list of lines, an ElementTree object (not just a parent + Element) is created and the root element is passed to the parser + as the parent. The ElementTree object is returned. + This should only be called on an entire document, not pieces. """ @@ -64,29 +66,30 @@ class BlockParser: return util.etree.ElementTree(self.root) def parseChunk(self, parent, text): - """ Parse a chunk of markdown text and attach to given etree node. - + """ Parse a chunk of markdown text and attach to given etree node. + While the ``text`` argument is generally assumed to contain multiple blocks which will be split on blank lines, it could contain only one block. Generally, this method would be called by extensions when - block parsing is required. - - The ``parent`` etree Element passed in is altered in place. + block parsing is required. + + The ``parent`` etree Element passed in is altered in place. Nothing is returned. """ self.parseBlocks(parent, text.split('\n\n')) def parseBlocks(self, parent, blocks): - """ Process blocks of markdown text and attach to given etree node. - + """ Process blocks of markdown text and attach to given etree node. + Given a list of ``blocks``, each blockprocessor is stepped through until there are no blocks left. While an extension could potentially - call this method directly, it's generally expected to be used internally. + call this method directly, it's generally expected to be used + internally. - This is a public method as an extension may need to add/alter additional - BlockProcessors which call this method to recursively parse a nested - block. + This is a public method as an extension may need to add/alter + additional BlockProcessors which call this method to recursively + parse a nested block. """ while blocks: @@ -95,5 +98,3 @@ class BlockParser: if processor.run(parent, blocks) is not False: # run returns True or None break - - diff --git a/src/calibre/ebooks/markdown/blockprocessors.py b/src/calibre/ebooks/markdown/blockprocessors.py index 61977b49a8..870151bec7 100644 --- a/src/calibre/ebooks/markdown/blockprocessors.py +++ b/src/calibre/ebooks/markdown/blockprocessors.py @@ -2,9 +2,9 @@ CORE MARKDOWN BLOCKPARSER =========================================================================== -This parser handles basic parsing of Markdown blocks. It doesn't concern itself -with inline elements such as **bold** or *italics*, but rather just catches -blocks, lists, quotes, etc. +This parser handles basic parsing of Markdown blocks. It doesn't concern +itself with inline elements such as **bold** or *italics*, but rather just +catches blocks, lists, quotes, etc. The BlockParser is made up of a bunch of BlockProssors, each handling a different type of block. Extensions may add/replace/remove BlockProcessors @@ -19,7 +19,7 @@ import re from . import util from .blockparser import BlockParser -logger = logging.getLogger('MARKDOWN') +logger = logging.getLogger('MARKDOWN') def build_block_parser(md_instance, **kwargs): @@ -38,9 +38,9 @@ def build_block_parser(md_instance, **kwargs): return parser -class BlockProcessor: - """ Base class for block processors. - +class BlockProcessor(object): + """ Base class for block processors. + Each subclass will provide the methods below to work with the source and tree. Each processor will need to define it's own ``test`` and ``run`` methods. The ``test`` method should return True or False, to indicate @@ -82,32 +82,32 @@ class BlockProcessor: return '\n'.join(lines) def test(self, parent, block): - """ Test for block type. Must be overridden by subclasses. - - As the parser loops through processors, it will call the ``test`` method - on each to determine if the given block of text is of that type. This - method must return a boolean ``True`` or ``False``. The actual method of - testing is left to the needs of that particular block type. It could - be as simple as ``block.startswith(some_string)`` or a complex regular - expression. As the block type may be different depending on the parent - of the block (i.e. inside a list), the parent etree element is also - provided and may be used as part of the test. + """ Test for block type. Must be overridden by subclasses. + + As the parser loops through processors, it will call the ``test`` + method on each to determine if the given block of text is of that + type. This method must return a boolean ``True`` or ``False``. The + actual method of testing is left to the needs of that particular + block type. It could be as simple as ``block.startswith(some_string)`` + or a complex regular expression. As the block type may be different + depending on the parent of the block (i.e. inside a list), the parent + etree element is also provided and may be used as part of the test. Keywords: - + * ``parent``: A etree element which will be the parent of the block. - * ``block``: A block of text from the source which has been split at + * ``block``: A block of text from the source which has been split at blank lines. """ - pass + pass # pragma: no cover def run(self, parent, blocks): - """ Run processor. Must be overridden by subclasses. - + """ Run processor. Must be overridden by subclasses. + When the parser determines the appropriate type of a block, the parser will call the corresponding processor's ``run`` method. This method should parse the individual lines of the block and append them to - the etree. + the etree. Note that both the ``parent`` and ``etree`` keywords are pointers to instances of the objects which should be edited in place. Each @@ -123,12 +123,12 @@ class BlockProcessor: * ``parent``: A etree element which is the parent of the current block. * ``blocks``: A list of all remaining blocks of the document. """ - pass + pass # pragma: no cover class ListIndentProcessor(BlockProcessor): - """ Process children of list items. - + """ Process children of list items. + Example: * a list item process this part @@ -141,17 +141,15 @@ class ListIndentProcessor(BlockProcessor): LIST_TYPES = ['ul', 'ol'] def __init__(self, *args): - BlockProcessor.__init__(self, *args) - self.INDENT_RE = re.compile(r'^(([ ]{%s})+)'% self.tab_length) + super(ListIndentProcessor, self).__init__(*args) + self.INDENT_RE = re.compile(r'^(([ ]{%s})+)' % self.tab_length) def test(self, parent, block): return block.startswith(' '*self.tab_length) and \ - not self.parser.state.isstate('detabbed') and \ - (parent.tag in self.ITEM_TYPES or \ - (len(parent) and parent[-1] and \ - (parent[-1].tag in self.LIST_TYPES) - ) - ) + not self.parser.state.isstate('detabbed') and \ + (parent.tag in self.ITEM_TYPES or + (len(parent) and parent[-1] is not None and + (parent[-1].tag in self.LIST_TYPES))) def run(self, parent, blocks): block = blocks.pop(0) @@ -162,7 +160,7 @@ class ListIndentProcessor(BlockProcessor): if parent.tag in self.ITEM_TYPES: # It's possible that this parent has a 'ul' or 'ol' child list # with a member. If that is the case, then that should be the - # parent. This is intended to catch the edge case of an indented + # parent. This is intended to catch the edge case of an indented # list whose first member was parsed previous to this point # see OListProcessor if len(parent) and parent[-1].tag in self.LIST_TYPES: @@ -193,7 +191,7 @@ class ListIndentProcessor(BlockProcessor): """ Create a new li and parse the block with it as the parent. """ li = util.etree.SubElement(parent, 'li') self.parser.parseBlocks(li, [block]) - + def get_level(self, parent, block): """ Get level of indent based on list level. """ # Get indent level @@ -211,7 +209,8 @@ class ListIndentProcessor(BlockProcessor): # Step through children of tree to find matching indent level. while indent_level > level: child = self.lastChild(parent) - if child and (child.tag in self.LIST_TYPES or child.tag in self.ITEM_TYPES): + if (child is not None and + (child.tag in self.LIST_TYPES or child.tag in self.ITEM_TYPES)): if child.tag in self.LIST_TYPES: level += 1 parent = child @@ -227,19 +226,21 @@ class CodeBlockProcessor(BlockProcessor): def test(self, parent, block): return block.startswith(' '*self.tab_length) - + def run(self, parent, blocks): sibling = self.lastChild(parent) block = blocks.pop(0) theRest = '' - if sibling and sibling.tag == "pre" and len(sibling) \ - and sibling[0].tag == "code": + if (sibling is not None and sibling.tag == "pre" and + len(sibling) and sibling[0].tag == "code"): # The previous block was a code block. As blank lines do not start # new code blocks, append this block to the previous, adding back # linebreaks removed from the split into a list. code = sibling[0] block, theRest = self.detab(block) - code.text = util.AtomicString('%s\n%s\n' % (code.text, block.rstrip())) + code.text = util.AtomicString( + '%s\n%s\n' % (code.text, block.rstrip()) + ) else: # This is a new codeblock. Create the elements and insert text. pre = util.etree.SubElement(parent, 'pre') @@ -247,7 +248,7 @@ class CodeBlockProcessor(BlockProcessor): block, theRest = self.detab(block) code.text = util.AtomicString('%s\n' % block.rstrip()) if theRest: - # This block contained unindented line(s) after the first indented + # This block contained unindented line(s) after the first indented # line. Insert these lines as the first block of the master blocks # list for future processing. blocks.insert(0, theRest) @@ -264,14 +265,15 @@ class BlockQuoteProcessor(BlockProcessor): block = blocks.pop(0) m = self.RE.search(block) if m: - before = block[:m.start()] # Lines before blockquote + before = block[:m.start()] # Lines before blockquote # Pass lines before blockquote in recursively for parsing forst. self.parser.parseBlocks(parent, [before]) # Remove ``> `` from begining of each line. - block = '\n'.join([self.clean(line) for line in - block[m.start():].split('\n')]) + block = '\n'.join( + [self.clean(line) for line in block[m.start():].split('\n')] + ) sibling = self.lastChild(parent) - if sibling and sibling.tag == "blockquote": + if sibling is not None and sibling.tag == "blockquote": # Previous block was a blockquote so set that as this blocks parent quote = sibling else: @@ -293,24 +295,30 @@ class BlockQuoteProcessor(BlockProcessor): else: return line + class OListProcessor(BlockProcessor): """ Process ordered list blocks. """ TAG = 'ol' - # Detect an item (``1. item``). ``group(1)`` contains contents of item. - RE = re.compile(r'^[ ]{0,3}\d+\.[ ]+(.*)') - # Detect items on secondary lines. they can be of either list type. - CHILD_RE = re.compile(r'^[ ]{0,3}((\d+\.)|[*+-])[ ]+(.*)') - # Detect indented (nested) items of either type - INDENT_RE = re.compile(r'^[ ]{4,7}((\d+\.)|[*+-])[ ]+.*') # The integer (python string) with which the lists starts (default=1) # Eg: If list is intialized as) # 3. Item # The ol tag will get starts="3" attribute STARTSWITH = '1' - # List of allowed sibling tags. + # List of allowed sibling tags. SIBLING_TAGS = ['ol', 'ul'] + def __init__(self, parser): + super(OListProcessor, self).__init__(parser) + # Detect an item (``1. item``). ``group(1)`` contains contents of item. + self.RE = re.compile(r'^[ ]{0,%d}\d+\.[ ]+(.*)' % (self.tab_length - 1)) + # Detect items on secondary lines. they can be of either list type. + self.CHILD_RE = re.compile(r'^[ ]{0,%d}((\d+\.)|[*+-])[ ]+(.*)' % + (self.tab_length - 1)) + # Detect indented (nested) items of either type + self.INDENT_RE = re.compile(r'^[ ]{%d,%d}((\d+\.)|[*+-])[ ]+.*' % + (self.tab_length, self.tab_length * 2 - 1)) + def test(self, parent, block): return bool(self.RE.match(block)) @@ -319,15 +327,15 @@ class OListProcessor(BlockProcessor): items = self.get_items(blocks.pop(0)) sibling = self.lastChild(parent) - if sibling and sibling.tag in self.SIBLING_TAGS: + if sibling is not None and sibling.tag in self.SIBLING_TAGS: # Previous block was a list item, so set that as parent lst = sibling - # make sure previous item is in a p- if the item has text, then it - # it isn't in a p - if lst[-1].text: - # since it's possible there are other children for this sibling, - # we can't just SubElement the p, we need to insert it as the - # first item + # make sure previous item is in a p- if the item has text, + # then it isn't in a p + if lst[-1].text: + # since it's possible there are other children for this + # sibling, we can't just SubElement the p, we need to + # insert it as the first item. p = util.etree.Element('p') p.text = lst[-1].text lst[-1].text = '' @@ -347,7 +355,7 @@ class OListProcessor(BlockProcessor): self.parser.parseBlocks(li, [firstitem]) self.parser.state.reset() elif parent.tag in ['ol', 'ul']: - # this catches the edge case of a multi-item indented list whose + # this catches the edge case of a multi-item indented list whose # first item is in a blank parent-list item: # * * subitem1 # * subitem2 @@ -357,7 +365,7 @@ class OListProcessor(BlockProcessor): # This is a new list so create parent with appropriate tag. lst = util.etree.SubElement(parent, self.TAG) # Check if a custom start integer is set - if not self.parser.markdown.lazy_ol and self.STARTSWITH !='1': + if not self.parser.markdown.lazy_ol and self.STARTSWITH != '1': lst.attrib['start'] = self.STARTSWITH self.parser.state.set('list') @@ -381,7 +389,7 @@ class OListProcessor(BlockProcessor): if m: # This is a new list item # Check first item for the start index - if not items and self.TAG=='ol': + if not items and self.TAG == 'ol': # Detect the integer value of first list item INTEGER_RE = re.compile('(\d+)') self.STARTSWITH = INTEGER_RE.match(m.group(1)).group() @@ -404,7 +412,11 @@ class UListProcessor(OListProcessor): """ Process unordered list blocks. """ TAG = 'ul' - RE = re.compile(r'^[ ]{0,3}[*+-][ ]+(.*)') + + def __init__(self, parser): + super(UListProcessor, self).__init__(parser) + # Detect an item (``1. item``). ``group(1)`` contains contents of item. + self.RE = re.compile(r'^[ ]{0,%d}[*+-][ ]+(.*)' % (self.tab_length - 1)) class HashHeaderProcessor(BlockProcessor): @@ -420,8 +432,8 @@ class HashHeaderProcessor(BlockProcessor): block = blocks.pop(0) m = self.RE.search(block) if m: - before = block[:m.start()] # All lines before header - after = block[m.end():] # All lines after header + before = block[:m.start()] # All lines before header + after = block[m.end():] # All lines after header if before: # As the header was not the first line of the block and the # lines before the header must be parsed first, @@ -433,7 +445,7 @@ class HashHeaderProcessor(BlockProcessor): if after: # Insert remaining lines as first block for future parsing. blocks.insert(0, after) - else: + else: # pragma: no cover # This should never happen, but just in case... logger.warn("We've got a problem header: %r" % block) @@ -495,7 +507,6 @@ class HRProcessor(BlockProcessor): blocks.insert(0, postlines) - class EmptyBlockProcessor(BlockProcessor): """ Process blocks that are empty or start with an empty line. """ @@ -515,9 +526,12 @@ class EmptyBlockProcessor(BlockProcessor): # Add remaining lines to master blocks for later. blocks.insert(0, theRest) sibling = self.lastChild(parent) - if sibling and sibling.tag == 'pre' and len(sibling) and sibling[0].tag == 'code': + if (sibling is not None and sibling.tag == 'pre' and + len(sibling) and sibling[0].tag == 'code'): # Last block is a codeblock. Append to preserve whitespace. - sibling[0].text = util.AtomicString('%s%s' % (sibling[0].text, filler)) + sibling[0].text = util.AtomicString( + '%s%s' % (sibling[0].text, filler) + ) class ParagraphProcessor(BlockProcessor): @@ -533,7 +547,7 @@ class ParagraphProcessor(BlockProcessor): if self.parser.state.isstate('list'): # The parent is a tight-list. # - # Check for any children. This will likely only happen in a + # Check for any children. This will likely only happen in a # tight-list when a header isn't followed by a blank line. # For example: # diff --git a/src/calibre/ebooks/markdown/extensions/__init__.py b/src/calibre/ebooks/markdown/extensions/__init__.py index 184c4d1b1d..6e7a08a1e1 100644 --- a/src/calibre/ebooks/markdown/extensions/__init__.py +++ b/src/calibre/ebooks/markdown/extensions/__init__.py @@ -4,17 +4,51 @@ Extensions """ from __future__ import unicode_literals +from ..util import parseBoolValue +import warnings + class Extension(object): """ Base class for extensions to subclass. """ - def __init__(self, configs = {}): - """Create an instance of an Extention. - Keyword arguments: + # Default config -- to be overriden by a subclass + # Must be of the following format: + # { + # 'key': ['value', 'description'] + # } + # Note that Extension.setConfig will raise a KeyError + # if a default is not set here. + config = {} - * configs: A dict of configuration setting used by an Extension. - """ - self.config = configs + def __init__(self, *args, **kwargs): + """ Initiate Extension and set up configs. """ + + # check for configs arg for backward compat. + # (there only ever used to be one so we use arg[0]) + if len(args): + if args[0] is not None: + self.setConfigs(args[0]) + warnings.warn('Extension classes accepting positional args is ' + 'pending Deprecation. Each setting should be ' + 'passed into the Class as a keyword. Positional ' + 'args are deprecated and will raise ' + 'an error in version 2.7. See the Release Notes for ' + 'Python-Markdown version 2.6 for more info.', + DeprecationWarning) + # check for configs kwarg for backward compat. + if 'configs' in kwargs.keys(): + if kwargs['configs'] is not None: + self.setConfigs(kwargs.pop('configs', {})) + warnings.warn('Extension classes accepting a dict on the single ' + 'keyword "config" is pending Deprecation. Each ' + 'setting should be passed into the Class as a ' + 'keyword directly. The "config" keyword is ' + 'deprecated and raise an error in ' + 'version 2.7. See the Release Notes for ' + 'Python-Markdown version 2.6 for more info.', + DeprecationWarning) + # finally, use kwargs + self.setConfigs(kwargs) def getConfig(self, key, default=''): """ Return a setting for the given key or an empty string. """ @@ -33,8 +67,20 @@ class Extension(object): def setConfig(self, key, value): """ Set a config setting for `key` with the given `value`. """ + if isinstance(self.config[key][0], bool): + value = parseBoolValue(value) + if self.config[key][0] is None: + value = parseBoolValue(value, preserve_none=True) self.config[key][0] = value + def setConfigs(self, items): + """ Set multiple config settings given a dict or list of tuples. """ + if hasattr(items, 'items'): + # it's a dict + items = items.items() + for key, value in items: + self.setConfig(key, value) + def extendMarkdown(self, md, md_globals): """ Add the various proccesors and patterns to the Markdown Instance. @@ -48,6 +94,7 @@ class Extension(object): * md_globals: Global variables in the markdown module namespace. """ - raise NotImplementedError('Extension "%s.%s" must define an "extendMarkdown"' \ - 'method.' % (self.__class__.__module__, self.__class__.__name__)) - + raise NotImplementedError( + 'Extension "%s.%s" must define an "extendMarkdown"' + 'method.' % (self.__class__.__module__, self.__class__.__name__) + ) diff --git a/src/calibre/ebooks/markdown/extensions/abbr.py b/src/calibre/ebooks/markdown/extensions/abbr.py index 5e46f1dc38..353d126f6f 100644 --- a/src/calibre/ebooks/markdown/extensions/abbr.py +++ b/src/calibre/ebooks/markdown/extensions/abbr.py @@ -4,22 +4,15 @@ Abbreviation Extension for Python-Markdown This extension adds abbreviation handling to Python-Markdown. -Simple Usage: +See +for documentation. - >>> import markdown - >>> text = """ - ... Some text with an ABBR and a REF. Ignore REFERENCE and ref. - ... - ... *[ABBR]: Abbreviation - ... *[REF]: Abbreviation Reference - ... """ - >>> print markdown.markdown(text, ['abbr']) -

Some text with an ABBR and a REF. Ignore REFERENCE and ref.

+Oringinal code Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/) and + [Seemant Kulleen](http://www.kulleen.org/) -Copyright 2007-2008 -* [Waylan Limberg](http://achinghead.com/) -* [Seemant Kulleen](http://www.kulleen.org/) - +All changes Copyright 2008-2014 The Python Markdown Project + +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) ''' @@ -28,20 +21,21 @@ from __future__ import unicode_literals from . import Extension from ..preprocessors import Preprocessor from ..inlinepatterns import Pattern -from ..util import etree +from ..util import etree, AtomicString import re # Global Vars ABBR_REF_RE = re.compile(r'[*]\[(?P[^\]]*)\][ ]?:\s*(?P.*)') + class AbbrExtension(Extension): """ Abbreviation Extension for Python-Markdown. """ def extendMarkdown(self, md, md_globals): """ Insert AbbrPreprocessor before ReferencePreprocessor. """ md.preprocessors.add('abbr', AbbrPreprocessor(md), '<reference') - - + + class AbbrPreprocessor(Preprocessor): """ Abbreviation Preprocessor - parse text for abbr references. """ @@ -49,7 +43,7 @@ class AbbrPreprocessor(Preprocessor): ''' Find and remove all Abbreviation references from the text. Each reference is set as a new AbbrPattern in the markdown instance. - + ''' new_text = [] for line in lines: @@ -57,19 +51,19 @@ class AbbrPreprocessor(Preprocessor): if m: abbr = m.group('abbr').strip() title = m.group('title').strip() - self.markdown.inlinePatterns['abbr-%s'%abbr] = \ + self.markdown.inlinePatterns['abbr-%s' % abbr] = \ AbbrPattern(self._generate_pattern(abbr), title) else: new_text.append(line) return new_text - + def _generate_pattern(self, text): ''' - Given a string, returns an regex pattern to match that string. - - 'HTML' -> r'(?P<abbr>[H][T][M][L])' - - Note: we force each char as a literal match (in brackets) as we don't + Given a string, returns an regex pattern to match that string. + + 'HTML' -> r'(?P<abbr>[H][T][M][L])' + + Note: we force each char as a literal match (in brackets) as we don't know what they will be beforehand. ''' @@ -88,9 +82,10 @@ class AbbrPattern(Pattern): def handleMatch(self, m): abbr = etree.Element('abbr') - abbr.text = m.group('abbr') + abbr.text = AtomicString(m.group('abbr')) abbr.set('title', self.title) return abbr -def makeExtension(configs=None): - return AbbrExtension(configs=configs) + +def makeExtension(*args, **kwargs): + return AbbrExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/admonition.py b/src/calibre/ebooks/markdown/extensions/admonition.py index 9a45b9249c..76e0fb588c 100644 --- a/src/calibre/ebooks/markdown/extensions/admonition.py +++ b/src/calibre/ebooks/markdown/extensions/admonition.py @@ -4,39 +4,16 @@ Admonition extension for Python-Markdown Adds rST-style admonitions. Inspired by [rST][] feature with the same name. -The syntax is (followed by an indented block with the contents): - !!! [type] [optional explicit title] +[rST]: http://docutils.sourceforge.net/docs/ref/rst/directives.html#specific-admonitions # noqa -Where `type` is used as a CSS class name of the div. If not present, `title` -defaults to the capitalized `type`, so "note" -> "Note". +See <https://pythonhosted.org/Markdown/extensions/admonition.html> +for documentation. -rST suggests the following `types`, but you're free to use whatever you want: - attention, caution, danger, error, hint, important, note, tip, warning +Original code Copyright [Tiago Serafim](http://www.tiagoserafim.com/). +All changes Copyright The Python Markdown Project -A simple example: - !!! note - This is the first line inside the box. - -Outputs: - <div class="admonition note"> - <p class="admonition-title">Note</p> - <p>This is the first line inside the box</p> - </div> - -You can also specify the title and CSS class of the admonition: - !!! custom "Did you know?" - Another line here. - -Outputs: - <div class="admonition custom"> - <p class="admonition-title">Did you know?</p> - <p>Another line here.</p> - </div> - -[rST]: http://docutils.sourceforge.net/docs/ref/rst/directives.html#specific-admonitions - -By [Tiago Serafim](http://www.tiagoserafim.com/). +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -69,8 +46,8 @@ class AdmonitionProcessor(BlockProcessor): def test(self, parent, block): sibling = self.lastChild(parent) return self.RE.search(block) or \ - (block.startswith(' ' * self.tab_length) and sibling and \ - sibling.get('class', '').find(self.CLASSNAME) != -1) + (block.startswith(' ' * self.tab_length) and sibling is not None and + sibling.get('class', '').find(self.CLASSNAME) != -1) def run(self, parent, blocks): sibling = self.lastChild(parent) @@ -105,7 +82,8 @@ class AdmonitionProcessor(BlockProcessor): klass, title = match.group(1).lower(), match.group(2) if title is None: # no title was provided, use the capitalized classname as title - # e.g.: `!!! note` will render `<p class="admonition-title">Note</p>` + # e.g.: `!!! note` will render + # `<p class="admonition-title">Note</p>` title = klass.capitalize() elif title == '': # an explicit blank title should not be rendered @@ -114,5 +92,5 @@ class AdmonitionProcessor(BlockProcessor): return klass, title -def makeExtension(configs={}): - return AdmonitionExtension(configs=configs) +def makeExtension(*args, **kwargs): + return AdmonitionExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/attr_list.py b/src/calibre/ebooks/markdown/extensions/attr_list.py index c98aa850a6..683bdf831c 100644 --- a/src/calibre/ebooks/markdown/extensions/attr_list.py +++ b/src/calibre/ebooks/markdown/extensions/attr_list.py @@ -2,19 +2,18 @@ Attribute List Extension for Python-Markdown ============================================ -Adds attribute list syntax. Inspired by +Adds attribute list syntax. Inspired by [maruku](http://maruku.rubyforge.org/proposal.html#attribute_lists)'s feature of the same name. -Copyright 2011 [Waylan Limberg](http://achinghead.com/). +See <https://pythonhosted.org/Markdown/extensions/attr_list.html> +for documentation. -Contact: markdown@freewisdom.org +Original code Copyright 2011 [Waylan Limberg](http://achinghead.com/). -License: BSD (see ../LICENSE.md for details) +All changes Copyright 2011-2014 The Python Markdown Project -Dependencies: -* [Python 2.4+](http://python.org) -* [Markdown 2.1+](http://packages.python.org/Markdown/) +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -27,21 +26,25 @@ import re try: Scanner = re.Scanner -except AttributeError: +except AttributeError: # pragma: no cover # must be on Python 2.4 from sre import Scanner + def _handle_double_quote(s, t): k, v = t.split('=') return k, v.strip('"') + def _handle_single_quote(s, t): k, v = t.split('=') return k, v.strip("'") -def _handle_key_value(s, t): + +def _handle_key_value(s, t): return t.split('=') + def _handle_word(s, t): if t.startswith('.'): return '.', t[1:] @@ -52,27 +55,31 @@ def _handle_word(s, t): _scanner = Scanner([ (r'[^ ]+=".*?"', _handle_double_quote), (r"[^ ]+='.*?'", _handle_single_quote), - (r'[^ ]+=[^ ]*', _handle_key_value), - (r'[^ ]+', _handle_word), + (r'[^ ]+=[^ =]+', _handle_key_value), + (r'[^ =]+', _handle_word), (r' ', None) ]) + def get_attrs(str): """ Parse attribute list and return a list of attribute tuples. """ return _scanner.scan(str)[0] + def isheader(elem): return elem.tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] + class AttrListTreeprocessor(Treeprocessor): - + BASE_RE = r'\{\:?([^\}]*)\}' - HEADER_RE = re.compile(r'[ ]*%s[ ]*$' % BASE_RE) + HEADER_RE = re.compile(r'[ ]+%s[ ]*$' % BASE_RE) BLOCK_RE = re.compile(r'\n[ ]*%s[ ]*$' % BASE_RE) INLINE_RE = re.compile(r'^%s' % BASE_RE) - NAME_RE = re.compile(r'[^A-Z_a-z\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u02ff\u0370-\u037d' - r'\u037f-\u1fff\u200c-\u200d\u2070-\u218f\u2c00-\u2fef' - r'\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd' + NAME_RE = re.compile(r'[^A-Z_a-z\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u02ff' + r'\u0370-\u037d\u037f-\u1fff\u200c-\u200d' + r'\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff' + r'\uf900-\ufdcf\ufdf0-\ufffd' r'\:\-\.0-9\u00b7\u0300-\u036f\u203f-\u2040]+') def run(self, doc): @@ -80,10 +87,36 @@ class AttrListTreeprocessor(Treeprocessor): if isBlockLevel(elem.tag): # Block level: check for attrs on last line of text RE = self.BLOCK_RE - if isheader(elem): - # header: check for attrs at end of line + if isheader(elem) or elem.tag == 'dt': + # header or def-term: check for attrs at end of line RE = self.HEADER_RE - if len(elem) and elem[-1].tail: + if len(elem) and elem.tag == 'li': + # special case list items. children may include a ul or ol. + pos = None + # find the ul or ol position + for i, child in enumerate(elem): + if child.tag in ['ul', 'ol']: + pos = i + break + if pos is None and elem[-1].tail: + # use tail of last child. no ul or ol. + m = RE.search(elem[-1].tail) + if m: + self.assign_attrs(elem, m.group(1)) + elem[-1].tail = elem[-1].tail[:m.start()] + elif pos is not None and pos > 0 and elem[pos-1].tail: + # use tail of last child before ul or ol + m = RE.search(elem[pos-1].tail) + if m: + self.assign_attrs(elem, m.group(1)) + elem[pos-1].tail = elem[pos-1].tail[:m.start()] + elif elem.text: + # use text. ul is first child. + m = RE.search(elem.text) + if m: + self.assign_attrs(elem, m.group(1)) + elem.text = elem.text[:m.start()] + elif len(elem) and elem[-1].tail: # has children. Get from tail of last child m = RE.search(elem[-1].tail) if m: @@ -95,6 +128,8 @@ class AttrListTreeprocessor(Treeprocessor): elif elem.text: # no children. Get from text. m = RE.search(elem.text) + if not m and elem.tag == 'td': + m = re.search(self.BASE_RE, elem.text) if m: self.assign_attrs(elem, m.group(1)) elem.text = elem.text[:m.start()] @@ -133,8 +168,10 @@ class AttrListTreeprocessor(Treeprocessor): class AttrListExtension(Extension): def extendMarkdown(self, md, md_globals): - md.treeprocessors.add('attr_list', AttrListTreeprocessor(md), '>prettify') + md.treeprocessors.add( + 'attr_list', AttrListTreeprocessor(md), '>prettify' + ) -def makeExtension(configs={}): - return AttrListExtension(configs=configs) +def makeExtension(*args, **kwargs): + return AttrListExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/codehilite.py b/src/calibre/ebooks/markdown/extensions/codehilite.py index 72b40fde78..0657c37681 100644 --- a/src/calibre/ebooks/markdown/extensions/codehilite.py +++ b/src/calibre/ebooks/markdown/extensions/codehilite.py @@ -4,17 +4,14 @@ CodeHilite Extension for Python-Markdown Adds code/syntax highlighting to standard Python-Markdown code blocks. -Copyright 2006-2008 [Waylan Limberg](http://achinghead.com/). +See <https://pythonhosted.org/Markdown/extensions/code_hilite.html> +for documentation. -Project website: <http://packages.python.org/Markdown/extensions/code_hilite.html> -Contact: markdown@freewisdom.org +Original code Copyright 2006-2008 [Waylan Limberg](http://achinghead.com/). -License: BSD (see ../LICENSE.md for details) +All changes Copyright 2008-2014 The Python Markdown Project -Dependencies: -* [Python 2.3+](http://python.org/) -* [Markdown 2.0+](http://packages.python.org/Markdown/) -* [Pygments](http://pygments.org/) +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -22,19 +19,35 @@ from __future__ import absolute_import from __future__ import unicode_literals from . import Extension from ..treeprocessors import Treeprocessor -import warnings + try: from pygments import highlight - from pygments.lexers import get_lexer_by_name, guess_lexer, TextLexer - from pygments.formatters import HtmlFormatter + from pygments.lexers import get_lexer_by_name, guess_lexer + from pygments.formatters import get_formatter_by_name pygments = True except ImportError: pygments = False + +def parse_hl_lines(expr): + """Support our syntax for emphasizing certain lines of code. + + expr should be like '1 2' to emphasize lines 1 and 2 of a code block. + Returns a list of ints, the line numbers to emphasize. + """ + if not expr: + return [] + + try: + return list(map(int, expr.split())) + except ValueError: + return [] + + # ------------------ The Main CodeHilite Class ---------------------- class CodeHilite(object): """ - Determine language of source code, and pass it into the pygments hilighter. + Determine language of source code, and pass it into pygments hilighter. Basic Usage: >>> code = CodeHilite(src = 'some text') @@ -42,24 +55,27 @@ class CodeHilite(object): * src: Source string or any object with a .readline attribute. - * linenums: (Boolean) Set line numbering to 'on' (True), 'off' (False) or 'auto'(None). - Set to 'auto' by default. + * linenums: (Boolean) Set line numbering to 'on' (True), + 'off' (False) or 'auto'(None). Set to 'auto' by default. - * guess_lang: (Boolean) Turn language auto-detection 'on' or 'off' (on by default). + * guess_lang: (Boolean) Turn language auto-detection + 'on' or 'off' (on by default). * css_class: Set class name of wrapper div ('codehilite' by default). + * hl_lines: (List of integers) Lines to emphasize, 1-indexed. + Low Level Usage: >>> code = CodeHilite() >>> code.src = 'some text' # String or anything with a .readline attr. - >>> code.linenos = True # True or False; Turns line numbering on or of. + >>> code.linenos = True # Turns line numbering on or of. >>> html = code.hilite() """ def __init__(self, src=None, linenums=None, guess_lang=True, - css_class="codehilite", lang=None, style='default', - noclasses=False, tab_length=4): + css_class="codehilite", lang=None, style='default', + noclasses=False, tab_length=4, hl_lines=None, use_pygments=True): self.src = src self.lang = lang self.linenums = linenums @@ -68,6 +84,8 @@ class CodeHilite(object): self.style = style self.noclasses = noclasses self.tab_length = tab_length + self.hl_lines = hl_lines or [] + self.use_pygments = use_pygments def hilite(self): """ @@ -83,9 +101,9 @@ class CodeHilite(object): self.src = self.src.strip('\n') if self.lang is None: - self._getLang() + self._parseHeader() - if pygments: + if pygments and self.use_pygments: try: lexer = get_lexer_by_name(self.lang) except ValueError: @@ -93,13 +111,15 @@ class CodeHilite(object): if self.guess_lang: lexer = guess_lexer(self.src) else: - lexer = TextLexer() + lexer = get_lexer_by_name('text') except ValueError: - lexer = TextLexer() - formatter = HtmlFormatter(linenos=self.linenums, - cssclass=self.css_class, - style=self.style, - noclasses=self.noclasses) + lexer = get_lexer_by_name('text') + formatter = get_formatter_by_name('html', + linenos=self.linenums, + cssclass=self.css_class, + style=self.style, + noclasses=self.noclasses, + hl_lines=self.hl_lines) return highlight(self.src, lexer, formatter) else: # just escape and build markup usable by JS highlighting libs @@ -114,36 +134,42 @@ class CodeHilite(object): classes.append('linenums') class_str = '' if classes: - class_str = ' class="%s"' % ' '.join(classes) - return '<pre class="%s"><code%s>%s</code></pre>\n'% \ - (self.css_class, class_str, txt) + class_str = ' class="%s"' % ' '.join(classes) + return '<pre class="%s"><code%s>%s</code></pre>\n' % \ + (self.css_class, class_str, txt) - def _getLang(self): + def _parseHeader(self): """ Determines language of a code block from shebang line and whether said line should be removed or left in place. If the sheband line contains a path (even a single /) then it is assumed to be a real shebang line and left alone. However, if no path is given (e.i.: #!python or :::python) - then it is assumed to be a mock shebang for language identifitation of a - code fragment and removed from the code block prior to processing for + then it is assumed to be a mock shebang for language identifitation of + a code fragment and removed from the code block prior to processing for code highlighting. When a mock shebang (e.i: #!python) is found, line numbering is turned on. When colons are found in place of a shebang (e.i.: :::python), line numbering is left in the current state - off by default. + Also parses optional list of highlight lines, like: + + :::python hl_lines="1 3" """ import re - #split text into lines + # split text into lines lines = self.src.split("\n") - #pull first line to examine + # pull first line to examine fl = lines.pop(0) c = re.compile(r''' - (?:(?:^::+)|(?P<shebang>^[#]!)) # Shebang or 2 or more colons. + (?:(?:^::+)|(?P<shebang>^[#]!)) # Shebang or 2 or more colons (?P<path>(?:/\w+)*[/ ])? # Zero or 1 path (?P<lang>[\w+-]*) # The language + \s* # Arbitrary whitespace + # Optional highlight lines, single- or double-quote-delimited + (hl_lines=(?P<quot>"|')(?P<hl_lines>.*?)(?P=quot))? ''', re.VERBOSE) # search first line for shebang m = c.search(fl) @@ -159,6 +185,8 @@ class CodeHilite(object): if self.linenums is None and m.group('shebang'): # Overridable and Shebang exists - use line numbers self.linenums = True + + self.hl_lines = parse_hl_lines(m.group('hl_lines')) else: # No match lines.insert(0, fl) @@ -166,24 +194,27 @@ class CodeHilite(object): self.src = "\n".join(lines).strip("\n") - # ------------------ The Markdown Extension ------------------------------- + + class HiliteTreeprocessor(Treeprocessor): """ Hilight source code in code blocks. """ def run(self, root): """ Find code blocks and store in htmlStash. """ - blocks = root.getiterator('pre') + blocks = root.iter('pre') for block in blocks: - children = block.getchildren() - if len(children) == 1 and children[0].tag == 'code': - code = CodeHilite(children[0].text, - linenums=self.config['linenums'], - guess_lang=self.config['guess_lang'], - css_class=self.config['css_class'], - style=self.config['pygments_style'], - noclasses=self.config['noclasses'], - tab_length=self.markdown.tab_length) + if len(block) == 1 and block[0].tag == 'code': + code = CodeHilite( + block[0].text, + linenums=self.config['linenums'], + guess_lang=self.config['guess_lang'], + css_class=self.config['css_class'], + style=self.config['pygments_style'], + noclasses=self.config['noclasses'], + tab_length=self.markdown.tab_length, + use_pygments=self.config['use_pygments'] + ) placeholder = self.markdown.htmlStash.store(code.hilite(), safe=True) # Clear codeblock in etree instance @@ -197,34 +228,29 @@ class HiliteTreeprocessor(Treeprocessor): class CodeHiliteExtension(Extension): """ Add source code hilighting to markdown codeblocks. """ - def __init__(self, configs): + def __init__(self, *args, **kwargs): # define default configs self.config = { - 'linenums': [None, "Use lines numbers. True=yes, False=no, None=auto"], - 'force_linenos' : [False, "Depreciated! Use 'linenums' instead. Force line numbers - Default: False"], - 'guess_lang' : [True, "Automatic language detection - Default: True"], - 'css_class' : ["codehilite", - "Set class name for wrapper <div> - Default: codehilite"], - 'pygments_style' : ['default', 'Pygments HTML Formatter Style (Colorscheme) - Default: default'], - 'noclasses': [False, 'Use inline styles instead of CSS classes - Default false'] + 'linenums': [None, + "Use lines numbers. True=yes, False=no, None=auto"], + 'guess_lang': [True, + "Automatic language detection - Default: True"], + 'css_class': ["codehilite", + "Set class name for wrapper <div> - " + "Default: codehilite"], + 'pygments_style': ['default', + 'Pygments HTML Formatter Style ' + '(Colorscheme) - Default: default'], + 'noclasses': [False, + 'Use inline styles instead of CSS classes - ' + 'Default false'], + 'use_pygments': [True, + 'Use Pygments to Highlight code blocks. ' + 'Disable if using a JavaScript library. ' + 'Default: True'] } - # Override defaults with user settings - for key, value in configs: - # convert strings to booleans - if value == 'True': value = True - if value == 'False': value = False - if value == 'None': value = None - - if key == 'force_linenos': - warnings.warn('The "force_linenos" config setting' - ' to the CodeHilite extension is deprecrecated.' - ' Use "linenums" instead.', PendingDeprecationWarning) - if value: - # Carry 'force_linenos' over to new 'linenos'. - self.setConfig('linenums', True) - - self.setConfig(key, value) + super(CodeHiliteExtension, self).__init__(*args, **kwargs) def extendMarkdown(self, md, md_globals): """ Add HilitePostprocessor to Markdown instance. """ @@ -235,6 +261,5 @@ class CodeHiliteExtension(Extension): md.registerExtension(self) -def makeExtension(configs={}): - return CodeHiliteExtension(configs=configs) - +def makeExtension(*args, **kwargs): + return CodeHiliteExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/def_list.py b/src/calibre/ebooks/markdown/extensions/def_list.py index 868465266c..77cca6eb8b 100644 --- a/src/calibre/ebooks/markdown/extensions/def_list.py +++ b/src/calibre/ebooks/markdown/extensions/def_list.py @@ -2,19 +2,16 @@ Definition List Extension for Python-Markdown ============================================= -Added parsing of Definition Lists to Python-Markdown. +Adds parsing of Definition Lists to Python-Markdown. -A simple example: +See <https://pythonhosted.org/Markdown/extensions/definition_lists.html> +for documentation. - Apple - : Pomaceous fruit of plants of the genus Malus in - the family Rosaceae. - : An american computer company. +Original code Copyright 2008 [Waylan Limberg](http://achinghead.com) - Orange - : The fruit of an evergreen tree of the genus Citrus. +All changes Copyright 2008-2014 The Python Markdown Project -Copyright 2008 - [Waylan Limberg](http://achinghead.com) +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -39,7 +36,8 @@ class DefListProcessor(BlockProcessor): raw_block = blocks.pop(0) m = self.RE.search(raw_block) - terms = [l.strip() for l in raw_block[:m.start()].split('\n') if l.strip()] + terms = [l.strip() for l in + raw_block[:m.start()].split('\n') if l.strip()] block = raw_block[m.end():] no_indent = self.NO_INDENT_RE.match(block) if no_indent: @@ -52,7 +50,7 @@ class DefListProcessor(BlockProcessor): d = m.group(2) sibling = self.lastChild(parent) if not terms and sibling is None: - # This is not a definition item. Most likely a paragraph that + # This is not a definition item. Most likely a paragraph that # starts with a colon at the begining of a document or list. blocks.insert(0, raw_block) return False @@ -66,10 +64,10 @@ class DefListProcessor(BlockProcessor): else: state = 'list' - if sibling and sibling.tag == 'dl': + if sibling is not None and sibling.tag == 'dl': # This is another item on an existing list dl = sibling - if len(dl) and dl[-1].tag == 'dd' and len(dl[-1]): + if not terms and len(dl) and dl[-1].tag == 'dd' and len(dl[-1]): state = 'looselist' else: # This is a new list @@ -87,6 +85,7 @@ class DefListProcessor(BlockProcessor): if theRest: blocks.insert(0, theRest) + class DefListIndentProcessor(ListIndentProcessor): """ Process indented children of definition list items. """ @@ -97,7 +96,6 @@ class DefListIndentProcessor(ListIndentProcessor): """ Create a new dd and parse the block with it as the parent. """ dd = etree.SubElement(parent, 'dd') self.parser.parseBlocks(dd, [block]) - class DefListExtension(Extension): @@ -108,11 +106,10 @@ class DefListExtension(Extension): md.parser.blockprocessors.add('defindent', DefListIndentProcessor(md.parser), '>indent') - md.parser.blockprocessors.add('deflist', + md.parser.blockprocessors.add('deflist', DefListProcessor(md.parser), '>ulist') -def makeExtension(configs={}): - return DefListExtension(configs=configs) - +def makeExtension(*args, **kwargs): + return DefListExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/extra.py b/src/calibre/ebooks/markdown/extensions/extra.py index e6a1e820ef..de5db03cd6 100644 --- a/src/calibre/ebooks/markdown/extensions/extra.py +++ b/src/calibre/ebooks/markdown/extensions/extra.py @@ -6,49 +6,127 @@ A compilation of various Python-Markdown extensions that imitates [PHP Markdown Extra](http://michelf.com/projects/php-markdown/extra/). Note that each of the individual extensions still need to be available -on your PYTHONPATH. This extension simply wraps them all up as a +on your PYTHONPATH. This extension simply wraps them all up as a convenience so that only one extension needs to be listed when initiating Markdown. See the documentation for each individual extension for specifics about that extension. -In the event that one or more of the supported extensions are not -available for import, Markdown will issue a warning and simply continue -without that extension. - -There may be additional extensions that are distributed with +There may be additional extensions that are distributed with Python-Markdown that are not included here in Extra. Those extensions are not part of PHP Markdown Extra, and therefore, not part of Python-Markdown Extra. If you really would like Extra to include additional extensions, we suggest creating your own clone of Extra -under a differant name. You could also edit the `extensions` global -variable defined below, but be aware that such changes may be lost +under a differant name. You could also edit the `extensions` global +variable defined below, but be aware that such changes may be lost when you upgrade to any future version of Python-Markdown. +See <https://pythonhosted.org/Markdown/extensions/extra.html> +for documentation. + +Copyright The Python Markdown Project + +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) + """ from __future__ import absolute_import from __future__ import unicode_literals from . import Extension +from ..blockprocessors import BlockProcessor +from .. import util +import re + +extensions = [ + 'markdown.extensions.smart_strong', + 'markdown.extensions.fenced_code', + 'markdown.extensions.footnotes', + 'markdown.extensions.attr_list', + 'markdown.extensions.def_list', + 'markdown.extensions.tables', + 'markdown.extensions.abbr' +] -extensions = ['smart_strong', - 'fenced_code', - 'footnotes', - 'attr_list', - 'def_list', - 'tables', - 'abbr', - ] - class ExtraExtension(Extension): """ Add various extensions to Markdown class.""" + def __init__(self, *args, **kwargs): + """ config is a dumb holder which gets passed to actual ext later. """ + self.config = kwargs.pop('configs', {}) + self.config.update(kwargs) + def extendMarkdown(self, md, md_globals): """ Register extension instances. """ md.registerExtensions(extensions, self.config) if not md.safeMode: # Turn on processing of markdown text within raw html md.preprocessors['html_block'].markdown_in_raw = True + md.parser.blockprocessors.add('markdown_block', + MarkdownInHtmlProcessor(md.parser), + '_begin') + md.parser.blockprocessors.tag_counter = -1 + md.parser.blockprocessors.contain_span_tags = re.compile( + r'^(p|h[1-6]|li|dd|dt|td|th|legend|address)$', re.IGNORECASE) -def makeExtension(configs={}): - return ExtraExtension(configs=dict(configs)) + +def makeExtension(*args, **kwargs): + return ExtraExtension(*args, **kwargs) + + +class MarkdownInHtmlProcessor(BlockProcessor): + """Process Markdown Inside HTML Blocks.""" + def test(self, parent, block): + return block == util.TAG_PLACEHOLDER % \ + str(self.parser.blockprocessors.tag_counter + 1) + + def _process_nests(self, element, block): + """Process the element's child elements in self.run.""" + # Build list of indexes of each nest within the parent element. + nest_index = [] # a list of tuples: (left index, right index) + i = self.parser.blockprocessors.tag_counter + 1 + while len(self._tag_data) > i and self._tag_data[i]['left_index']: + left_child_index = self._tag_data[i]['left_index'] + right_child_index = self._tag_data[i]['right_index'] + nest_index.append((left_child_index - 1, right_child_index)) + i += 1 + + # Create each nest subelement. + for i, (left_index, right_index) in enumerate(nest_index[:-1]): + self.run(element, block[left_index:right_index], + block[right_index:nest_index[i + 1][0]], True) + self.run(element, block[nest_index[-1][0]:nest_index[-1][1]], # last + block[nest_index[-1][1]:], True) # nest + + def run(self, parent, blocks, tail=None, nest=False): + self._tag_data = self.parser.markdown.htmlStash.tag_data + + self.parser.blockprocessors.tag_counter += 1 + tag = self._tag_data[self.parser.blockprocessors.tag_counter] + + # Create Element + markdown_value = tag['attrs'].pop('markdown') + element = util.etree.SubElement(parent, tag['tag'], tag['attrs']) + + # Slice Off Block + if nest: + self.parser.parseBlocks(parent, tail) # Process Tail + block = blocks[1:] + else: # includes nests since a third level of nesting isn't supported + block = blocks[tag['left_index'] + 1: tag['right_index']] + del blocks[:tag['right_index']] + + # Process Text + if (self.parser.blockprocessors.contain_span_tags.match( # Span Mode + tag['tag']) and markdown_value != 'block') or \ + markdown_value == 'span': + element.text = '\n'.join(block) + else: # Block Mode + i = self.parser.blockprocessors.tag_counter + 1 + if len(self._tag_data) > i and self._tag_data[i]['left_index']: + first_subelement_index = self._tag_data[i]['left_index'] - 1 + self.parser.parseBlocks( + element, block[:first_subelement_index]) + if not nest: + block = self._process_nests(element, block) + else: + self.parser.parseBlocks(element, block) diff --git a/src/calibre/ebooks/markdown/extensions/fenced_code.py b/src/calibre/ebooks/markdown/extensions/fenced_code.py index ecdb20d439..4af8891a8b 100644 --- a/src/calibre/ebooks/markdown/extensions/fenced_code.py +++ b/src/calibre/ebooks/markdown/extensions/fenced_code.py @@ -4,89 +4,24 @@ Fenced Code Extension for Python Markdown This extension adds Fenced Code Blocks to Python-Markdown. - >>> import markdown - >>> text = ''' - ... A paragraph before a fenced code block: - ... - ... ~~~ - ... Fenced code block - ... ~~~ - ... ''' - >>> html = markdown.markdown(text, extensions=['fenced_code']) - >>> print html - <p>A paragraph before a fenced code block:</p> - <pre><code>Fenced code block - </code></pre> +See <https://pythonhosted.org/Markdown/extensions/fenced_code_blocks.html> +for documentation. -Works with safe_mode also (we check this because we are using the HtmlStash): +Original code Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/). - >>> print markdown.markdown(text, extensions=['fenced_code'], safe_mode='replace') - <p>A paragraph before a fenced code block:</p> - <pre><code>Fenced code block - </code></pre> -Include tilde's in a code block and wrap with blank lines: - - >>> text = ''' - ... ~~~~~~~~ - ... - ... ~~~~ - ... ~~~~~~~~''' - >>> print markdown.markdown(text, extensions=['fenced_code']) - <pre><code> - ~~~~ - </code></pre> - -Language tags: - - >>> text = ''' - ... ~~~~{.python} - ... # Some python code - ... ~~~~''' - >>> print markdown.markdown(text, extensions=['fenced_code']) - <pre><code class="python"># Some python code - </code></pre> - -Optionally backticks instead of tildes as per how github's code block markdown is identified: - - >>> text = ''' - ... ````` - ... # Arbitrary code - ... ~~~~~ # these tildes will not close the block - ... `````''' - >>> print markdown.markdown(text, extensions=['fenced_code']) - <pre><code># Arbitrary code - ~~~~~ # these tildes will not close the block - </code></pre> - -Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/). - -Project website: <http://packages.python.org/Markdown/extensions/fenced_code_blocks.html> -Contact: markdown@freewisdom.org - -License: BSD (see ../docs/LICENSE for details) - -Dependencies: -* [Python 2.4+](http://python.org) -* [Markdown 2.0+](http://packages.python.org/Markdown/) -* [Pygments (optional)](http://pygments.org) +All changes Copyright 2008-2014 The Python Markdown Project +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ from __future__ import absolute_import from __future__ import unicode_literals from . import Extension from ..preprocessors import Preprocessor -from .codehilite import CodeHilite, CodeHiliteExtension +from .codehilite import CodeHilite, CodeHiliteExtension, parse_hl_lines import re -# Global vars -FENCED_BLOCK_RE = re.compile( \ - r'(?P<fence>^(?:~{3,}|`{3,}))[ ]*(\{?\.?(?P<lang>[a-zA-Z0-9_+-]*)\}?)?[ ]*\n(?P<code>.*?)(?<=\n)(?P=fence)[ ]*$', - re.MULTILINE|re.DOTALL - ) -CODE_WRAP = '<pre><code%s>%s</code></pre>' -LANG_TAG = ' class="%s"' class FencedCodeExtension(Extension): @@ -95,11 +30,21 @@ class FencedCodeExtension(Extension): md.registerExtension(self) md.preprocessors.add('fenced_code_block', - FencedBlockPreprocessor(md), - ">normalize_whitespace") + FencedBlockPreprocessor(md), + ">normalize_whitespace") class FencedBlockPreprocessor(Preprocessor): + FENCED_BLOCK_RE = re.compile(r''' +(?P<fence>^(?:~{3,}|`{3,}))[ ]* # Opening ``` or ~~~ +(\{?\.?(?P<lang>[a-zA-Z0-9_+-]*))?[ ]* # Optional {, and lang +# Optional highlight lines, single- or double-quote-delimited +(hl_lines=(?P<quot>"|')(?P<hl_lines>.*?)(?P=quot))?[ ]* +}?[ ]*\n # Optional closing } +(?P<code>.*?)(?<=\n) +(?P=fence)[ ]*$''', re.MULTILINE | re.DOTALL | re.VERBOSE) + CODE_WRAP = '<pre><code%s>%s</code></pre>' + LANG_TAG = ' class="%s"' def __init__(self, md): super(FencedBlockPreprocessor, self).__init__(md) @@ -121,29 +66,35 @@ class FencedBlockPreprocessor(Preprocessor): text = "\n".join(lines) while 1: - m = FENCED_BLOCK_RE.search(text) + m = self.FENCED_BLOCK_RE.search(text) if m: lang = '' if m.group('lang'): - lang = LANG_TAG % m.group('lang') + lang = self.LANG_TAG % m.group('lang') # If config is not empty, then the codehighlite extension - # is enabled, so we call it to highlite the code + # is enabled, so we call it to highlight the code if self.codehilite_conf: - highliter = CodeHilite(m.group('code'), - linenums=self.codehilite_conf['linenums'][0], - guess_lang=self.codehilite_conf['guess_lang'][0], - css_class=self.codehilite_conf['css_class'][0], - style=self.codehilite_conf['pygments_style'][0], - lang=(m.group('lang') or None), - noclasses=self.codehilite_conf['noclasses'][0]) + highliter = CodeHilite( + m.group('code'), + linenums=self.codehilite_conf['linenums'][0], + guess_lang=self.codehilite_conf['guess_lang'][0], + css_class=self.codehilite_conf['css_class'][0], + style=self.codehilite_conf['pygments_style'][0], + lang=(m.group('lang') or None), + noclasses=self.codehilite_conf['noclasses'][0], + hl_lines=parse_hl_lines(m.group('hl_lines')) + ) code = highliter.hilite() else: - code = CODE_WRAP % (lang, self._escape(m.group('code'))) + code = self.CODE_WRAP % (lang, + self._escape(m.group('code'))) placeholder = self.markdown.htmlStash.store(code, safe=True) - text = '%s\n%s\n%s'% (text[:m.start()], placeholder, text[m.end():]) + text = '%s\n%s\n%s' % (text[:m.start()], + placeholder, + text[m.end():]) else: break return text.split("\n") @@ -157,5 +108,5 @@ class FencedBlockPreprocessor(Preprocessor): return txt -def makeExtension(configs=None): - return FencedCodeExtension(configs=configs) +def makeExtension(*args, **kwargs): + return FencedCodeExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/footnotes.py b/src/calibre/ebooks/markdown/extensions/footnotes.py index 65ed597a7b..b52815f3c8 100644 --- a/src/calibre/ebooks/markdown/extensions/footnotes.py +++ b/src/calibre/ebooks/markdown/extensions/footnotes.py @@ -1,25 +1,15 @@ """ -========================= FOOTNOTES ================================= +Footnotes Extension for Python-Markdown +======================================= -This section adds footnote handling to markdown. It can be used as -an example for extending python-markdown with relatively complex -functionality. While in this case the extension is included inside -the module itself, it could just as easily be added from outside the -module. Not that all markdown classes above are ignorant about -footnotes. All footnote functionality is provided separately and -then added to the markdown instance at the run time. +Adds footnote handling to Python-Markdown. -Footnote functionality is attached by calling extendMarkdown() -method of FootnoteExtension. The method also registers the -extension to allow it's state to be reset by a call to reset() -method. +See <https://pythonhosted.org/Markdown/extensions/footnotes.html> +for documentation. -Example: - Footnotes[^1] have a label[^label] and a definition[^!DEF]. +Copyright The Python Markdown Project - [^1]: This is a footnote - [^label]: A footnote on "label" - [^!DEF]: The footnote for definition +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -35,29 +25,31 @@ from ..odict import OrderedDict import re FN_BACKLINK_TEXT = "zz1337820767766393qq" -NBSP_PLACEHOLDER = "qq3936677670287331zz" +NBSP_PLACEHOLDER = "qq3936677670287331zz" DEF_RE = re.compile(r'[ ]{0,3}\[\^([^\]]*)\]:\s*(.*)') TABBED_RE = re.compile(r'((\t)|( ))(.*)') + class FootnoteExtension(Extension): """ Footnote Extension. """ - def __init__ (self, configs): + def __init__(self, *args, **kwargs): """ Setup configs. """ - self.config = {'PLACE_MARKER': - ["///Footnotes Go Here///", - "The text string that marks where the footnotes go"], - 'UNIQUE_IDS': - [False, - "Avoid name collisions across " - "multiple calls to reset()."], - "BACKLINK_TEXT": - ["↩", - "The text string that links from the footnote to the reader's place."] - } - for key, value in configs: - self.config[key][0] = value + self.config = { + 'PLACE_MARKER': + ["///Footnotes Go Here///", + "The text string that marks where the footnotes go"], + 'UNIQUE_IDS': + [False, + "Avoid name collisions across " + "multiple calls to reset()."], + "BACKLINK_TEXT": + ["↩", + "The text string that links from the footnote " + "to the reader's place."] + } + super(FootnoteExtension, self).__init__(*args, **kwargs) # In multiple invocations, emit links that don't get tangled. self.unique_prefix = 0 @@ -69,27 +61,28 @@ class FootnoteExtension(Extension): md.registerExtension(self) self.parser = md.parser self.md = md - self.sep = ':' - if self.md.output_format in ['html5', 'xhtml5']: - self.sep = '-' # Insert a preprocessor before ReferencePreprocessor - md.preprocessors.add("footnote", FootnotePreprocessor(self), - "<reference") + md.preprocessors.add( + "footnote", FootnotePreprocessor(self), "<reference" + ) # Insert an inline pattern before ImageReferencePattern - FOOTNOTE_RE = r'\[\^([^\]]*)\]' # blah blah [^1] blah - md.inlinePatterns.add("footnote", FootnotePattern(FOOTNOTE_RE, self), - "<reference") + FOOTNOTE_RE = r'\[\^([^\]]*)\]' # blah blah [^1] blah + md.inlinePatterns.add( + "footnote", FootnotePattern(FOOTNOTE_RE, self), "<reference" + ) # Insert a tree-processor that would actually add the footnote div - # This must be before all other treeprocessors (i.e., inline and + # This must be before all other treeprocessors (i.e., inline and # codehilite) so they can run on the the contents of the div. - md.treeprocessors.add("footnote", FootnoteTreeprocessor(self), - "_begin") + md.treeprocessors.add( + "footnote", FootnoteTreeprocessor(self), "_begin" + ) # Insert a postprocessor after amp_substitute oricessor - md.postprocessors.add("footnote", FootnotePostprocessor(self), - ">amp_substitute") + md.postprocessors.add( + "footnote", FootnotePostprocessor(self), ">amp_substitute" + ) def reset(self): - """ Clear the footnotes on reset, and prepare for a distinct document. """ + """ Clear footnotes on reset, and prepare for distinct document. """ self.footnotes = OrderedDict() self.unique_prefix += 1 @@ -103,9 +96,11 @@ class FootnoteExtension(Extension): if child.tail: if child.tail.find(self.getConfig("PLACE_MARKER")) > -1: return child, element, False - finder(child) + child_res = finder(child) + if child_res is not None: + return child_res return None - + res = finder(root) return res @@ -113,19 +108,25 @@ class FootnoteExtension(Extension): """ Store a footnote for later retrieval. """ self.footnotes[id] = text + def get_separator(self): + if self.md.output_format in ['html5', 'xhtml5']: + return '-' + return ':' + def makeFootnoteId(self, id): """ Return footnote link id. """ if self.getConfig("UNIQUE_IDS"): - return 'fn%s%d-%s' % (self.sep, self.unique_prefix, id) + return 'fn%s%d-%s' % (self.get_separator(), self.unique_prefix, id) else: - return 'fn%s%s' % (self.sep, id) + return 'fn%s%s' % (self.get_separator(), id) def makeFootnoteRefId(self, id): """ Return footnote back-link id. """ if self.getConfig("UNIQUE_IDS"): - return 'fnref%s%d-%s' % (self.sep, self.unique_prefix, id) + return 'fnref%s%d-%s' % (self.get_separator(), + self.unique_prefix, id) else: - return 'fnref%s%s' % (self.sep, id) + return 'fnref%s%s' % (self.get_separator(), id) def makeFootnotesDiv(self, root): """ Return div of footnotes as et Element. """ @@ -145,10 +146,13 @@ class FootnoteExtension(Extension): backlink = etree.Element("a") backlink.set("href", "#" + self.makeFootnoteRefId(id)) if self.md.output_format not in ['html5', 'xhtml5']: - backlink.set("rev", "footnote") # Invalid in HTML5 + backlink.set("rev", "footnote") # Invalid in HTML5 backlink.set("class", "footnote-backref") - backlink.set("title", "Jump back to footnote %d in the text" % \ - (self.footnotes.index(id)+1)) + backlink.set( + "title", + "Jump back to footnote %d in the text" % + (self.footnotes.index(id)+1) + ) backlink.text = FN_BACKLINK_TEXT if li.getchildren(): @@ -165,7 +169,7 @@ class FootnoteExtension(Extension): class FootnotePreprocessor(Preprocessor): """ Find all footnote references and store for later use. """ - def __init__ (self, footnotes): + def __init__(self, footnotes): self.footnotes = footnotes def run(self, lines): @@ -186,7 +190,7 @@ class FootnotePreprocessor(Preprocessor): if m: fn, _i = self.detectTabbed(lines[i+1:]) fn.insert(0, m.group(2)) - i += _i-1 # skip past footnote + i += _i-1 # skip past footnote self.footnotes.setFootnote(m.group(1), "\n".join(fn)) else: newlines.append(lines[i]) @@ -207,16 +211,16 @@ class FootnotePreprocessor(Preprocessor): """ items = [] - blank_line = False # have we encountered a blank line yet? - i = 0 # to keep track of where we are + blank_line = False # have we encountered a blank line yet? + i = 0 # to keep track of where we are def detab(line): match = TABBED_RE.match(line) if match: - return match.group(4) + return match.group(4) for line in lines: - if line.strip(): # Non-blank line + if line.strip(): # Non-blank line detabbed_line = detab(line) if detabbed_line: items.append(detabbed_line) @@ -230,23 +234,24 @@ class FootnotePreprocessor(Preprocessor): else: return items, i+1 - else: # Blank line: _maybe_ we are done. + else: # Blank line: _maybe_ we are done. blank_line = True - i += 1 # advance + i += 1 # advance # Find the next non-blank line for j in range(i, len(lines)): if lines[j].strip(): - next_line = lines[j]; break + next_line = lines[j] + break else: - break # There is no more text; we are done. + break # There is no more text; we are done. # Check if the next non-blank line is tabbed - if detab(next_line): # Yes, more work to do. + if detab(next_line): # Yes, more work to do. items.append("") continue else: - break # No, we are done. + break # No, we are done. else: i += 1 @@ -268,7 +273,7 @@ class FootnotePattern(Pattern): sup.set('id', self.footnotes.makeFootnoteRefId(id)) a.set('href', '#' + self.footnotes.makeFootnoteId(id)) if self.footnotes.md.output_format not in ['html5', 'xhtml5']: - a.set('rel', 'footnote') # invalid in HTML5 + a.set('rel', 'footnote') # invalid in HTML5 a.set('class', 'footnote-ref') a.text = text_type(self.footnotes.footnotes.index(id) + 1) return sup @@ -279,12 +284,12 @@ class FootnotePattern(Pattern): class FootnoteTreeprocessor(Treeprocessor): """ Build and append footnote div to end of document. """ - def __init__ (self, footnotes): + def __init__(self, footnotes): self.footnotes = footnotes def run(self, root): footnotesDiv = self.footnotes.makeFootnotesDiv(root) - if footnotesDiv: + if footnotesDiv is not None: result = self.footnotes.findFootnotesPlaceholder(root) if result: child, parent, isText = result @@ -298,16 +303,19 @@ class FootnoteTreeprocessor(Treeprocessor): else: root.append(footnotesDiv) + class FootnotePostprocessor(Postprocessor): """ Replace placeholders with html entities. """ def __init__(self, footnotes): self.footnotes = footnotes def run(self, text): - text = text.replace(FN_BACKLINK_TEXT, self.footnotes.getConfig("BACKLINK_TEXT")) + text = text.replace( + FN_BACKLINK_TEXT, self.footnotes.getConfig("BACKLINK_TEXT") + ) return text.replace(NBSP_PLACEHOLDER, " ") -def makeExtension(configs=[]): - """ Return an instance of the FootnoteExtension """ - return FootnoteExtension(configs=configs) +def makeExtension(*args, **kwargs): + """ Return an instance of the FootnoteExtension """ + return FootnoteExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/headerid.py b/src/calibre/ebooks/markdown/extensions/headerid.py index 7681b8d499..2cb20b97ab 100644 --- a/src/calibre/ebooks/markdown/extensions/headerid.py +++ b/src/calibre/ebooks/markdown/extensions/headerid.py @@ -4,73 +4,14 @@ HeaderID Extension for Python-Markdown Auto-generate id attributes for HTML headers. -Basic usage: +See <https://pythonhosted.org/Markdown/extensions/header_id.html> +for documentation. - >>> import markdown - >>> text = "# Some Header #" - >>> md = markdown.markdown(text, ['headerid']) - >>> print md - <h1 id="some-header">Some Header</h1> +Original code Copyright 2007-2011 [Waylan Limberg](http://achinghead.com/). -All header IDs are unique: +All changes Copyright 2011-2014 The Python Markdown Project - >>> text = ''' - ... #Header - ... #Header - ... #Header''' - >>> md = markdown.markdown(text, ['headerid']) - >>> print md - <h1 id="header">Header</h1> - <h1 id="header_1">Header</h1> - <h1 id="header_2">Header</h1> - -To fit within a html template's hierarchy, set the header base level: - - >>> text = ''' - ... #Some Header - ... ## Next Level''' - >>> md = markdown.markdown(text, ['headerid(level=3)']) - >>> print md - <h3 id="some-header">Some Header</h3> - <h4 id="next-level">Next Level</h4> - -Works with inline markup. - - >>> text = '#Some *Header* with [markup](http://example.com).' - >>> md = markdown.markdown(text, ['headerid']) - >>> print md - <h1 id="some-header-with-markup">Some <em>Header</em> with <a href="http://example.com">markup</a>.</h1> - -Turn off auto generated IDs: - - >>> text = ''' - ... # Some Header - ... # Another Header''' - >>> md = markdown.markdown(text, ['headerid(forceid=False)']) - >>> print md - <h1>Some Header</h1> - <h1>Another Header</h1> - -Use with MetaData extension: - - >>> text = '''header_level: 2 - ... header_forceid: Off - ... - ... # A Header''' - >>> md = markdown.markdown(text, ['headerid', 'meta']) - >>> print md - <h2>A Header</h2> - -Copyright 2007-2011 [Waylan Limberg](http://achinghead.com/). - -Project website: <http://packages.python.org/Markdown/extensions/header_id.html> -Contact: markdown@freewisdom.org - -License: BSD (see ../docs/LICENSE for details) - -Dependencies: -* [Python 2.3+](http://python.org) -* [Markdown 2.0+](http://packages.python.org/Markdown/) +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -78,47 +19,9 @@ from __future__ import absolute_import from __future__ import unicode_literals from . import Extension from ..treeprocessors import Treeprocessor -import re -import logging -import unicodedata - -logger = logging.getLogger('MARKDOWN') - -IDCOUNT_RE = re.compile(r'^(.*)_([0-9]+)$') - - -def slugify(value, separator): - """ Slugify a string, to make it URL friendly. """ - value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') - value = re.sub('[^\w\s-]', '', value.decode('ascii')).strip().lower() - return re.sub('[%s\s]+' % separator, separator, value) - - -def unique(id, ids): - """ Ensure id is unique in set of ids. Append '_1', '_2'... if not """ - while id in ids or not id: - m = IDCOUNT_RE.match(id) - if m: - id = '%s_%d'% (m.group(1), int(m.group(2))+1) - else: - id = '%s_%d'% (id, 1) - ids.add(id) - return id - - -def itertext(elem): - """ Loop through all children and return text only. - - Reimplements method of same name added to ElementTree in Python 2.7 - - """ - if elem.text: - yield elem.text - for e in elem: - for s in itertext(e): - yield s - if e.tail: - yield e.tail +from ..util import parseBoolValue +from .toc import slugify, unique, stashedHTML2text +import warnings class HeaderIdTreeprocessor(Treeprocessor): @@ -130,13 +33,14 @@ class HeaderIdTreeprocessor(Treeprocessor): start_level, force_id = self._get_meta() slugify = self.config['slugify'] sep = self.config['separator'] - for elem in doc.getiterator(): + for elem in doc: if elem.tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']: if force_id: if "id" in elem.attrib: id = elem.get('id') else: - id = slugify(''.join(itertext(elem)), sep) + id = stashedHTML2text(''.join(elem.itertext()), self.md) + id = slugify(id, sep) elem.set('id', unique(id, self.IDs)) if start_level: level = int(elem.tag[-1]) + start_level @@ -144,40 +48,34 @@ class HeaderIdTreeprocessor(Treeprocessor): level = 6 elem.tag = 'h%d' % level - def _get_meta(self): """ Return meta data suported by this ext as a tuple """ level = int(self.config['level']) - 1 - force = self._str2bool(self.config['forceid']) + force = parseBoolValue(self.config['forceid']) if hasattr(self.md, 'Meta'): if 'header_level' in self.md.Meta: level = int(self.md.Meta['header_level'][0]) - 1 - if 'header_forceid' in self.md.Meta: - force = self._str2bool(self.md.Meta['header_forceid'][0]) + if 'header_forceid' in self.md.Meta: + force = parseBoolValue(self.md.Meta['header_forceid'][0]) return level, force - def _str2bool(self, s, default=False): - """ Convert a string to a booleen value. """ - s = str(s) - if s.lower() in ['0', 'f', 'false', 'off', 'no', 'n']: - return False - elif s.lower() in ['1', 't', 'true', 'on', 'yes', 'y']: - return True - return default - class HeaderIdExtension(Extension): - def __init__(self, configs): + def __init__(self, *args, **kwargs): # set defaults self.config = { - 'level' : ['1', 'Base level for headers.'], - 'forceid' : ['True', 'Force all headers to have an id.'], - 'separator' : ['-', 'Word separator.'], - 'slugify' : [slugify, 'Callable to generate anchors'], - } + 'level': ['1', 'Base level for headers.'], + 'forceid': ['True', 'Force all headers to have an id.'], + 'separator': ['-', 'Word separator.'], + 'slugify': [slugify, 'Callable to generate anchors'] + } - for key, value in configs: - self.setConfig(key, value) + super(HeaderIdExtension, self).__init__(*args, **kwargs) + + warnings.warn( + 'The HeaderId Extension is pending deprecation. Use the TOC Extension instead.', + PendingDeprecationWarning + ) def extendMarkdown(self, md, md_globals): md.registerExtension(self) @@ -195,5 +93,5 @@ class HeaderIdExtension(Extension): self.processor.IDs = set() -def makeExtension(configs=None): - return HeaderIdExtension(configs=configs) +def makeExtension(*args, **kwargs): + return HeaderIdExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/meta.py b/src/calibre/ebooks/markdown/extensions/meta.py index aaff4365a8..711235ef4a 100644 --- a/src/calibre/ebooks/markdown/extensions/meta.py +++ b/src/calibre/ebooks/markdown/extensions/meta.py @@ -4,38 +4,14 @@ Meta Data Extension for Python-Markdown This extension adds Meta Data handling to markdown. -Basic Usage: +See <https://pythonhosted.org/Markdown/extensions/meta_data.html> +for documentation. - >>> import markdown - >>> text = '''Title: A Test Doc. - ... Author: Waylan Limberg - ... John Doe - ... Blank_Data: - ... - ... The body. This is paragraph one. - ... ''' - >>> md = markdown.Markdown(['meta']) - >>> print md.convert(text) - <p>The body. This is paragraph one.</p> - >>> print md.Meta - {u'blank_data': [u''], u'author': [u'Waylan Limberg', u'John Doe'], u'title': [u'A Test Doc.']} +Original code Copyright 2007-2008 [Waylan Limberg](http://achinghead.com). -Make sure text without Meta Data still works (markdown < 1.6b returns a <p>). +All changes Copyright 2008-2014 The Python Markdown Project - >>> text = ' Some Code - not extra lines of meta data.' - >>> md = markdown.Markdown(['meta']) - >>> print md.convert(text) - <pre><code>Some Code - not extra lines of meta data. - </code></pre> - >>> md.Meta - {} - -Copyright 2007-2008 [Waylan Limberg](http://achinghead.com). - -Project website: <http://packages.python.org/Markdown/meta_data.html> -Contact: markdown@freewisdom.org - -License: BSD (see ../LICENSE.md for details) +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -44,18 +20,25 @@ from __future__ import unicode_literals from . import Extension from ..preprocessors import Preprocessor import re +import logging + +log = logging.getLogger('MARKDOWN') # Global Vars META_RE = re.compile(r'^[ ]{0,3}(?P<key>[A-Za-z0-9_-]+):\s*(?P<value>.*)') META_MORE_RE = re.compile(r'^[ ]{4,}(?P<value>.*)') +BEGIN_RE = re.compile(r'^-{3}(\s.*)?') +END_RE = re.compile(r'^(-{3}|\.{3})(\s.*)?') + class MetaExtension (Extension): """ Meta-Data extension for Python-Markdown. """ def extendMarkdown(self, md, md_globals): """ Add MetaPreprocessor to Markdown instance. """ - - md.preprocessors.add("meta", MetaPreprocessor(md), "_begin") + md.preprocessors.add("meta", + MetaPreprocessor(md), + ">normalize_whitespace") class MetaPreprocessor(Preprocessor): @@ -65,11 +48,13 @@ class MetaPreprocessor(Preprocessor): """ Parse Meta-Data and store in Markdown.Meta. """ meta = {} key = None - while 1: + if lines and BEGIN_RE.match(lines[0]): + lines.pop(0) + while lines: line = lines.pop(0) - if line.strip() == '': - break # blank line - done m1 = META_RE.match(line) + if line.strip() == '' or END_RE.match(line): + break # blank line or end of YAML header - done if m1: key = m1.group('key').lower().strip() value = m1.group('value').strip() @@ -84,10 +69,10 @@ class MetaPreprocessor(Preprocessor): meta[key].append(m2.group('value').strip()) else: lines.insert(0, line) - break # no meta data - done + break # no meta data - done self.markdown.Meta = meta return lines - -def makeExtension(configs={}): - return MetaExtension(configs=configs) + +def makeExtension(*args, **kwargs): + return MetaExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/nl2br.py b/src/calibre/ebooks/markdown/extensions/nl2br.py index da4b339958..8acd60c2e1 100644 --- a/src/calibre/ebooks/markdown/extensions/nl2br.py +++ b/src/calibre/ebooks/markdown/extensions/nl2br.py @@ -5,18 +5,14 @@ NL2BR Extension A Python-Markdown extension to treat newlines as hard breaks; like GitHub-flavored Markdown does. -Usage: +See <https://pythonhosted.org/Markdown/extensions/nl2br.html> +for documentation. - >>> import markdown - >>> print markdown.markdown('line 1\\nline 2', extensions=['nl2br']) - <p>line 1<br /> - line 2</p> +Oringinal code Copyright 2011 [Brian Neal](http://deathofagremmie.com/) -Copyright 2011 [Brian Neal](http://deathofagremmie.com/) +All changes Copyright 2011-2014 The Python Markdown Project -Dependencies: -* [Python 2.4+](http://python.org) -* [Markdown 2.1+](http://packages.python.org/Markdown/) +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -27,6 +23,7 @@ from ..inlinepatterns import SubstituteTagPattern BR_RE = r'\n' + class Nl2BrExtension(Extension): def extendMarkdown(self, md, md_globals): @@ -34,5 +31,5 @@ class Nl2BrExtension(Extension): md.inlinePatterns.add('nl', br_tag, '_end') -def makeExtension(configs=None): - return Nl2BrExtension(configs) +def makeExtension(*args, **kwargs): + return Nl2BrExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/sane_lists.py b/src/calibre/ebooks/markdown/extensions/sane_lists.py index 23e9a7f4a6..828ae7ab34 100644 --- a/src/calibre/ebooks/markdown/extensions/sane_lists.py +++ b/src/calibre/ebooks/markdown/extensions/sane_lists.py @@ -2,19 +2,16 @@ Sane List Extension for Python-Markdown ======================================= -Modify the behavior of Lists in Python-Markdown t act in a sane manor. +Modify the behavior of Lists in Python-Markdown to act in a sane manor. -In standard Markdown sytex, the following would constitute a single -ordered list. However, with this extension, the output would include -two lists, the first an ordered list and the second and unordered list. +See <https://pythonhosted.org/Markdown/extensions/sane_lists.html> +for documentation. - 1. ordered - 2. list +Original code Copyright 2011 [Waylan Limberg](http://achinghead.com) - * unordered - * list +All changes Copyright 2011-2014 The Python Markdown Project -Copyright 2011 - [Waylan Limberg](http://achinghead.com) +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -26,16 +23,24 @@ import re class SaneOListProcessor(OListProcessor): - - CHILD_RE = re.compile(r'^[ ]{0,3}((\d+\.))[ ]+(.*)') + SIBLING_TAGS = ['ol'] + def __init__(self, parser): + super(SaneOListProcessor, self).__init__(parser) + self.CHILD_RE = re.compile(r'^[ ]{0,%d}((\d+\.))[ ]+(.*)' % + (self.tab_length - 1)) + class SaneUListProcessor(UListProcessor): - - CHILD_RE = re.compile(r'^[ ]{0,3}(([*+-]))[ ]+(.*)') + SIBLING_TAGS = ['ul'] + def __init__(self, parser): + super(SaneUListProcessor, self).__init__(parser) + self.CHILD_RE = re.compile(r'^[ ]{0,%d}(([*+-]))[ ]+(.*)' % + (self.tab_length - 1)) + class SaneListExtension(Extension): """ Add sane lists to Markdown. """ @@ -46,6 +51,5 @@ class SaneListExtension(Extension): md.parser.blockprocessors['ulist'] = SaneUListProcessor(md.parser) -def makeExtension(configs={}): - return SaneListExtension(configs=configs) - +def makeExtension(*args, **kwargs): + return SaneListExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/smart_strong.py b/src/calibre/ebooks/markdown/extensions/smart_strong.py index 4818cf9ea8..58570bb55e 100644 --- a/src/calibre/ebooks/markdown/extensions/smart_strong.py +++ b/src/calibre/ebooks/markdown/extensions/smart_strong.py @@ -4,21 +4,14 @@ Smart_Strong Extension for Python-Markdown This extention adds smarter handling of double underscores within words. -Simple Usage: +See <https://pythonhosted.org/Markdown/extensions/smart_strong.html> +for documentation. - >>> import markdown - >>> print markdown.markdown('Text with double__underscore__words.', - ... extensions=['smart_strong']) - <p>Text with double__underscore__words.</p> - >>> print markdown.markdown('__Strong__ still works.', - ... extensions=['smart_strong']) - <p><strong>Strong</strong> still works.</p> - >>> print markdown.markdown('__this__works__too__.', - ... extensions=['smart_strong']) - <p><strong>this__works__too</strong>.</p> +Original code Copyright 2011 [Waylan Limberg](http://achinghead.com) -Copyright 2011 -[Waylan Limberg](http://achinghead.com) +All changes Copyright 2011-2014 The Python Markdown Project + +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) ''' @@ -30,13 +23,19 @@ from ..inlinepatterns import SimpleTagPattern SMART_STRONG_RE = r'(?<!\w)(_{2})(?!_)(.+?)(?<!_)\2(?!\w)' STRONG_RE = r'(\*{2})(.+?)\2' + class SmartEmphasisExtension(Extension): """ Add smart_emphasis extension to Markdown class.""" def extendMarkdown(self, md, md_globals): """ Modify inline patterns. """ md.inlinePatterns['strong'] = SimpleTagPattern(STRONG_RE, 'strong') - md.inlinePatterns.add('strong2', SimpleTagPattern(SMART_STRONG_RE, 'strong'), '>emphasis2') + md.inlinePatterns.add( + 'strong2', + SimpleTagPattern(SMART_STRONG_RE, 'strong'), + '>emphasis2' + ) -def makeExtension(configs={}): - return SmartEmphasisExtension(configs=dict(configs)) + +def makeExtension(*args, **kwargs): + return SmartEmphasisExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/tables.py b/src/calibre/ebooks/markdown/extensions/tables.py index ad52ec11c7..494aaeb3e4 100644 --- a/src/calibre/ebooks/markdown/extensions/tables.py +++ b/src/calibre/ebooks/markdown/extensions/tables.py @@ -4,29 +4,32 @@ Tables Extension for Python-Markdown Added parsing of tables to Python-Markdown. -A simple example: +See <https://pythonhosted.org/Markdown/extensions/tables.html> +for documentation. - First Header | Second Header - ------------- | ------------- - Content Cell | Content Cell - Content Cell | Content Cell +Original code Copyright 2009 [Waylan Limberg](http://achinghead.com) + +All changes Copyright 2008-2014 The Python Markdown Project + +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) -Copyright 2009 - [Waylan Limberg](http://achinghead.com) """ from __future__ import absolute_import from __future__ import unicode_literals from . import Extension from ..blockprocessors import BlockProcessor +from ..inlinepatterns import BacktickPattern, BACKTICK_RE from ..util import etree + class TableProcessor(BlockProcessor): """ Process Tables. """ def test(self, parent, block): rows = block.split('\n') - return (len(rows) > 2 and '|' in rows[0] and - '|' in rows[1] and '-' in rows[1] and + return (len(rows) > 1 and '|' in rows[0] and + '|' in rows[1] and '-' in rows[1] and rows[1].strip()[0] in ['|', ':', '-']) def run(self, parent, blocks): @@ -34,7 +37,7 @@ class TableProcessor(BlockProcessor): block = blocks.pop(0).split('\n') header = block[0].strip() seperator = block[1].strip() - rows = block[2:] + rows = [] if len(block) < 3 else block[2:] # Get format type (bordered by pipes or not) border = False if header.startswith('|'): @@ -65,13 +68,17 @@ class TableProcessor(BlockProcessor): if parent.tag == 'thead': tag = 'th' cells = self._split_row(row, border) - # We use align here rather than cells to ensure every row + # We use align here rather than cells to ensure every row # contains the same number of columns. for i, a in enumerate(align): c = etree.SubElement(tr, tag) try: - c.text = cells[i].strip() - except IndexError: + if isinstance(cells[i], str) or isinstance(cells[i], unicode): + c.text = cells[i].strip() + else: + # we've already inserted a code element + c.append(cells[i]) + except IndexError: # pragma: no cover c.text = "" if a: c.set('align', a) @@ -83,7 +90,49 @@ class TableProcessor(BlockProcessor): row = row[1:] if row.endswith('|'): row = row[:-1] - return row.split('|') + return self._split(row, '|') + + def _split(self, row, marker): + """ split a row of text with some code into a list of cells. """ + if self._row_has_unpaired_backticks(row): + # fallback on old behaviour + return row.split(marker) + # modify the backtick pattern to only match at the beginning of the search string + backtick_pattern = BacktickPattern('^' + BACKTICK_RE) + elements = [] + current = '' + i = 0 + while i < len(row): + letter = row[i] + if letter == marker: + if current != '' or len(elements) == 0: + # Don't append empty string unless it is the first element + # The border is already removed when we get the row, then the line is strip()'d + # If the first element is a marker, then we have an empty first cell + elements.append(current) + current = '' + else: + match = backtick_pattern.getCompiledRegExp().match(row[i:]) + if not match: + current += letter + else: + groups = match.groups() + delim = groups[1] # the code block delimeter (ie 1 or more backticks) + row_contents = groups[2] # the text contained inside the code block + i += match.start(4) # jump pointer to the beginning of the rest of the text (group #4) + element = delim + row_contents + delim # reinstert backticks + current += element + i += 1 + elements.append(current) + return elements + + def _row_has_unpaired_backticks(self, row): + count_total_backtick = row.count('`') + count_escaped_backtick = row.count('\`') + count_backtick = count_total_backtick - count_escaped_backtick + # odd number of backticks, + # we won't be able to build correct code blocks + return count_backtick & 1 class TableExtension(Extension): @@ -91,10 +140,10 @@ class TableExtension(Extension): def extendMarkdown(self, md, md_globals): """ Add an instance of TableProcessor to BlockParser. """ - md.parser.blockprocessors.add('table', + md.parser.blockprocessors.add('table', TableProcessor(md.parser), '<hashheader') -def makeExtension(configs={}): - return TableExtension(configs=configs) +def makeExtension(*args, **kwargs): + return TableExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/toc.py b/src/calibre/ebooks/markdown/extensions/toc.py index 73b0844517..b3cf898f42 100644 --- a/src/calibre/ebooks/markdown/extensions/toc.py +++ b/src/calibre/ebooks/markdown/extensions/toc.py @@ -1,11 +1,15 @@ """ Table of Contents Extension for Python-Markdown -* * * +=============================================== -(c) 2008 [Jack Miller](http://codezen.org) +See <https://pythonhosted.org/Markdown/extensions/toc.html> +for documentation. -Dependencies: -* [Markdown 2.1+](http://packages.python.org/Markdown/) +Oringinal code Copyright 2008 [Jack Miller](http://codezen.org) + +All changes Copyright 2008-2014 The Python Markdown Project + +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) """ @@ -13,99 +17,192 @@ from __future__ import absolute_import from __future__ import unicode_literals from . import Extension from ..treeprocessors import Treeprocessor -from ..util import etree -from .headerid import slugify, unique, itertext +from ..util import etree, parseBoolValue, AMP_SUBSTITUTE, HTML_PLACEHOLDER_RE, string_type import re +import unicodedata -def order_toc_list(toc_list): +def slugify(value, separator): + """ Slugify a string, to make it URL friendly. """ + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') + value = re.sub('[^\w\s-]', '', value.decode('ascii')).strip().lower() + return re.sub('[%s\s]+' % separator, separator, value) + + +IDCOUNT_RE = re.compile(r'^(.*)_([0-9]+)$') + + +def unique(id, ids): + """ Ensure id is unique in set of ids. Append '_1', '_2'... if not """ + while id in ids or not id: + m = IDCOUNT_RE.match(id) + if m: + id = '%s_%d' % (m.group(1), int(m.group(2))+1) + else: + id = '%s_%d' % (id, 1) + ids.add(id) + return id + + +def stashedHTML2text(text, md): + """ Extract raw HTML from stash, reduce to plain text and swap with placeholder. """ + def _html_sub(m): + """ Substitute raw html with plain text. """ + try: + raw, safe = md.htmlStash.rawHtmlBlocks[int(m.group(1))] + except (IndexError, TypeError): # pragma: no cover + return m.group(0) + if md.safeMode and not safe: # pragma: no cover + return '' + # Strip out tags and entities - leaveing text + return re.sub(r'(<[^>]+>)|(&[\#a-zA-Z0-9]+;)', '', raw) + + return HTML_PLACEHOLDER_RE.sub(_html_sub, text) + + +def nest_toc_tokens(toc_list): """Given an unsorted list with errors and skips, return a nested one. [{'level': 1}, {'level': 2}] => [{'level': 1, 'children': [{'level': 2, 'children': []}]}] - + A wrong list is also converted: [{'level': 2}, {'level': 1}] => [{'level': 2, 'children': []}, {'level': 1, 'children': []}] """ - - def build_correct(remaining_list, prev_elements=[{'level': 1000}]): - - if not remaining_list: - return [], [] - - current = remaining_list.pop(0) - if not 'children' in current.keys(): - current['children'] = [] - - if not prev_elements: - # This happens for instance with [8, 1, 1], ie. when some - # header level is outside a scope. We treat it as a - # top-level - next_elements, children = build_correct(remaining_list, [current]) - current['children'].append(children) - return [current] + next_elements, [] - - prev_element = prev_elements.pop() - children = [] - next_elements = [] - # Is current part of the child list or next list? - if current['level'] > prev_element['level']: - #print "%d is a child of %d" % (current['level'], prev_element['level']) - prev_elements.append(prev_element) - prev_elements.append(current) - prev_element['children'].append(current) - next_elements2, children2 = build_correct(remaining_list, prev_elements) - children += children2 - next_elements += next_elements2 - else: - #print "%d is ancestor of %d" % (current['level'], prev_element['level']) - if not prev_elements: - #print "No previous elements, so appending to the next set" - next_elements.append(current) - prev_elements = [current] - next_elements2, children2 = build_correct(remaining_list, prev_elements) - current['children'].extend(children2) + + ordered_list = [] + if len(toc_list): + # Initialize everything by processing the first entry + last = toc_list.pop(0) + last['children'] = [] + levels = [last['level']] + ordered_list.append(last) + parents = [] + + # Walk the rest nesting the entries properly + while toc_list: + t = toc_list.pop(0) + current_level = t['level'] + t['children'] = [] + + # Reduce depth if current level < last item's level + if current_level < levels[-1]: + # Pop last level since we know we are less than it + levels.pop() + + # Pop parents and levels we are less than or equal to + to_pop = 0 + for p in reversed(parents): + if current_level <= p['level']: + to_pop += 1 + else: # pragma: no cover + break + if to_pop: + levels = levels[:-to_pop] + parents = parents[:-to_pop] + + # Note current level as last + levels.append(current_level) + + # Level is the same, so append to + # the current parent (if available) + if current_level == levels[-1]: + (parents[-1]['children'] if parents + else ordered_list).append(t) + + # Current level is > last item's level, + # So make last item a parent and append current as child else: - #print "Previous elements, comparing to those first" - remaining_list.insert(0, current) - next_elements2, children2 = build_correct(remaining_list, prev_elements) - children.extend(children2) - next_elements += next_elements2 - - return next_elements, children - - ordered_list, __ = build_correct(toc_list) + last['children'].append(t) + parents.append(last) + levels.append(current_level) + last = t + return ordered_list class TocTreeprocessor(Treeprocessor): - - # Iterator wrapper to get parent and child all at once + def __init__(self, md, config): + super(TocTreeprocessor, self).__init__(md) + + self.marker = config["marker"] + self.title = config["title"] + self.base_level = int(config["baselevel"]) - 1 + self.slugify = config["slugify"] + self.sep = config["separator"] + self.use_anchors = parseBoolValue(config["anchorlink"]) + self.use_permalinks = parseBoolValue(config["permalink"], False) + if self.use_permalinks is None: + self.use_permalinks = config["permalink"] + + self.header_rgx = re.compile("[Hh][123456]") + def iterparent(self, root): - for parent in root.getiterator(): + ''' Iterator wrapper to get parent and child all at once. ''' + for parent in root.iter(): for child in parent: yield parent, child - - def add_anchor(self, c, elem_id): #@ReservedAssignment - if self.use_anchors: - anchor = etree.Element("a") - anchor.text = c.text - anchor.attrib["href"] = "#" + elem_id - anchor.attrib["class"] = "toclink" - c.text = "" - for elem in c.getchildren(): - anchor.append(elem) - c.remove(elem) - c.append(anchor) - - def build_toc_etree(self, div, toc_list): + + def replace_marker(self, root, elem): + ''' Replace marker with elem. ''' + for (p, c) in self.iterparent(root): + text = ''.join(c.itertext()).strip() + if not text: + continue + + # To keep the output from screwing up the + # validation by putting a <div> inside of a <p> + # we actually replace the <p> in its entirety. + # We do not allow the marker inside a header as that + # would causes an enless loop of placing a new TOC + # inside previously generated TOC. + if c.text and c.text.strip() == self.marker and \ + not self.header_rgx.match(c.tag) and c.tag not in ['pre', 'code']: + for i in range(len(p)): + if p[i] == c: + p[i] = elem + break + + def set_level(self, elem): + ''' Adjust header level according to base level. ''' + level = int(elem.tag[-1]) + self.base_level + if level > 6: + level = 6 + elem.tag = 'h%d' % level + + def add_anchor(self, c, elem_id): # @ReservedAssignment + anchor = etree.Element("a") + anchor.text = c.text + anchor.attrib["href"] = "#" + elem_id + anchor.attrib["class"] = "toclink" + c.text = "" + for elem in c: + anchor.append(elem) + c.remove(elem) + c.append(anchor) + + def add_permalink(self, c, elem_id): + permalink = etree.Element("a") + permalink.text = ("%spara;" % AMP_SUBSTITUTE + if self.use_permalinks is True + else self.use_permalinks) + permalink.attrib["href"] = "#" + elem_id + permalink.attrib["class"] = "headerlink" + permalink.attrib["title"] = "Permanent link" + c.append(permalink) + + def build_toc_div(self, toc_list): + """ Return a string div given a toc list. """ + div = etree.Element("div") + div.attrib["class"] = "toc" + # Add title to the div - if self.config["title"]: + if self.title: header = etree.SubElement(div, "span") header.attrib["class"] = "toctitle" - header.text = self.config["title"] + header.text = self.title def build_etree_ul(toc_list, parent): ul = etree.SubElement(parent, "ul") @@ -118,104 +215,95 @@ class TocTreeprocessor(Treeprocessor): if item['children']: build_etree_ul(item['children'], li) return ul - - return build_etree_ul(toc_list, div) - - def run(self, doc): - div = etree.Element("div") - div.attrib["class"] = "toc" - header_rgx = re.compile("[Hh][123456]") - - self.use_anchors = self.config["anchorlink"] in [1, '1', True, 'True', 'true'] - + build_etree_ul(toc_list, div) + prettify = self.markdown.treeprocessors.get('prettify') + if prettify: + prettify.run(div) + return div + + def run(self, doc): # Get a list of id attributes used_ids = set() - for c in doc.getiterator(): - if "id" in c.attrib: - used_ids.add(c.attrib["id"]) + for el in doc.iter(): + if "id" in el.attrib: + used_ids.add(el.attrib["id"]) - toc_list = [] - marker_found = False - for (p, c) in self.iterparent(doc): - text = ''.join(itertext(c)).strip() - if not text: - continue + toc_tokens = [] + for el in doc.iter(): + if isinstance(el.tag, string_type) and self.header_rgx.match(el.tag): + self.set_level(el) + text = ''.join(el.itertext()).strip() - # To keep the output from screwing up the - # validation by putting a <div> inside of a <p> - # we actually replace the <p> in its entirety. - # We do not allow the marker inside a header as that - # would causes an enless loop of placing a new TOC - # inside previously generated TOC. - if c.text and c.text.strip() == self.config["marker"] and \ - not header_rgx.match(c.tag) and c.tag not in ['pre', 'code']: - for i in range(len(p)): - if p[i] == c: - p[i] = div - break - marker_found = True - - if header_rgx.match(c.tag): - - # Do not override pre-existing ids - if not "id" in c.attrib: - elem_id = unique(self.config["slugify"](text, '-'), used_ids) - c.attrib["id"] = elem_id - else: - elem_id = c.attrib["id"] + # Do not override pre-existing ids + if "id" not in el.attrib: + innertext = stashedHTML2text(text, self.markdown) + el.attrib["id"] = unique(self.slugify(innertext, self.sep), used_ids) - tag_level = int(c.tag[-1]) - - toc_list.append({'level': tag_level, - 'id': elem_id, - 'name': text}) - - self.add_anchor(c, elem_id) - - toc_list_nested = order_toc_list(toc_list) - self.build_toc_etree(div, toc_list_nested) - prettify = self.markdown.treeprocessors.get('prettify') - if prettify: prettify.run(div) - if not marker_found: - # serialize and attach to markdown instance. - toc = self.markdown.serializer(div) - for pp in self.markdown.postprocessors.values(): - toc = pp.run(toc) - self.markdown.toc = toc + toc_tokens.append({ + 'level': int(el.tag[-1]), + 'id': el.attrib["id"], + 'name': text + }) + + if self.use_anchors: + self.add_anchor(el, el.attrib["id"]) + if self.use_permalinks: + self.add_permalink(el, el.attrib["id"]) + + div = self.build_toc_div(nest_toc_tokens(toc_tokens)) + if self.marker: + self.replace_marker(doc, div) + + # serialize and attach to markdown instance. + toc = self.markdown.serializer(div) + for pp in self.markdown.postprocessors.values(): + toc = pp.run(toc) + self.markdown.toc = toc class TocExtension(Extension): - - TreeProcessorClass = TocTreeprocessor - - def __init__(self, configs=[]): - self.config = { "marker" : ["[TOC]", - "Text to find and replace with Table of Contents -" - "Defaults to \"[TOC]\""], - "slugify" : [slugify, - "Function to generate anchors based on header text-" - "Defaults to the headerid ext's slugify function."], - "title" : [None, - "Title to insert into TOC <div> - " - "Defaults to None"], - "anchorlink" : [0, - "1 if header should be a self link" - "Defaults to 0"]} - for key, value in configs: - self.setConfig(key, value) + TreeProcessorClass = TocTreeprocessor + + def __init__(self, *args, **kwargs): + self.config = { + "marker": ['[TOC]', + 'Text to find and replace with Table of Contents - ' + 'Set to an empty string to disable. Defaults to "[TOC]"'], + "title": ["", + "Title to insert into TOC <div> - " + "Defaults to an empty string"], + "anchorlink": [False, + "True if header should be a self link - " + "Defaults to False"], + "permalink": [0, + "True or link text if a Sphinx-style permalink should " + "be added - Defaults to False"], + "baselevel": ['1', 'Base level for headers.'], + "slugify": [slugify, + "Function to generate anchors based on header text - " + "Defaults to the headerid ext's slugify function."], + 'separator': ['-', 'Word separator. Defaults to "-".'] + } + + super(TocExtension, self).__init__(*args, **kwargs) def extendMarkdown(self, md, md_globals): - tocext = self.TreeProcessorClass(md) - tocext.config = self.getConfigs() + md.registerExtension(self) + self.md = md + self.reset() + tocext = self.TreeProcessorClass(md, self.getConfigs()) # Headerid ext is set to '>prettify'. With this set to '_end', - # it should always come after headerid ext (and honor ids assinged - # by the header id extension) if both are used. Same goes for + # it should always come after headerid ext (and honor ids assinged + # by the header id extension) if both are used. Same goes for # attr_list extension. This must come last because we don't want # to redefine ids after toc is created. But we do want toc prettified. md.treeprocessors.add("toc", tocext, "_end") + def reset(self): + self.md.toc = '' -def makeExtension(configs={}): - return TocExtension(configs=configs) + +def makeExtension(*args, **kwargs): + return TocExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/extensions/wikilinks.py b/src/calibre/ebooks/markdown/extensions/wikilinks.py index 877890b8ab..94e1b67948 100644 --- a/src/calibre/ebooks/markdown/extensions/wikilinks.py +++ b/src/calibre/ebooks/markdown/extensions/wikilinks.py @@ -2,78 +2,17 @@ WikiLinks Extension for Python-Markdown ====================================== -Converts [[WikiLinks]] to relative links. Requires Python-Markdown 2.0+ +Converts [[WikiLinks]] to relative links. -Basic usage: +See <https://pythonhosted.org/Markdown/extensions/wikilinks.html> +for documentation. - >>> import markdown - >>> text = "Some text with a [[WikiLink]]." - >>> html = markdown.markdown(text, ['wikilinks']) - >>> print html - <p>Some text with a <a class="wikilink" href="/WikiLink/">WikiLink</a>.</p> +Original code Copyright [Waylan Limberg](http://achinghead.com/). -Whitespace behavior: +All changes Copyright The Python Markdown Project - >>> print markdown.markdown('[[ foo bar_baz ]]', ['wikilinks']) - <p><a class="wikilink" href="/foo_bar_baz/">foo bar_baz</a></p> - >>> print markdown.markdown('foo [[ ]] bar', ['wikilinks']) - <p>foo bar</p> +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) -To define custom settings the simple way: - - >>> print markdown.markdown(text, - ... ['wikilinks(base_url=/wiki/,end_url=.html,html_class=foo)'] - ... ) - <p>Some text with a <a class="foo" href="/wiki/WikiLink.html">WikiLink</a>.</p> - -Custom settings the complex way: - - >>> md = markdown.Markdown( - ... extensions = ['wikilinks'], - ... extension_configs = {'wikilinks': [ - ... ('base_url', 'http://example.com/'), - ... ('end_url', '.html'), - ... ('html_class', '') ]}, - ... safe_mode = True) - >>> print md.convert(text) - <p>Some text with a <a href="http://example.com/WikiLink.html">WikiLink</a>.</p> - -Use MetaData with mdx_meta.py (Note the blank html_class in MetaData): - - >>> text = """wiki_base_url: http://example.com/ - ... wiki_end_url: .html - ... wiki_html_class: - ... - ... Some text with a [[WikiLink]].""" - >>> md = markdown.Markdown(extensions=['meta', 'wikilinks']) - >>> print md.convert(text) - <p>Some text with a <a href="http://example.com/WikiLink.html">WikiLink</a>.</p> - -MetaData should not carry over to next document: - - >>> print md.convert("No [[MetaData]] here.") - <p>No <a class="wikilink" href="/MetaData/">MetaData</a> here.</p> - -Define a custom URL builder: - - >>> def my_url_builder(label, base, end): - ... return '/bar/' - >>> md = markdown.Markdown(extensions=['wikilinks'], - ... extension_configs={'wikilinks' : [('build_url', my_url_builder)]}) - >>> print md.convert('[[foo]]') - <p><a class="wikilink" href="/bar/">foo</a></p> - -From the command line: - - python markdown.py -x wikilinks(base_url=http://example.com/,end_url=.html,html_class=foo) src.txt - -By [Waylan Limberg](http://achinghead.com/). - -License: [BSD](http://www.opensource.org/licenses/bsd-license.php) - -Dependencies: -* [Python 2.3+](http://python.org) -* [Markdown 2.0+](http://packages.python.org/Markdown/) ''' from __future__ import absolute_import @@ -83,29 +22,28 @@ from ..inlinepatterns import Pattern from ..util import etree import re + def build_url(label, base, end): """ Build a url from the label, a base, and an end. """ clean_label = re.sub(r'([ ]+_)|(_[ ]+)|([ ]+)', '_', label) - return '%s%s%s'% (base, clean_label, end) + return '%s%s%s' % (base, clean_label, end) class WikiLinkExtension(Extension): - def __init__(self, configs): - # set extension defaults + + def __init__(self, *args, **kwargs): self.config = { - 'base_url' : ['/', 'String to append to beginning or URL.'], - 'end_url' : ['/', 'String to append to end of URL.'], - 'html_class' : ['wikilink', 'CSS hook. Leave blank for none.'], - 'build_url' : [build_url, 'Callable formats URL from label.'], + 'base_url': ['/', 'String to append to beginning or URL.'], + 'end_url': ['/', 'String to append to end of URL.'], + 'html_class': ['wikilink', 'CSS hook. Leave blank for none.'], + 'build_url': [build_url, 'Callable formats URL from label.'], } - - # Override defaults with user settings - for key, value in configs : - self.setConfig(key, value) - + + super(WikiLinkExtension, self).__init__(*args, **kwargs) + def extendMarkdown(self, md, md_globals): self.md = md - + # append to end of inline patterns WIKILINK_RE = r'\[\[([\w0-9_ -]+)\]\]' wikilinkPattern = WikiLinks(WIKILINK_RE, self.getConfigs()) @@ -117,14 +55,14 @@ class WikiLinks(Pattern): def __init__(self, pattern, config): super(WikiLinks, self).__init__(pattern) self.config = config - + def handleMatch(self, m): if m.group(2).strip(): base_url, end_url, html_class = self._getMeta() label = m.group(2).strip() url = self.config['build_url'](label, base_url, end_url) a = etree.Element('a') - a.text = label + a.text = label a.set('href', url) if html_class: a.set('class', html_class) @@ -145,7 +83,7 @@ class WikiLinks(Pattern): if 'wiki_html_class' in self.md.Meta: html_class = self.md.Meta['wiki_html_class'][0] return base_url, end_url, html_class - -def makeExtension(configs=None) : - return WikiLinkExtension(configs=configs) + +def makeExtension(*args, **kwargs): + return WikiLinkExtension(*args, **kwargs) diff --git a/src/calibre/ebooks/markdown/inlinepatterns.py b/src/calibre/ebooks/markdown/inlinepatterns.py index de957ef480..95d358d715 100644 --- a/src/calibre/ebooks/markdown/inlinepatterns.py +++ b/src/calibre/ebooks/markdown/inlinepatterns.py @@ -46,13 +46,13 @@ from __future__ import unicode_literals from . import util from . import odict import re -try: +try: # pragma: no cover from urllib.parse import urlparse, urlunparse -except ImportError: +except ImportError: # pragma: no cover from urlparse import urlparse, urlunparse -try: +try: # pragma: no cover from html import entities -except ImportError: +except ImportError: # pragma: no cover import htmlentitydefs as entities @@ -64,10 +64,12 @@ def build_inlinepatterns(md_instance, **kwargs): inlinePatterns["reference"] = ReferencePattern(REFERENCE_RE, md_instance) inlinePatterns["link"] = LinkPattern(LINK_RE, md_instance) inlinePatterns["image_link"] = ImagePattern(IMAGE_LINK_RE, md_instance) - inlinePatterns["image_reference"] = \ - ImageReferencePattern(IMAGE_REFERENCE_RE, md_instance) - inlinePatterns["short_reference"] = \ - ReferencePattern(SHORT_REF_RE, md_instance) + inlinePatterns["image_reference"] = ImageReferencePattern( + IMAGE_REFERENCE_RE, md_instance + ) + inlinePatterns["short_reference"] = ReferencePattern( + SHORT_REF_RE, md_instance + ) inlinePatterns["autolink"] = AutolinkPattern(AUTOLINK_RE, md_instance) inlinePatterns["automail"] = AutomailPattern(AUTOMAIL_RE, md_instance) inlinePatterns["linebreak"] = SubstituteTagPattern(LINE_BREAK_RE, 'br') @@ -75,7 +77,8 @@ def build_inlinepatterns(md_instance, **kwargs): inlinePatterns["html"] = HtmlPattern(HTML_RE, md_instance) inlinePatterns["entity"] = HtmlPattern(ENTITY_RE, md_instance) inlinePatterns["not_strong"] = SimpleTextPattern(NOT_STRONG_RE) - inlinePatterns["strong_em"] = DoubleTagPattern(STRONG_EM_RE, 'strong,em') + inlinePatterns["em_strong"] = DoubleTagPattern(EM_STRONG_RE, 'strong,em') + inlinePatterns["strong_em"] = DoubleTagPattern(STRONG_EM_RE, 'em,strong') inlinePatterns["strong"] = SimpleTagPattern(STRONG_RE, 'strong') inlinePatterns["emphasis"] = SimpleTagPattern(EMPHASIS_RE, 'em') if md_instance.smart_emphasis: @@ -90,46 +93,84 @@ The actual regular expressions for patterns """ NOBRACKET = r'[^\]\[]*' -BRK = ( r'\[(' - + (NOBRACKET + r'(\[')*6 - + (NOBRACKET+ r'\])*')*6 - + NOBRACKET + r')\]' ) +BRK = ( + r'\[(' + + (NOBRACKET + r'(\[')*6 + + (NOBRACKET + r'\])*')*6 + + NOBRACKET + r')\]' +) NOIMG = r'(?<!\!)' -BACKTICK_RE = r'(?<!\\)(`+)(.+?)(?<!`)\2(?!`)' # `e=f()` or ``e=f("`")`` -ESCAPE_RE = r'\\(.)' # \< -EMPHASIS_RE = r'(\*)([^\*]+)\2' # *emphasis* -STRONG_RE = r'(\*{2}|_{2})(.+?)\2' # **strong** -STRONG_EM_RE = r'(\*{3}|_{3})(.+?)\2' # ***strong*** -SMART_EMPHASIS_RE = r'(?<!\w)(_)(?!_)(.+?)(?<!_)\2(?!\w)' # _smart_emphasis_ -EMPHASIS_2_RE = r'(_)(.+?)\2' # _emphasis_ -LINK_RE = NOIMG + BRK + \ -r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*?)\12\s*)?\)''' +# `e=f()` or ``e=f("`")`` +BACKTICK_RE = r'(?<!\\)(`+)(.+?)(?<!`)\2(?!`)' + +# \< +ESCAPE_RE = r'\\(.)' + +# *emphasis* +EMPHASIS_RE = r'(\*)([^\*]+)\2' + +# **strong** +STRONG_RE = r'(\*{2}|_{2})(.+?)\2' + +# ***strongem*** or ***em*strong** +EM_STRONG_RE = r'(\*|_)\2{2}(.+?)\2(.*?)\2{2}' + +# ***strong**em* +STRONG_EM_RE = r'(\*|_)\2{2}(.+?)\2{2}(.*?)\2' + +# _smart_emphasis_ +SMART_EMPHASIS_RE = r'(?<!\w)(_)(?!_)(.+?)(?<!_)\2(?!\w)' + +# _emphasis_ +EMPHASIS_2_RE = r'(_)(.+?)\2' + # [text](url) or [text](<url>) or [text](url "title") +LINK_RE = NOIMG + BRK + \ + r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*?)\12\s*)?\)''' -IMAGE_LINK_RE = r'\!' + BRK + r'\s*\((<.*?>|([^\)]*))\)' # ![alttxt](http://x.com/) or ![alttxt](<http://x.com/>) -REFERENCE_RE = NOIMG + BRK+ r'\s?\[([^\]]*)\]' # [Google][3] -SHORT_REF_RE = NOIMG + r'\[([^\]]+)\]' # [Google] -IMAGE_REFERENCE_RE = r'\!' + BRK + '\s?\[([^\]]*)\]' # ![alt text][2] -NOT_STRONG_RE = r'((^| )(\*|_)( |$))' # stand-alone * or _ -AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^>]*)>' # <http://www.123.com> -AUTOMAIL_RE = r'<([^> \!]*@[^> ]*)>' # <me@example.com> +IMAGE_LINK_RE = r'\!' + BRK + r'\s*\((<.*?>|([^")]+"[^"]*"|[^\)]*))\)' -HTML_RE = r'(\<([a-zA-Z/][^\>]*?|\!--.*?--)\>)' # <...> -ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)' # & -LINE_BREAK_RE = r' \n' # two spaces at end of line +# [Google][3] +REFERENCE_RE = NOIMG + BRK + r'\s?\[([^\]]*)\]' + +# [Google] +SHORT_REF_RE = NOIMG + r'\[([^\]]+)\]' + +# ![alt text][2] +IMAGE_REFERENCE_RE = r'\!' + BRK + '\s?\[([^\]]*)\]' + +# stand-alone * or _ +NOT_STRONG_RE = r'((^| )(\*|_)( |$))' + +# <http://www.123.com> +AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^>]*)>' + +# <me@example.com> +AUTOMAIL_RE = r'<([^> \!]*@[^> ]*)>' + +# <...> +HTML_RE = r'(\<([a-zA-Z/][^\>]*?|\!--.*?--)\>)' + +# & +ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)' + +# two spaces at end of line +LINE_BREAK_RE = r' \n' def dequote(string): """Remove quotes from around a string.""" - if ( ( string.startswith('"') and string.endswith('"')) - or (string.startswith("'") and string.endswith("'")) ): + if ((string.startswith('"') and string.endswith('"')) or + (string.startswith("'") and string.endswith("'"))): return string[1:-1] else: return string -ATTR_RE = re.compile("\{@([^\}]*)=([^\}]*)}") # {@id=123} + +ATTR_RE = re.compile("\{@([^\}]*)=([^\}]*)}") # {@id=123} + def handleAttributes(text, parent): """Set values of an element based on attribute definitions ({@id=123}).""" @@ -143,6 +184,7 @@ The pattern classes ----------------------------------------------------------------------------- """ + class Pattern(object): """Base class that inline patterns subclass. """ @@ -156,7 +198,7 @@ class Pattern(object): """ self.pattern = pattern - self.compiled_re = re.compile("^(.*?)%s(.*?)$" % pattern, + self.compiled_re = re.compile("^(.*?)%s(.*?)$" % pattern, re.DOTALL | re.UNICODE) # Api for Markdown to pass safe_mode into instance @@ -178,7 +220,7 @@ class Pattern(object): * m: A re match object containing a match of the pattern. """ - pass + pass # pragma: no cover def type(self): """ Return class name, to define pattern type """ @@ -188,9 +230,10 @@ class Pattern(object): """ Return unescaped text given text with an inline placeholder. """ try: stash = self.markdown.treeprocessors['inline'].stashed_nodes - except KeyError: + except KeyError: # pragma: no cover return text - def itertext(el): + + def itertext(el): # pragma: no cover ' Reimplement Element.itertext for older python versions ' tag = el.tag if not isinstance(tag, util.string_type) and tag is not None: @@ -202,6 +245,7 @@ class Pattern(object): yield s if e.tail: yield e.tail + def get_stash(m): id = m.group(1) if id in stash: @@ -210,17 +254,14 @@ class Pattern(object): return value else: # An etree Element - return text content only - return ''.join(itertext(value)) + return ''.join(itertext(value)) return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text) class SimpleTextPattern(Pattern): """ Return a simple text of group(2) of a Pattern. """ def handleMatch(self, m): - text = m.group(2) - if text == util.INLINE_PLACEHOLDER_PREFIX: - return None - return text + return m.group(2) class EscapePattern(Pattern): @@ -231,7 +272,7 @@ class EscapePattern(Pattern): if char in self.markdown.ESCAPED_CHARS: return '%s%s%s' % (util.STX, ord(char), util.ETX) else: - return '\\%s' % char + return None class SimpleTagPattern(Pattern): @@ -240,7 +281,7 @@ class SimpleTagPattern(Pattern): of a Pattern. """ - def __init__ (self, pattern, tag): + def __init__(self, pattern, tag): Pattern.__init__(self, pattern) self.tag = tag @@ -252,13 +293,13 @@ class SimpleTagPattern(Pattern): class SubstituteTagPattern(SimpleTagPattern): """ Return an element of type `tag` with no children. """ - def handleMatch (self, m): + def handleMatch(self, m): return util.etree.Element(self.tag) class BacktickPattern(Pattern): """ Return a `<code>` element containing the matching text. """ - def __init__ (self, pattern): + def __init__(self, pattern): Pattern.__init__(self, pattern) self.tag = "code" @@ -279,12 +320,14 @@ class DoubleTagPattern(SimpleTagPattern): el1 = util.etree.Element(tag1) el2 = util.etree.SubElement(el1, tag2) el2.text = m.group(3) + if len(m.groups()) == 5: + el2.tail = m.group(4) return el1 class HtmlPattern(Pattern): """ Store raw inline html and return a placeholder. """ - def handleMatch (self, m): + def handleMatch(self, m): rawhtml = self.unescape(m.group(2)) place_holder = self.markdown.htmlStash.store(rawhtml) return place_holder @@ -293,8 +336,9 @@ class HtmlPattern(Pattern): """ Return unescaped text given text with an inline placeholder. """ try: stash = self.markdown.treeprocessors['inline'].stashed_nodes - except KeyError: + except KeyError: # pragma: no cover return text + def get_stash(m): id = m.group(1) value = stash.get(id) @@ -303,7 +347,7 @@ class HtmlPattern(Pattern): return self.markdown.serializer(value) except: return '\%s' % value - + return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text) @@ -323,7 +367,7 @@ class LinkPattern(Pattern): el.set("href", "") if title: - title = dequote(self.unescape(title)) + title = dequote(self.unescape(title)) el.set("title", title) return el @@ -344,35 +388,36 @@ class LinkPattern(Pattern): `username:password@host:port`. """ - url = url.replace(' ', '%20') if not self.markdown.safeMode: # Return immediately bipassing parsing. return url - + try: scheme, netloc, path, params, query, fragment = url = urlparse(url) - except ValueError: + except ValueError: # pragma: no cover # Bad url - so bad it couldn't be parsed. return '' - + locless_schemes = ['', 'mailto', 'news'] allowed_schemes = locless_schemes + ['http', 'https', 'ftp', 'ftps'] if scheme not in allowed_schemes: # Not a known (allowed) scheme. Not safe. return '' - - if netloc == '' and scheme not in locless_schemes: + + if netloc == '' and scheme not in locless_schemes: # pragma: no cover # This should not happen. Treat as suspect. return '' for part in url[2:]: if ":" in part: - # A colon in "path", "parameters", "query" or "fragment" is suspect. + # A colon in "path", "parameters", "query" + # or "fragment" is suspect. return '' # Url passes all tests. Return url as-is. return urlunparse(url) + class ImagePattern(LinkPattern): """ Return a img element from the given match. """ def handleMatch(self, m): @@ -396,6 +441,7 @@ class ImagePattern(LinkPattern): el.set('alt', self.unescape(truealt)) return el + class ReferencePattern(LinkPattern): """ Match to a stored reference and return link element. """ @@ -413,7 +459,7 @@ class ReferencePattern(LinkPattern): # Clean up linebreaks in id id = self.NEWLINE_CLEANUP_RE.sub(' ', id) - if not id in self.markdown.references: # ignore undefined refs + if id not in self.markdown.references: # ignore undefined refs return None href, title = self.markdown.references[id] @@ -454,6 +500,7 @@ class AutolinkPattern(Pattern): el.text = util.AtomicString(m.group(2)) return el + class AutomailPattern(Pattern): """ Return a mailto link Element given an automail link (`<foo@example.com>`). @@ -480,4 +527,3 @@ class AutomailPattern(Pattern): ord(letter) for letter in mailto]) el.set('href', mailto) return el - diff --git a/src/calibre/ebooks/markdown/odict.py b/src/calibre/ebooks/markdown/odict.py index 8089ece21a..584ad7c173 100644 --- a/src/calibre/ebooks/markdown/odict.py +++ b/src/calibre/ebooks/markdown/odict.py @@ -1,18 +1,13 @@ from __future__ import unicode_literals from __future__ import absolute_import from . import util - from copy import deepcopy -def iteritems_compat(d): - """Return an iterator over the (key, value) pairs of a dictionary. - Copied from `six` module.""" - return iter(getattr(d, _iteritems)()) class OrderedDict(dict): """ A dictionary that keeps its keys in the order in which they're inserted. - + Copied from Django's SortedDict with some modifications. """ @@ -87,11 +82,11 @@ class OrderedDict(dict): for key in self.keyOrder: yield self[key] - if util.PY3: + if util.PY3: # pragma: no cover items = _iteritems keys = _iterkeys values = _itervalues - else: + else: # pragma: no cover iteritems = _iteritems iterkeys = _iterkeys itervalues = _itervalues @@ -106,8 +101,8 @@ class OrderedDict(dict): return [self[k] for k in self.keyOrder] def update(self, dict_): - for k, v in iteritems_compat(dict_): - self[k] = v + for k in dict_: + self[k] = dict_[k] def setdefault(self, key, default): if key not in self: @@ -138,7 +133,9 @@ class OrderedDict(dict): Replaces the normal dict.__repr__ with a version that returns the keys in their Ordered order. """ - return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in iteritems_compat(self)]) + return '{%s}' % ', '.join( + ['%r: %r' % (k, v) for k, v in self._iteritems()] + ) def clear(self): super(OrderedDict, self).clear() diff --git a/src/calibre/ebooks/markdown/postprocessors.py b/src/calibre/ebooks/markdown/postprocessors.py index 5f3f032c15..2d4dcb589e 100644 --- a/src/calibre/ebooks/markdown/postprocessors.py +++ b/src/calibre/ebooks/markdown/postprocessors.py @@ -42,7 +42,7 @@ class Postprocessor(util.Processor): (possibly modified) string. """ - pass + pass # pragma: no cover class RawHtmlPostprocessor(Postprocessor): @@ -51,7 +51,7 @@ class RawHtmlPostprocessor(Postprocessor): def run(self, text): """ Iterate over html stash and restore "safe" html. """ for i in range(self.markdown.htmlStash.html_counter): - html, safe = self.markdown.htmlStash.rawHtmlBlocks[i] + html, safe = self.markdown.htmlStash.rawHtmlBlocks[i] if self.markdown.safeMode and not safe: if str(self.markdown.safeMode).lower() == 'escape': html = self.escape(html) @@ -59,12 +59,16 @@ class RawHtmlPostprocessor(Postprocessor): html = '' else: html = self.markdown.html_replacement_text - if self.isblocklevel(html) and (safe or not self.markdown.safeMode): - text = text.replace("<p>%s</p>" % - (self.markdown.htmlStash.get_placeholder(i)), - html + "\n") - text = text.replace(self.markdown.htmlStash.get_placeholder(i), - html) + if (self.isblocklevel(html) and + (safe or not self.markdown.safeMode)): + text = text.replace( + "<p>%s</p>" % + (self.markdown.htmlStash.get_placeholder(i)), + html + "\n" + ) + text = text.replace( + self.markdown.htmlStash.get_placeholder(i), html + ) return text def escape(self, html): @@ -88,7 +92,7 @@ class AndSubstitutePostprocessor(Postprocessor): """ Restore valid entities """ def run(self, text): - text = text.replace(util.AMP_SUBSTITUTE, "&") + text = text.replace(util.AMP_SUBSTITUTE, "&") return text diff --git a/src/calibre/ebooks/markdown/preprocessors.py b/src/calibre/ebooks/markdown/preprocessors.py index 72b2ed6f35..7ea4fcf9f5 100644 --- a/src/calibre/ebooks/markdown/preprocessors.py +++ b/src/calibre/ebooks/markdown/preprocessors.py @@ -3,7 +3,7 @@ PRE-PROCESSORS ============================================================================= Preprocessors work on source text before we start doing anything too -complicated. +complicated. """ from __future__ import absolute_import @@ -41,7 +41,7 @@ class Preprocessor(util.Processor): the (possibly modified) list of lines. """ - pass + pass # pragma: no cover class NormalizeWhitespace(Preprocessor): @@ -61,13 +61,14 @@ class HtmlBlockPreprocessor(Preprocessor): right_tag_patterns = ["</%s>", "%s>"] attrs_pattern = r""" - \s+(?P<attr>[^>"'/= ]+)=(?P<q>['"])(?P<value>.*?)(?P=q) # attr="value" - | # OR - \s+(?P<attr1>[^>"'/= ]+)=(?P<value1>[^> ]+) # attr=value - | # OR - \s+(?P<attr2>[^>"'/= ]+) # attr + \s+(?P<attr>[^>"'/= ]+)=(?P<q>['"])(?P<value>.*?)(?P=q) # attr="value" + | # OR + \s+(?P<attr1>[^>"'/= ]+)=(?P<value1>[^> ]+) # attr=value + | # OR + \s+(?P<attr2>[^>"'/= ]+) # attr """ - left_tag_pattern = r'^\<(?P<tag>[^> ]+)(?P<attrs>(%s)*)\s*\/?\>?' % attrs_pattern + left_tag_pattern = r'^\<(?P<tag>[^> ]+)(?P<attrs>(%s)*)\s*\/?\>?' % \ + attrs_pattern attrs_re = re.compile(attrs_pattern, re.VERBOSE) left_tag_re = re.compile(left_tag_pattern, re.VERBOSE) markdown_in_raw = False @@ -87,7 +88,9 @@ class HtmlBlockPreprocessor(Preprocessor): attrs[ma.group('attr').strip()] = "" elif ma.group('attr1'): if ma.group('value1'): - attrs[ma.group('attr1').strip()] = ma.group('value1') + attrs[ma.group('attr1').strip()] = ma.group( + 'value1' + ) else: attrs[ma.group('attr1').strip()] = "" elif ma.group('attr2'): @@ -102,7 +105,7 @@ class HtmlBlockPreprocessor(Preprocessor): i = block.find(rtag, start_index) if i == -1: return -1 - j = block.find(ltag, start_index) + j = block.find(ltag, start_index) # if no ltag, or rtag found before another ltag, return index if (j > i or j == -1): return i + len(rtag) @@ -111,27 +114,28 @@ class HtmlBlockPreprocessor(Preprocessor): j = block.find('>', j) start_index = self._recursive_tagfind(ltag, rtag, j + 1, block) if start_index == -1: - # HTML potentially malformed- ltag has no corresponding + # HTML potentially malformed- ltag has no corresponding # rtag return -1 def _get_right_tag(self, left_tag, left_index, block): for p in self.right_tag_patterns: tag = p % left_tag - i = self._recursive_tagfind("<%s" % left_tag, tag, left_index, block) + i = self._recursive_tagfind( + "<%s" % left_tag, tag, left_index, block + ) if i > 2: return tag.lstrip("<").rstrip(">"), i return block.rstrip()[-left_index:-1].lower(), len(block) - + def _equal_tags(self, left_tag, right_tag): - if left_tag[0] in ['?', '@', '%']: # handle PHP, etc. + if left_tag[0] in ['?', '@', '%']: # handle PHP, etc. return True if ("/" + left_tag) == right_tag: return True if (right_tag == "--" and left_tag == "--"): return True - elif left_tag == right_tag[1:] \ - and right_tag[0] == "/": + elif left_tag == right_tag[1:] and right_tag[0] == "/": return True else: return False @@ -139,6 +143,49 @@ class HtmlBlockPreprocessor(Preprocessor): def _is_oneliner(self, tag): return (tag in ['hr', 'hr/']) + def _stringindex_to_listindex(self, stringindex, items): + """ + Same effect as concatenating the strings in items, + finding the character to which stringindex refers in that string, + and returning the index of the item in which that character resides. + """ + items.append('dummy') + i, count = 0, 0 + while count <= stringindex: + count += len(items[i]) + i += 1 + return i - 1 + + def _nested_markdown_in_html(self, items): + """Find and process html child elements of the given element block.""" + for i, item in enumerate(items): + if self.left_tag_re.match(item): + left_tag, left_index, attrs = \ + self._get_left_tag(''.join(items[i:])) + right_tag, data_index = self._get_right_tag( + left_tag, left_index, ''.join(items[i:])) + right_listindex = \ + self._stringindex_to_listindex(data_index, items[i:]) + i + if 'markdown' in attrs.keys(): + items[i] = items[i][left_index:] # remove opening tag + placeholder = self.markdown.htmlStash.store_tag( + left_tag, attrs, i + 1, right_listindex + 1) + items.insert(i, placeholder) + if len(items) - right_listindex <= 1: # last nest, no tail + right_listindex -= 1 + items[right_listindex] = items[right_listindex][ + :-len(right_tag) - 2] # remove closing tag + else: # raw html + if len(items) - right_listindex <= 1: # last element + right_listindex -= 1 + if right_listindex <= i: + right_listindex = i + 1 + placeholder = self.markdown.htmlStash.store('\n\n'.join( + items[i:right_listindex])) + del items[i:right_listindex] + items.insert(i, placeholder) + return items + def run(self, lines): text = "\n".join(lines) new_blocks = [] @@ -146,7 +193,7 @@ class HtmlBlockPreprocessor(Preprocessor): items = [] left_tag = '' right_tag = '' - in_tag = False # flag + in_tag = False # flag while text: block = text[0] @@ -160,24 +207,21 @@ class HtmlBlockPreprocessor(Preprocessor): if not in_tag: if block.startswith("<") and len(block.strip()) > 1: - if block[1] == "!": + if block[1:4] == "!--": # is a comment block - left_tag, left_index, attrs = "--", 2, {} + left_tag, left_index, attrs = "--", 2, {} else: left_tag, left_index, attrs = self._get_left_tag(block) - right_tag, data_index = self._get_right_tag(left_tag, + right_tag, data_index = self._get_right_tag(left_tag, left_index, block) # keep checking conditions below and maybe just append - - if data_index < len(block) \ - and (util.isBlockLevel(left_tag) - or left_tag == '--'): + + if data_index < len(block) and (util.isBlockLevel(left_tag) or left_tag == '--'): text.insert(0, block[data_index:]) block = block[:data_index] - if not (util.isBlockLevel(left_tag) \ - or block[1] in ["!", "?", "@", "%"]): + if not (util.isBlockLevel(left_tag) or block[1] in ["!", "?", "@", "%"]): new_blocks.append(block) continue @@ -186,35 +230,30 @@ class HtmlBlockPreprocessor(Preprocessor): continue if block.rstrip().endswith(">") \ - and self._equal_tags(left_tag, right_tag): + and self._equal_tags(left_tag, right_tag): if self.markdown_in_raw and 'markdown' in attrs.keys(): - start = re.sub(r'\smarkdown(=[\'"]?[^> ]*[\'"]?)?', - '', block[:left_index]) - end = block[-len(right_tag)-2:] - block = block[left_index:-len(right_tag)-2] - new_blocks.append( - self.markdown.htmlStash.store(start)) - new_blocks.append(block) - new_blocks.append( - self.markdown.htmlStash.store(end)) + block = block[left_index:-len(right_tag) - 2] + new_blocks.append(self.markdown.htmlStash. + store_tag(left_tag, attrs, 0, 2)) + new_blocks.extend([block]) else: new_blocks.append( self.markdown.htmlStash.store(block.strip())) continue - else: + else: # if is block level tag and is not complete - - if util.isBlockLevel(left_tag) or left_tag == "--" \ - and not block.rstrip().endswith(">"): + if (not self._equal_tags(left_tag, right_tag)) and \ + (util.isBlockLevel(left_tag) or left_tag == "--"): items.append(block.strip()) in_tag = True else: new_blocks.append( - self.markdown.htmlStash.store(block.strip())) - + self.markdown.htmlStash.store(block.strip()) + ) continue - new_blocks.append(block) + else: + new_blocks.append(block) else: items.append(block) @@ -223,7 +262,7 @@ class HtmlBlockPreprocessor(Preprocessor): if self._equal_tags(left_tag, right_tag): # if find closing tag - + if data_index < len(block): # we have more text after right_tag items[-1] = block[:data_index] @@ -231,16 +270,21 @@ class HtmlBlockPreprocessor(Preprocessor): in_tag = False if self.markdown_in_raw and 'markdown' in attrs.keys(): - start = re.sub(r'\smarkdown(=[\'"]?[^> ]*[\'"]?)?', - '', items[0][:left_index]) items[0] = items[0][left_index:] - end = items[-1][-len(right_tag)-2:] - items[-1] = items[-1][:-len(right_tag)-2] - new_blocks.append( - self.markdown.htmlStash.store(start)) - new_blocks.extend(items) - new_blocks.append( - self.markdown.htmlStash.store(end)) + items[-1] = items[-1][:-len(right_tag) - 2] + if items[len(items) - 1]: # not a newline/empty string + right_index = len(items) + 3 + else: + right_index = len(items) + 2 + new_blocks.append(self.markdown.htmlStash.store_tag( + left_tag, attrs, 0, right_index)) + placeholderslen = len(self.markdown.htmlStash.tag_data) + new_blocks.extend( + self._nested_markdown_in_html(items)) + nests = len(self.markdown.htmlStash.tag_data) - \ + placeholderslen + self.markdown.htmlStash.tag_data[-1 - nests][ + 'right_index'] += nests - 2 else: new_blocks.append( self.markdown.htmlStash.store('\n\n'.join(items))) @@ -248,21 +292,23 @@ class HtmlBlockPreprocessor(Preprocessor): if items: if self.markdown_in_raw and 'markdown' in attrs.keys(): - start = re.sub(r'\smarkdown(=[\'"]?[^> ]*[\'"]?)?', - '', items[0][:left_index]) items[0] = items[0][left_index:] - end = items[-1][-len(right_tag)-2:] - items[-1] = items[-1][:-len(right_tag)-2] + items[-1] = items[-1][:-len(right_tag) - 2] + if items[len(items) - 1]: # not a newline/empty string + right_index = len(items) + 3 + else: + right_index = len(items) + 2 new_blocks.append( - self.markdown.htmlStash.store(start)) - new_blocks.extend(items) - if end.strip(): - new_blocks.append( - self.markdown.htmlStash.store(end)) + self.markdown.htmlStash.store_tag( + left_tag, attrs, 0, right_index)) + placeholderslen = len(self.markdown.htmlStash.tag_data) + new_blocks.extend(self._nested_markdown_in_html(items)) + nests = len(self.markdown.htmlStash.tag_data) - placeholderslen + self.markdown.htmlStash.tag_data[-1 - nests][ + 'right_index'] += nests - 2 else: new_blocks.append( self.markdown.htmlStash.store('\n\n'.join(items))) - #new_blocks.append(self.markdown.htmlStash.store('\n\n'.join(items))) new_blocks.append('\n') new_text = "\n\n".join(new_blocks) @@ -273,11 +319,13 @@ class ReferencePreprocessor(Preprocessor): """ Remove reference definitions from text and store for later use. """ TITLE = r'[ ]*(\"(.*)\"|\'(.*)\'|\((.*)\))[ ]*' - RE = re.compile(r'^[ ]{0,3}\[([^\]]*)\]:\s*([^ ]*)[ ]*(%s)?$' % TITLE, re.DOTALL) + RE = re.compile( + r'^[ ]{0,3}\[([^\]]*)\]:\s*([^ ]*)[ ]*(%s)?$' % TITLE, re.DOTALL + ) TITLE_RE = re.compile(r'^%s$' % TITLE) - def run (self, lines): - new_text = []; + def run(self, lines): + new_text = [] while lines: line = lines.pop(0) m = self.RE.match(line) @@ -295,4 +343,4 @@ class ReferencePreprocessor(Preprocessor): else: new_text.append(line) - return new_text #+ "\n" + return new_text # + "\n" diff --git a/src/calibre/ebooks/markdown/serializers.py b/src/calibre/ebooks/markdown/serializers.py index b19d61c93d..1e8d9dd288 100644 --- a/src/calibre/ebooks/markdown/serializers.py +++ b/src/calibre/ebooks/markdown/serializers.py @@ -42,9 +42,9 @@ from __future__ import unicode_literals from . import util ElementTree = util.etree.ElementTree QName = util.etree.QName -if hasattr(util.etree, 'test_comment'): +if hasattr(util.etree, 'test_comment'): # pragma: no cover Comment = util.etree.test_comment -else: +else: # pragma: no cover Comment = util.etree.Comment PI = util.etree.PI ProcessingInstruction = util.etree.ProcessingInstruction @@ -56,7 +56,7 @@ HTML_EMPTY = ("area", "base", "basefont", "br", "col", "frame", "hr", try: HTML_EMPTY = set(HTML_EMPTY) -except NameError: +except NameError: # pragma: no cover pass _namespace_map = { @@ -73,17 +73,19 @@ _namespace_map = { } -def _raise_serialization_error(text): +def _raise_serialization_error(text): # pragma: no cover raise TypeError( "cannot serialize %r (type %s)" % (text, type(text).__name__) ) + def _encode(text, encoding): try: return text.encode(encoding, "xmlcharrefreplace") - except (TypeError, AttributeError): + except (TypeError, AttributeError): # pragma: no cover _raise_serialization_error(text) + def _escape_cdata(text): # escape character data try: @@ -97,7 +99,7 @@ def _escape_cdata(text): if ">" in text: text = text.replace(">", ">") return text - except (TypeError, AttributeError): + except (TypeError, AttributeError): # pragma: no cover _raise_serialization_error(text) @@ -115,9 +117,10 @@ def _escape_attrib(text): if "\n" in text: text = text.replace("\n", " ") return text - except (TypeError, AttributeError): + except (TypeError, AttributeError): # pragma: no cover _raise_serialization_error(text) + def _escape_attrib_html(text): # escape attribute value try: @@ -130,7 +133,7 @@ def _escape_attrib_html(text): if "\"" in text: text = text.replace("\"", """) return text - except (TypeError, AttributeError): + except (TypeError, AttributeError): # pragma: no cover _raise_serialization_error(text) @@ -152,7 +155,7 @@ def _serialize_html(write, elem, qnames, namespaces, format): write("<" + tag) items = elem.items() if items or namespaces: - items.sort() # lexical order + items = sorted(items) # lexical order for k, v in items: if isinstance(k, QName): k = k.text @@ -167,28 +170,28 @@ def _serialize_html(write, elem, qnames, namespaces, format): write(" %s=\"%s\"" % (qnames[k], v)) if namespaces: items = namespaces.items() - items.sort(key=lambda x: x[1]) # sort on prefix + items.sort(key=lambda x: x[1]) # sort on prefix for v, k in items: if k: k = ":" + k write(" xmlns%s=\"%s\"" % (k, _escape_attrib(v))) - if format == "xhtml" and tag in HTML_EMPTY: + if format == "xhtml" and tag.lower() in HTML_EMPTY: write(" />") else: write(">") - tag = tag.lower() if text: - if tag == "script" or tag == "style": + if tag.lower() in ["script", "style"]: write(text) else: write(_escape_cdata(text)) for e in elem: _serialize_html(write, e, qnames, None, format) - if tag not in HTML_EMPTY: + if tag.lower() not in HTML_EMPTY: write("</" + tag + ">") if elem.tail: write(_escape_cdata(elem.tail)) + def _write_html(root, encoding=None, default_namespace=None, @@ -233,7 +236,7 @@ def _namespaces(elem, default_namespace=None): if prefix: qnames[qname] = "%s:%s" % (prefix, tag) else: - qnames[qname] = tag # default element + qnames[qname] = tag # default element else: if default_namespace: raise ValueError( @@ -241,14 +244,14 @@ def _namespaces(elem, default_namespace=None): "default_namespace option" ) qnames[qname] = qname - except TypeError: + except TypeError: # pragma: no cover _raise_serialization_error(qname) # populate qname and namespaces table try: iterate = elem.iter except AttributeError: - iterate = elem.getiterator # cET compatibility + iterate = elem.getiterator # cET compatibility for elem in iterate(): tag = elem.tag if isinstance(tag, QName) and tag.text not in qnames: @@ -270,8 +273,10 @@ def _namespaces(elem, default_namespace=None): add_qname(text.text) return qnames, namespaces + def to_html_string(element): return _write_html(ElementTree(element).getroot(), format="html") + def to_xhtml_string(element): return _write_html(ElementTree(element).getroot(), format="xhtml") diff --git a/src/calibre/ebooks/markdown/treeprocessors.py b/src/calibre/ebooks/markdown/treeprocessors.py index e6d3dc9381..d06f192885 100644 --- a/src/calibre/ebooks/markdown/treeprocessors.py +++ b/src/calibre/ebooks/markdown/treeprocessors.py @@ -34,11 +34,11 @@ class Treeprocessor(util.Processor): def run(self, root): """ Subclasses of Treeprocessor should implement a `run` method, which - takes a root ElementTree. This method can return another ElementTree - object, and the existing root ElementTree will be replaced, or it can + takes a root ElementTree. This method can return another ElementTree + object, and the existing root ElementTree will be replaced, or it can modify the current tree and return None. """ - pass + pass # pragma: no cover class InlineProcessor(Treeprocessor): @@ -53,6 +53,7 @@ class InlineProcessor(Treeprocessor): + len(self.__placeholder_suffix) self.__placeholder_re = util.INLINE_PLACEHOLDER_RE self.markdown = md + self.inlinePatterns = md.inlinePatterns def __makePlaceholder(self, type): """ Generate a placeholder """ @@ -70,7 +71,7 @@ class InlineProcessor(Treeprocessor): * index: index, from which we start search Returns: placeholder id and string index, after the found placeholder. - + """ m = self.__placeholder_re.search(data, index) if m: @@ -99,9 +100,9 @@ class InlineProcessor(Treeprocessor): """ if not isinstance(data, util.AtomicString): startIndex = 0 - while patternIndex < len(self.markdown.inlinePatterns): + while patternIndex < len(self.inlinePatterns): data, matched, startIndex = self.__applyPattern( - self.markdown.inlinePatterns.value_for_index(patternIndex), + self.inlinePatterns.value_for_index(patternIndex), data, patternIndex, startIndex) if not matched: patternIndex += 1 @@ -128,11 +129,10 @@ class InlineProcessor(Treeprocessor): text = subnode.tail subnode.tail = None - childResult = self.__processPlaceholders(text, subnode) + childResult = self.__processPlaceholders(text, subnode, isText) if not isText and node is not subnode: - pos = node.getchildren().index(subnode) - node.remove(subnode) + pos = list(node).index(subnode) + 1 else: pos = 0 @@ -140,7 +140,7 @@ class InlineProcessor(Treeprocessor): for newChild in childResult: node.insert(pos, newChild) - def __processPlaceholders(self, data, parent): + def __processPlaceholders(self, data, parent, isText=True): """ Process string with placeholders and generate ElementTree tree. @@ -150,7 +150,7 @@ class InlineProcessor(Treeprocessor): * parent: Element, which contains processing inline data Returns: list with ElementTree elements with applied inline patterns. - + """ def linkText(text): if text: @@ -159,6 +159,11 @@ class InlineProcessor(Treeprocessor): result[-1].tail += text else: result[-1].tail = text + elif not isText: + if parent.tail: + parent.tail += text + else: + parent.tail = text else: if parent.text: parent.text += text @@ -178,15 +183,17 @@ class InlineProcessor(Treeprocessor): text = data[strartIndex:index] linkText(text) - if not isString(node): # it's Element - for child in [node] + node.getchildren(): + if not isString(node): # it's Element + for child in [node] + list(node): if child.tail: if child.tail.strip(): - self.__processElementText(node, child,False) + self.__processElementText( + node, child, False + ) if child.text: if child.text.strip(): self.__processElementText(child, child) - else: # it's just a string + else: # it's just a string linkText(node) strartIndex = phEndIndex continue @@ -194,7 +201,7 @@ class InlineProcessor(Treeprocessor): strartIndex = phEndIndex result.append(node) - else: # wrong placeholder + else: # wrong placeholder end = index + len(self.__placeholder_prefix) linkText(data[strartIndex:end]) strartIndex = end @@ -237,14 +244,16 @@ class InlineProcessor(Treeprocessor): if not isString(node): if not isinstance(node.text, util.AtomicString): # We need to process current node too - for child in [node] + node.getchildren(): + for child in [node] + list(node): if not isString(node): - if child.text: - child.text = self.__handleInline(child.text, - patternIndex + 1) + if child.text: + child.text = self.__handleInline( + child.text, patternIndex + 1 + ) if child.tail: - child.tail = self.__handleInline(child.tail, - patternIndex) + child.tail = self.__handleInline( + child.tail, patternIndex + ) placeholder = self.__stashNode(node, pattern.type()) @@ -257,8 +266,8 @@ class InlineProcessor(Treeprocessor): Iterate over ElementTree, find elements with inline tag, apply inline patterns and append newly created Elements to tree. If you don't - want to process your data with inline paterns, instead of normal string, - use subclass AtomicString: + want to process your data with inline paterns, instead of normal + string, use subclass AtomicString: node.text = markdown.AtomicString("This will not be processed.") @@ -276,47 +285,49 @@ class InlineProcessor(Treeprocessor): while stack: currElement = stack.pop() insertQueue = [] - for child in currElement.getchildren(): - if child.text and not isinstance(child.text, util.AtomicString): + for child in currElement: + if child.text and not isinstance( + child.text, util.AtomicString + ): text = child.text child.text = None - lst = self.__processPlaceholders(self.__handleInline( - text), child) + lst = self.__processPlaceholders( + self.__handleInline(text), child + ) stack += lst insertQueue.append((child, lst)) if child.tail: tail = self.__handleInline(child.tail) dumby = util.etree.Element('d') - tailResult = self.__processPlaceholders(tail, dumby) - if dumby.text: - child.tail = dumby.text - else: - child.tail = None - pos = currElement.getchildren().index(child) + 1 + child.tail = None + tailResult = self.__processPlaceholders(tail, dumby, False) + if dumby.tail: + child.tail = dumby.tail + pos = list(currElement).index(child) + 1 tailResult.reverse() for newChild in tailResult: currElement.insert(pos, newChild) - if child.getchildren(): + if len(child): stack.append(child) for element, lst in insertQueue: if self.markdown.enable_attributes: if element.text and isString(element.text): - element.text = \ - inlinepatterns.handleAttributes(element.text, - element) + element.text = inlinepatterns.handleAttributes( + element.text, element + ) i = 0 for newChild in lst: if self.markdown.enable_attributes: # Processing attributes if newChild.tail and isString(newChild.tail): - newChild.tail = \ - inlinepatterns.handleAttributes(newChild.tail, - element) + newChild.tail = inlinepatterns.handleAttributes( + newChild.tail, element + ) if newChild.text and isString(newChild.text): - newChild.text = \ - inlinepatterns.handleAttributes(newChild.text, - newChild) + newChild.text = inlinepatterns.handleAttributes( + newChild.text, newChild + ) element.insert(i, newChild) i += 1 return tree @@ -357,4 +368,4 @@ class PrettifyTreeprocessor(Treeprocessor): pres = root.getiterator('pre') for pre in pres: if len(pre) and pre[0].tag == 'code': - pre[0].text = pre[0].text.rstrip() + '\n' + pre[0].text = util.AtomicString(pre[0].text.rstrip() + '\n') diff --git a/src/calibre/ebooks/markdown/util.py b/src/calibre/ebooks/markdown/util.py index f07da1e111..d3d48f0999 100644 --- a/src/calibre/ebooks/markdown/util.py +++ b/src/calibre/ebooks/markdown/util.py @@ -10,14 +10,14 @@ Python 3 Stuff """ PY3 = sys.version_info[0] == 3 -if PY3: +if PY3: # pragma: no cover string_type = str text_type = str int2str = chr -else: - string_type = basestring - text_type = unicode - int2str = unichr +else: # pragma: no cover + string_type = basestring # noqa + text_type = unicode # noqa + int2str = unichr # noqa """ @@ -25,12 +25,16 @@ Constants you might want to modify ----------------------------------------------------------------------------- """ -BLOCK_LEVEL_ELEMENTS = re.compile("^(p|div|h[1-6]|blockquote|pre|table|dl|ol|ul" - "|script|noscript|form|fieldset|iframe|math" - "|hr|hr/|style|li|dt|dd|thead|tbody" - "|tr|th|td|section|footer|header|group|figure" - "|figcaption|aside|article|canvas|output" - "|progress|video)$", re.IGNORECASE) + +BLOCK_LEVEL_ELEMENTS = re.compile( + "^(p|div|h[1-6]|blockquote|pre|table|dl|ol|ul" + "|script|noscript|form|fieldset|iframe|math" + "|hr|hr/|style|li|dt|dd|thead|tbody" + "|tr|th|td|section|footer|header|group|figure" + "|figcaption|aside|article|canvas|output" + "|progress|video|nav)$", + re.IGNORECASE +) # Placeholders STX = '\u0002' # Use STX ("Start of text") for start-of-placeholder ETX = '\u0003' # Use ETX ("End of text") for end-of-placeholder @@ -38,30 +42,36 @@ INLINE_PLACEHOLDER_PREFIX = STX+"klzzwxh:" INLINE_PLACEHOLDER = INLINE_PLACEHOLDER_PREFIX + "%s" + ETX INLINE_PLACEHOLDER_RE = re.compile(INLINE_PLACEHOLDER % r'([0-9]+)') AMP_SUBSTITUTE = STX+"amp"+ETX +HTML_PLACEHOLDER = STX + "wzxhzdk:%s" + ETX +HTML_PLACEHOLDER_RE = re.compile(HTML_PLACEHOLDER % r'([0-9]+)') +TAG_PLACEHOLDER = STX + "hzzhzkh:%s" + ETX + """ Constants you probably do not need to change ----------------------------------------------------------------------------- """ -RTL_BIDI_RANGES = ( ('\u0590', '\u07FF'), - # Hebrew (0590-05FF), Arabic (0600-06FF), - # Syriac (0700-074F), Arabic supplement (0750-077F), - # Thaana (0780-07BF), Nko (07C0-07FF). - ('\u2D30', '\u2D7F'), # Tifinagh - ) +RTL_BIDI_RANGES = ( + ('\u0590', '\u07FF'), + # Hebrew (0590-05FF), Arabic (0600-06FF), + # Syriac (0700-074F), Arabic supplement (0750-077F), + # Thaana (0780-07BF), Nko (07C0-07FF). + ('\u2D30', '\u2D7F') # Tifinagh +) # Extensions should use "markdown.util.etree" instead of "etree" (or do `from # markdown.util import etree`). Do not import it by yourself. -try: # Is the C implemenation of ElementTree available? +try: # pragma: no cover + # Is the C implementation of ElementTree available? import xml.etree.cElementTree as etree from xml.etree.ElementTree import Comment # Serializers (including ours) test with non-c Comment etree.test_comment = Comment if etree.VERSION < "1.0.5": raise RuntimeError("cElementTree version 1.0.5 or higher is required.") -except (ImportError, RuntimeError): +except (ImportError, RuntimeError): # pragma: no cover # Use the Python implementation of ElementTree? import xml.etree.ElementTree as etree if etree.VERSION < "1.1": @@ -81,11 +91,32 @@ def isBlockLevel(tag): # Some ElementTree tags are not strings, so return False. return False + +def parseBoolValue(value, fail_on_errors=True, preserve_none=False): + """Parses a string representing bool value. If parsing was successful, + returns True or False. If preserve_none=True, returns True, False, + or None. If parsing was not successful, raises ValueError, or, if + fail_on_errors=False, returns None.""" + if not isinstance(value, string_type): + if preserve_none and value is None: + return value + return bool(value) + elif preserve_none and value.lower() == 'none': + return None + elif value.lower() in ('true', 'yes', 'y', 'on', '1'): + return True + elif value.lower() in ('false', 'no', 'n', 'off', '0', 'none'): + return False + elif fail_on_errors: + raise ValueError('Cannot parse bool value: %r' % value) + + """ MISC AUXILIARY CLASSES ============================================================================= """ + class AtomicString(text_type): """A string which should not be further processed.""" pass @@ -103,10 +134,12 @@ class HtmlStash(object): in the beginning and replace with place-holders. """ - def __init__ (self): + def __init__(self): """ Create a HtmlStash. """ - self.html_counter = 0 # for counting inline html segments - self.rawHtmlBlocks=[] + self.html_counter = 0 # for counting inline html segments + self.rawHtmlBlocks = [] + self.tag_counter = 0 + self.tag_data = [] # list of dictionaries in the order tags appear def store(self, html, safe=False): """ @@ -132,5 +165,13 @@ class HtmlStash(object): self.rawHtmlBlocks = [] def get_placeholder(self, key): - return "%swzxhzdk:%d%s" % (STX, key, ETX) + return HTML_PLACEHOLDER % key + def store_tag(self, tag, attrs, left_index, right_index): + """Store tag data and return a placeholder.""" + self.tag_data.append({'tag': tag, 'attrs': attrs, + 'left_index': left_index, + 'right_index': right_index}) + placeholder = TAG_PLACEHOLDER % str(self.tag_counter) + self.tag_counter += 1 # equal to the tag's index in self.tag_data + return placeholder diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index ba41378deb..a397dfaa03 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -98,8 +98,8 @@ def convert_basic(txt, title='', epub_split_size_kb=0): def convert_markdown(txt, title='', extensions=('footnotes', 'tables', 'toc')): from calibre.ebooks.conversion.plugins.txt_input import MD_EXTENSIONS from calibre.ebooks.markdown import Markdown - extensions = [x.lower() for x in extensions if x.lower() in MD_EXTENSIONS] - md = Markdown(extensions=extensions, safe_mode=False) + extensions = ['calibre.ebooks.markdown.extensions.' + x.lower() for x in extensions if x.lower() in MD_EXTENSIONS] + md = Markdown(extensions=extensions) return HTML_TEMPLATE % (title, md.convert(txt)) def convert_textile(txt, title=''): diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 9c68a91f41..3fce5203f7 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -233,6 +233,11 @@ def test_terminal(): del readline print ('readline and curses OK!') +def test_markdown(): + from calibre.ebooks.markdown import Markdown + Markdown(extensions=['extra']) + print('Markdown OK!') + def test(): if iswindows: test_dlls() @@ -256,6 +261,7 @@ def test(): test_netifaces() test_psutil() test_podofo() + test_markdown() if iswindows: test_wpd() test_winutil()