"""loads Markdown files, converts each one to HTML and parses the HTML into an ElementTree structure. The collection of ElementTrees are further parsed to generate a table of contents structure, and are manipulated to replace various markdown-generated HTML with specific Mako tags before being written to Mako templates, which then re-access the table of contents structure at runtime. Much thanks to Alexey Shamrin, who came up with the original idea and did all the heavy Markdown/Elementtree lifting for this module.""" import sys, re, os from toc import TOCElement try: import xml.etree.ElementTree as et except: try: import elementtree.ElementTree as et except: raise "This module requires ElementTree to run (http://effbot.org/zone/element-index.htm)" import markdown def dump_tree(elem, stream): if elem.tag.startswith('MAKO:'): dump_mako_tag(elem, stream) else: if elem.tag != 'html': if len(elem.attrib): stream.write("<%s %s>" % (elem.tag, " ".join(["%s=%s" % (key, repr(val)) for key, val in elem.attrib.iteritems()]))) else: stream.write("<%s>" % elem.tag) if elem.text: stream.write(elem.text) for child in elem: dump_tree(child, stream) if child.tail: stream.write(child.tail) stream.write("</%s>" % elem.tag) def dump_mako_tag(elem, stream): tag = elem.tag[5:] params = ','.join(['%s=%s' % i for i in elem.items()]) stream.write('<%%call expr="%s(%s)">' % (tag, params)) if elem.text: stream.write(elem.text) for n in elem: dump_tree(n, stream) if n.tail: stream.write(n.tail) stream.write("</%call>") def create_toc(filename, tree, tocroot): title = [None] current = [tocroot] level = [0] def process(tree): while True: i = find_header_index(tree) if i is None: return node = tree[i] taglevel = int(node.tag[1]) start, end = i, end_of_header(tree, taglevel, i+1) content = tree[start+1:end] description = node.text.strip() if title[0] is None: title[0] = description name = node.get('name') if name is None: name = description.split()[0].lower() taglevel = node.tag[1] if taglevel > level[0]: current[0] = TOCElement(filename, name, description, current[0]) elif taglevel == level[0]: current[0] = TOCElement(filename, name, description, current[0].parent) else: current[0] = TOCElement(filename, name, description, current[0].parent.parent) level[0] = taglevel tag = et.Element("MAKO:formatting.section", path=repr(current[0].path), paged='paged', extension='extension', toc='toc') tag.text = (node.tail or "") + '\n' tag.tail = '\n' tag[:] = content tree[start:end] = [tag] process(tag) process(tree) return (title[0], tocroot.get_by_file(filename)) def index(parent, item): for n, i in enumerate(parent): if i is item: return n def find_header_index(tree): for i, node in enumerate(tree): if is_header(node): return i def is_header(node): t = node.tag return (isinstance(t, str) and len(t) == 2 and t[0] == 'h' and t[1] in '123456789') def end_of_header(tree, level, start): for i, node in enumerate(tree[start:]): if is_header(node) and int(node.tag[1]) <= level: return start + i return len(tree) def process_rel_href(tree): parent = get_parent_map(tree) for a in tree.findall('.//a'): m = re.match(r'(bold)?rel\:(.+)', a.get('href')) if m: (bold, path) = m.group(1,2) text = a.text if text == path: tag = et.Element("MAKO:nav.toclink", path=repr(path), extension='extension', paged='paged', toc='toc') else: tag = et.Element("MAKO:nav.toclink", path=repr(path), description=repr(text), extension='extension', paged='paged', toc='toc') a_parent = parent[a] if bold: bold = et.Element('strong') bold.tail = a.tail bold.append(tag) a_parent[index(a_parent, a)] = bold else: tag.tail = a.tail a_parent[index(a_parent, a)] = tag def replace_pre_with_mako(tree): parents = get_parent_map(tree) for precode in tree.findall('.//pre/code'): reg = re.compile(r'\{(python|mako|html|ini)(?: title="(.*?)"){0,1}\}(.*)', re.S) m = reg.match(precode[0].text.lstrip()) if m: code = m.group(1) title = m.group(2) precode[0].text = m.group(3) else: code = title = None tag = et.Element("MAKO:formatting.code") if code: tag.attrib["syntaxtype"] = repr(code) if title: tag.attrib["title"] = repr(title) tag.text = precode.text [tag.append(x) for x in precode] pre = parents[precode] tag.tail = pre.tail pre_parent = parents[pre] pre_parent[reverse_parent(pre_parent, pre)] = tag def safety_code(tree): parents = get_parent_map(tree) for code in tree.findall('.//code'): tag = et.Element('%text') if parents[code].tag != 'pre': tag.attrib["filter"] = "h" tag.text = code.text code.append(tag) code.text = "" def reverse_parent(parent, item): for n, i in enumerate(parent): if i is item: return n def get_parent_map(tree): return dict([(c, p) for p in tree.getiterator() for c in p]) def header(toc, title, filename): return """# -*- coding: utf-8 -*- <%%inherit file="content_layout.html"/> <%%page args="toc, extension, paged"/> <%%namespace name="formatting" file="formatting.html"/> <%%namespace name="nav" file="nav.html"/> <%%def name="title()">%s - %s</%%def> <%%! filename = '%s' %%> ## This file is generated. Edit the .txt files instead of this one. """ % (toc.root.doctitle, title, filename) class utf8stream(object): def __init__(self, stream): self.stream = stream def write(self, str): self.stream.write(str.encode('utf8')) def parse_markdown_files(toc, files): for inname in files: infile = 'content/%s.txt' % inname if not os.access(infile, os.F_OK): continue html = markdown.markdown(file(infile).read()) tree = et.fromstring("<html>" + html + "</html>") (title, toc_element) = create_toc(inname, tree, toc) safety_code(tree) replace_pre_with_mako(tree) process_rel_href(tree) outname = 'output/%s.html' % inname print infile, '->', outname outfile = utf8stream(file(outname, 'w')) outfile.write(header(toc, title, inname)) dump_tree(tree, outfile)