doc/mkhtml.py - master - valemobil/lxml

doc/mkhtml.py
332 строки13 KB
Stefan Behnel
Update several links in the docs.
14 дек 2021, 01:23
14 дек 2021, 01:232b9e047
from __future__ import absolute_import

from docstructure import SITE_STRUCTURE, HREF_MAP, BASENAME_MAP
from lxml.etree import (parse, fromstring, ElementTree,
                        Element, SubElement, XPath, XML)
import glob
import hashlib
import os
import re
import sys
import copy
import shutil
import textwrap
import subprocess

from io import open as open_file

RST2HTML_OPTIONS = " ".join([
    '--no-toc-backlinks',
    '--strip-comments',
    '--language en',
    '--date',
    ])

XHTML_NS = 'http://www.w3.org/1999/xhtml'
htmlnsmap = {"h": XHTML_NS}

find_head = XPath("/h:html/h:head[1]", namespaces=htmlnsmap)
find_body = XPath("/h:html/h:body[1]", namespaces=htmlnsmap)
find_title = XPath("/h:html/h:head/h:title/text()", namespaces=htmlnsmap)
find_title_tag = XPath("/h:html/h:head/h:title", namespaces=htmlnsmap)
find_headings = XPath("//h:h1[not(@class)]//text()", namespaces=htmlnsmap)
find_heading_tag = XPath("//h:h1[@class = 'title'][1]", namespaces=htmlnsmap)
find_menu = XPath("//h:ul[@id=$name]", namespaces=htmlnsmap)
find_page_end = XPath("/h:html/h:body/h:div[last()]", namespaces=htmlnsmap)

find_words = re.compile(r'(\w+)').findall
replace_invalid = re.compile(r'[-_/.\s\\]').sub


def make_menu_section_head(section, menuroot):
    section_id = section + '-section'
    section_head = menuroot.xpath("//ul[@id=$section]/li", section=section_id)
    if not section_head:
        ul = SubElement(menuroot, "ul", id=section_id)
        section_head = SubElement(ul, "li")
        title = SubElement(section_head, "span", {"class":"section title"})
        title.text = section
    else:
        section_head = section_head[0]
    return section_head


def build_menu(tree, basename, section_head):
    page_title = find_title(tree)
    if page_title:
        page_title = page_title[0]
    else:
        page_title = replace_invalid('', basename.capitalize())
    build_menu_entry(page_title, basename+".html", section_head,
                     headings=find_headings(tree))


def build_menu_entry(page_title, url, section_head, headings=None):
    page_id = replace_invalid(' ', os.path.splitext(url)[0]) + '-menu'
    ul = SubElement(section_head, "ul", {"class":"menu foreign", "id":page_id})

    title = SubElement(ul, "li", {"class":"menu title"})
    a = SubElement(title, "a", href=url)
    a.text = page_title

    if headings:
        subul = SubElement(title, "ul", {"class":"submenu"})
        for heading in headings:
            li = SubElement(subul, "li", {"class":"menu item"})
            try:
                ref = heading.getparent().getparent().get('id')
            except AttributeError:
                ref = None
            if ref is None:
                ref = '-'.join(find_words(replace_invalid(' ', heading.lower())))
            a = SubElement(li, "a", href=url+'#'+ref)
            a.text = heading


def merge_menu(tree, menu, name):
    menu_root = copy.deepcopy(menu)
    tree.getroot()[1][0].insert(0, menu_root)  # html->body->div[class=document]
    for el in menu_root.iter():
        tag = el.tag
        if tag[0] != '{':
            el.tag = "{http://www.w3.org/1999/xhtml}" + tag
    current_menu = find_menu(
        menu_root, name=replace_invalid(' ', name) + '-menu')
    if not current_menu:
        current_menu = find_menu(
            menu_root, name=replace_invalid('-', name) + '-menu')
    if current_menu:
        for submenu in current_menu:
            submenu.set("class", submenu.get("class", "").
                        replace("foreign", "current"))
    return tree


def inject_flatter_button(tree):
    head = tree.xpath('h:head[1]', namespaces=htmlnsmap)[0]
    script = SubElement(head, '{%s}script' % XHTML_NS, type='text/javascript')
    script.text = """
    (function() {
        var s = document.createElement('script');
        var t = document.getElementsByTagName('script')[0];
        s.type = 'text/javascript';
        s.async = true;
        s.src = 'http://api.flattr.com/js/0.6/load.js?mode=auto';
        t.parentNode.insertBefore(s, t);
    })();
"""
    script.tail = '\n'
    intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]', namespaces=htmlnsmap)[0]
    intro_div.insert(-1, XML(
        '<p style="text-align: center;">Like working with lxml? '
        'Happy about the time that it just saved you? <br />'
        'Show your appreciation with <a href="http://flattr.com/thing/268156/lxml-The-Python-XML-Toolkit">Flattr</a>.<br />'
        '<a class="FlattrButton" style="display:none;" rev="flattr;button:compact;" href="https://lxml.de/"></a>'
        '</p>'
        ))


def inject_donate_buttons(lxml_path, rst2html_script, tree):
    command = ([sys.executable, rst2html_script]
               + RST2HTML_OPTIONS.split() + [os.path.join(lxml_path, 'README.rst')])
    rst2html = subprocess.Popen(command, stdout=subprocess.PIPE)
    stdout, _ = rst2html.communicate()
    readme = fromstring(stdout)

    intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]',
                           namespaces=htmlnsmap)[0]
    support_div = readme.xpath('h:body//h:div[@id = "support-the-project"][1]',
                               namespaces=htmlnsmap)[0]
    intro_div.append(support_div)

    finance_div = readme.xpath('h:body//h:div[@id = "project-income-report"][1]',
                               namespaces=htmlnsmap)[0]
    legal = readme.xpath('h:body//h:div[@id = "legal-notice-for-donations"][1]',
                         namespaces=htmlnsmap)[0]
    last_div = tree.xpath('h:body//h:div//h:div', namespaces=htmlnsmap)[-1]
    last_div.addnext(finance_div)
    finance_div.addnext(legal)


def inject_banner(parent):
    banner = parent.makeelement('div', {'class': 'banner'})
    parent.insert(0, banner)

    banner_image = SubElement(banner, 'div', {'class': "banner_image"})
    SubElement(banner_image, 'img', src="python-xml-title.png")

    banner_text = SubElement(banner, 'div', {'class': "banner_link"})
    banner_link = SubElement(banner_text, 'a', href="index.html#support-the-project")
    banner_link.text = "Like the tool? "
    SubElement(banner_link, 'br', {'class': "first"}).tail = "Help making it better! "
    SubElement(banner_link, 'br', {'class': "second"}).tail = "Your donation helps!"


def rest2html(script, source_path, dest_path, stylesheet_url):
    command = ('%s %s %s --stylesheet=%s --link-stylesheet %s > %s' %
               (sys.executable, script, RST2HTML_OPTIONS,
                stylesheet_url, source_path, dest_path))
    subprocess.call(command, shell=True)


def convert_changelog(lxml_path, changelog_file_path, rst2html_script, stylesheet_url):
    f = open_file(os.path.join(lxml_path, 'CHANGES.txt'), 'r', encoding='utf-8')
    try:
        content = f.read()
    finally:
        f.close()

    links = dict(LP='`%s <https://bugs.launchpad.net/lxml/+bug/%s>`_',
                 GH='`%s <https://github.com/lxml/lxml/issues/%s>`_')
    replace_tracker_links = re.compile('((LP|GH)#([0-9]+))').sub
    def insert_link(match):
        text, ref_type, ref_id = match.groups()
        return links[ref_type] % (text, ref_id)
    content = replace_tracker_links(insert_link, content)

    command = [sys.executable, rst2html_script] + RST2HTML_OPTIONS.split() + [
        '--link-stylesheet', '--stylesheet', stylesheet_url ]
    out_file = open(changelog_file_path, 'wb')
    try:
        rst2html = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=out_file)
        rst2html.communicate(content.encode('utf8'))
    finally:
        out_file.close()


def publish(dirname, lxml_path, release, with_donations=True):
    if not os.path.exists(dirname):
        os.mkdir(dirname)

    doc_dir = os.path.join(lxml_path, 'doc')
    script = os.path.join(doc_dir, 'rest2html.py')
    pubkey = os.path.join(doc_dir, 'pubkey.asc')
    stylesheet_file = 'style.css'

    shutil.copy(pubkey, dirname)
    # FIXME: find a way to make hashed filenames work both locally and in the versioned directories.
    stylesheet_url = stylesheet_file
    """
    style_file_pattern = "style_%s.css"
    for old_stylesheet in glob.iglob(os.path.join(dirname, style_file_pattern % "*")):
        os.unlink(old_stylesheet)
    with open(os.path.join(dirname, stylesheet_file), 'rb') as f:
        css = f.read()
        checksum = hashlib.sha256(css).hexdigest()[:32]

        stylesheet_url = style_file_pattern % checksum
        with open(os.path.join(dirname, stylesheet_url), 'wb') as out:
            out.write(css)
    """

    href_map = HREF_MAP.copy()
    changelog_basename = 'changes-%s' % release
    href_map['Release Changelog'] = changelog_basename + '.html'

    menu_js = textwrap.dedent('''
    function trigger_menu(event) {
        var sidemenu = document.getElementById("sidemenu");
        var classes = sidemenu.getAttribute("class");
        classes = (classes.indexOf(" visible") === -1) ? classes + " visible" : classes.replace(" visible", "");
        sidemenu.setAttribute("class", classes);
        event.preventDefault();
        event.stopPropagation();
    }
    function hide_menu() {
        var sidemenu = document.getElementById("sidemenu");
        var classes = sidemenu.getAttribute("class");
        if (classes.indexOf(" visible") !== -1) {
            sidemenu.setAttribute("class", classes.replace(" visible", ""));
        }
    }
    ''')

    trees = {}
    menu = Element("div", {'class': 'sidemenu', 'id': 'sidemenu'})
    SubElement(menu, 'div', {'class': 'menutrigger', 'onclick': 'trigger_menu(event)'}).text = "Menu"
    menu_div = SubElement(menu, 'div', {'class': 'menu'})
    if with_donations:
        inject_banner(menu_div)

    # build HTML pages and parse them back
    for section, text_files in SITE_STRUCTURE:
        section_head = make_menu_section_head(section, menu_div)
        for filename in text_files:
            if filename.startswith('@'):
                # special menu entry
                page_title = filename[1:]
                url = href_map[page_title]
                build_menu_entry(page_title, url, section_head)
            else:
                path = os.path.join(doc_dir, filename)
                basename = os.path.splitext(os.path.basename(filename))[0]
                basename = BASENAME_MAP.get(basename, basename)
                outname = basename + '.html'
                outpath = os.path.join(dirname, outname)

                rest2html(script, path, outpath, stylesheet_url)
                tree = parse(outpath)

                if with_donations:
                    page_div = tree.getroot()[1][0]  # html->body->div[class=document]
                    inject_banner(page_div)

                    if filename == 'main.txt':
                        # inject donation buttons
                        #inject_flatter_button(tree)
                        inject_donate_buttons(lxml_path, script, tree)

                trees[filename] = (tree, basename, outpath)
                build_menu(tree, basename, section_head)

    # also convert CHANGES.txt
    convert_changelog(lxml_path, os.path.join(dirname, 'changes-%s.html' % release),
                      script, stylesheet_url)

    # generate sitemap from menu
    sitemap = XML(textwrap.dedent('''\
    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
    <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
      <head>
        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
        <title>Sitemap of lxml.de - Processing XML and HTML with Python</title>
        <meta content="lxml - the most feature-rich and easy-to-use library for processing XML and HTML in the Python language"
              name="description" />
        <meta content="Python XML, XML, XML processing, HTML, lxml, simple XML, ElementTree, etree, lxml.etree, objectify, XML parsing, XML validation, XPath, XSLT"
              name="keywords" />
      </head>
      <body>
        <h1>Sitemap of lxml.de - Processing XML and HTML with Python</h1>
      </body>
    </html>
    '''))
    sitemap_menu = copy.deepcopy(menu)
    SubElement(SubElement(sitemap_menu[-1], 'li'), 'a', href='https://lxml.de/files/').text = 'Download files'
    sitemap[-1].append(sitemap_menu)  # append to body
    ElementTree(sitemap).write(os.path.join(dirname, 'sitemap.html'))

    # integrate sitemap into the menu
    SubElement(SubElement(menu_div[-1], 'li'), 'a', href='/sitemap.html').text = 'Sitemap'

    # integrate menu into web pages
    for tree, basename, outpath in trees.values():
        head = find_head(tree)[0]
        SubElement(head, 'script', type='text/javascript').text = menu_js
        SubElement(head, 'meta', name='viewport', content="width=device-width, initial-scale=1")
        find_body(tree)[0].set('onclick', 'hide_menu()')

        new_tree = merge_menu(tree, menu, basename)
        title = find_title_tag(new_tree)
        if title and title[0].text == 'lxml':
            title[0].text = "lxml - Processing XML and HTML with Python"
            heading = find_heading_tag(new_tree)
            if heading:
                heading[0].text = "lxml - XML and HTML with Python"
        new_tree.write(outpath)


if __name__ == '__main__':
    no_donations = '--no-donations' in sys.argv[1:]
    if no_donations:
        sys.argv.remove('--no-donations')
    publish(sys.argv[1], sys.argv[2], sys.argv[3], with_donations=not no_donations)