Replace duplicated dox_to_sphinx script with sphinxygen dependency

6 files changed, 29 insertions, 663 deletions

diff --git a/.gitignore b/.gitignore
index 72b54d39..b81dfb21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,9 @@
 # Copyright 2017-2022 David Robillard <d@drobilla.net>
 # SPDX-License-Identifier: 0BSD OR ISC
 
-build/**
-__pycache__
 .meson-subproject-wrap-hash.txt
+__pycache__
+build/**
+subprojects/packagecache/
+subprojects/sphinxygen-1.0.0/
+subprojects/sphinxygen/
diff --git a/NEWS b/NEWS
index ebc97088..2c896ec3 100644
--- a/NEWS
+++ b/NEWS
@@ -7,9 +7,10 @@ serd (0.30.17) unstable; urgency=medium
   * Gracefully handle bad characters in Turtle datatype syntax
   * Improve serdi man page
   * Override pkg-config dependency within meson
+  * Replace duplicated dox_to_sphinx script with sphinxygen dependency
   * Test header for warnings more strictly
 
- -- David Robillard <d@drobilla.net>  Mon, 28 Nov 2022 18:45:50 +0000
+ -- David Robillard <d@drobilla.net>  Sun, 04 Dec 2022 18:33:52 +0000
 
 serd (0.30.16) stable; urgency=medium
 
diff --git a/doc/c/api/meson.build b/doc/c/api/meson.build
index 377cb2b4..b40af2f1 100644
--- a/doc/c/api/meson.build
+++ b/doc/c/api/meson.build
@@ -3,7 +3,7 @@
 
 c_serd_rst = custom_target(
   'serd.rst',
-  command: [dox_to_sphinx, '-f', '@INPUT0@', '@OUTDIR@'],
+  command: [sphinxygen, '-f', '@INPUT0@', '@OUTDIR@'],
   input: [c_index_xml] + c_rst_files,
   output: 'serd.rst',
 )
diff --git a/doc/meson.build b/doc/meson.build
index b97ba610..e1a20aa4 100644
--- a/doc/meson.build
+++ b/doc/meson.build
@@ -4,11 +4,16 @@
 docdir = get_option('datadir') / 'doc'
 
 doxygen = find_program('doxygen', required: get_option('docs'))
-dox_to_sphinx = files('../scripts/dox_to_sphinx.py')
+sphinxygen = find_program('sphinxygen', required: false)
 sphinx_build = find_program('sphinx-build', required: get_option('docs'))
 mandoc = find_program('mandoc', required: false)
 
-build_docs = doxygen.found() and sphinx_build.found()
+if not sphinxygen.found()
+  subproject('sphinxygen')
+  sphinxygen = find_program('sphinxygen', required: get_option('docs'))
+endif
+
+build_docs = doxygen.found() and sphinxygen.found() and sphinx_build.found()
 
 if build_docs
   subdir('c')
diff --git a/scripts/dox_to_sphinx.py b/scripts/dox_to_sphinx.py
deleted file mode 100755
index 6fcabadc..00000000
--- a/scripts/dox_to_sphinx.py
+++ /dev/null
@@ -1,657 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2020-2021 David Robillard <d@drobilla.net>
-# SPDX-License-Identifier: ISC
-
-"""
-Write Sphinx markup from Doxygen XML.
-
-Takes a path to a directory of XML generated by Doxygen, and emits a directory
-with a reStructuredText file for every documented symbol.
-"""
-
-import argparse
-import os
-import sys
-import textwrap
-import xml.etree.ElementTree
-
-__author__ = "David Robillard"
-__date__ = "2020-11-18"
-__email__ = "d@drobilla.net"
-__license__ = "ISC"
-__version__ = __date__.replace("-", ".")
-
-
-def load_index(index_path):
-    """
-    Load the index from XML.
-
-    :returns: A dictionary from ID to skeleton records with basic information
-    for every documented entity.  Some records have an ``xml_filename`` key
-    with the filename of a definition file.  These files will be loaded later
-    to flesh out the records in the index.
-    """
-
-    root = xml.etree.ElementTree.parse(index_path).getroot()
-    index = {}
-
-    for compound in root:
-        compound_id = compound.get("refid")
-        compound_kind = compound.get("kind")
-        compound_name = compound.find("name").text
-        if compound_kind in ["dir", "file", "page"]:
-            continue
-
-        # Add record for compound (compounds appear only once in the index)
-        assert compound_id not in index
-        index[compound_id] = {
-            "kind": compound_kind,
-            "name": compound_name,
-            "xml_filename": compound_id + ".xml",
-            "children": [],
-        }
-
-        name_prefix = (
-            ("%s::" % compound_name) if compound_kind == "namespace" else ""
-        )
-
-        for child in compound.findall("member"):
-            if child.get("refid") in index:
-                assert compound_kind == "group"
-                continue
-
-            # Everything has a kind and a name
-            child_record = {
-                "kind": child.get("kind"),
-                "name": name_prefix + child.find("name").text,
-            }
-
-            if child.get("kind") == "enum":
-                # Enums are not compounds, but we want to resolve the parent of
-                # their values so they are not written as top level documents
-                child_record["children"] = []
-
-            if child.get("kind") == "enumvalue":
-                # Remove namespace prefix
-                child_record["name"] = child.find("name").text
-
-            index[child.get("refid")] = child_record
-
-    return index
-
-
-def resolve_index(index, root):
-    """
-    Walk a definition document and extend the index for linking.
-
-    This does two things: sets the "parent" and "children" fields of all
-    applicable records, and sets the "strong" field of enums so that the
-    correct Sphinx role can be used when referring to them.
-    """
-
-    def add_child(index, parent_id, child_id):
-        parent = index[parent_id]
-        child = index[child_id]
-
-        if child["kind"] == "enumvalue":
-            assert parent["kind"] == "enum"
-            assert "parent" not in child or child["parent"] == parent_id
-            child["parent"] = parent_id
-
-        else:
-            if parent["kind"] in ["class", "struct", "union"]:
-                assert "parent" not in child or child["parent"] == parent_id
-                child["parent"] = parent_id
-
-        if child_id not in parent["children"]:
-            parent["children"] += [child_id]
-
-    compound = root.find("compounddef")
-    compound_kind = compound.get("kind")
-
-    if compound_kind == "group":
-        for subgroup in compound.findall("innergroup"):
-            add_child(index, compound.get("id"), subgroup.get("refid"))
-
-        for klass in compound.findall("innerclass"):
-            add_child(index, compound.get("id"), klass.get("refid"))
-
-    for section in compound.findall("sectiondef"):
-        if section.get("kind").startswith("private"):
-            for member in section.findall("memberdef"):
-                if member.get("id") in index:
-                    del index[member.get("id")]
-        else:
-            for member in section.findall("memberdef"):
-                member_id = member.get("id")
-                add_child(index, compound.get("id"), member_id)
-
-                if member.get("kind") == "enum":
-                    index[member_id]["strong"] = member.get("strong") == "yes"
-                    for value in member.findall("enumvalue"):
-                        add_child(index, member_id, value.get("id"))
-
-
-def sphinx_role(record, lang):
-    """
-    Return the Sphinx role used for a record.
-
-    This is used for the description directive like ".. c:function::", and
-    links like ":c:func:`foo`.
-    """
-
-    kind = record["kind"]
-
-    if kind in ["class", "function", "namespace", "struct", "union"]:
-        return lang + ":" + kind
-
-    if kind == "define":
-        return "c:macro"
-
-    if kind == "enum":
-        return lang + (":enum-class" if record["strong"] else ":enum")
-
-    if kind == "typedef":
-        return lang + ":type"
-
-    if kind == "enumvalue":
-        return lang + ":enumerator"
-
-    if kind == "variable":
-        return lang + (":member" if "parent" in record else ":var")
-
-    raise RuntimeError("No known role for kind '%s'" % kind)
-
-
-def child_identifier(lang, parent_name, child_name):
-    """
-    Return the identifier for an enum value or struct member.
-
-    Sphinx, for some reason, uses a different syntax for this in C and C++.
-    """
-
-    separator = "::" if lang == "cpp" else "."
-
-    return "%s%s%s" % (parent_name, separator, child_name)
-
-
-def link_markup(index, lang, refid):
-    """Return a Sphinx link for a Doxygen reference."""
-
-    record = index[refid]
-    kind, name = record["kind"], record["name"]
-    role = sphinx_role(record, lang)
-
-    if kind in ["class", "enum", "struct", "typedef", "union"]:
-        return ":%s:`%s`" % (role, name)
-
-    if kind == "define":
-        return ":%s:macro:`%s`" % (lang, name)
-
-    if kind == "function":
-        return ":%s:func:`%s`" % (lang, name)
-
-    if kind == "enumvalue":
-        parent_name = index[record["parent"]]["name"]
-        return ":%s:`%s`" % (role, child_identifier(lang, parent_name, name))
-
-    if kind == "variable":
-        if "parent" not in record:
-            return ":%s:var:`%s`" % (lang, name)
-
-        parent_name = index[record["parent"]]["name"]
-        return ":%s:`%s`" % (role, child_identifier(lang, parent_name, name))
-
-    raise RuntimeError("Unknown link target kind: %s" % kind)
-
-
-def indent(markup, depth):
-    """
-    Indent markup to a depth level.
-
-    Like textwrap.indent() but takes an integer and works in reST indentation
-    levels for clarity."
-    """
-
-    return textwrap.indent(markup, "   " * depth)
-
-
-def heading(text, level):
-    """
-    Return a ReST heading at a given level.
-
-    Follows the style in the Python documentation guide, see
-    <https://devguide.python.org/documenting/#sections>.
-    """
-
-    assert 1 <= level <= 6
-
-    chars = ("#", "*", "=", "-", "^", '"')
-    line = chars[level] * len(text)
-
-    return "%s%s\n%s\n\n" % (line + "\n" if level < 3 else "", text, line)
-
-
-def dox_to_rst(index, lang, node):
-    """
-    Convert documentation commands (docCmdGroup) to Sphinx markup.
-
-    This is used to convert the content of descriptions in the documentation.
-    It recursively parses all children tags and raises a RuntimeError if any
-    unknown tag is encountered.
-    """
-
-    def field_value(markup):
-        """Return a value for a field as a single line or indented block."""
-        if "\n" in markup.strip():
-            return "\n" + indent(markup, 1)
-
-        return " " + markup.strip()
-
-    if node.tag == "emphasis":
-        return "*%s*" % plain_text(node)
-
-    if node.tag == "lsquo":
-        return "‘"
-
-    if node.tag == "rsquo":
-        return "’"
-
-    if node.tag == "computeroutput":
-        return "``%s``" % plain_text(node)
-
-    if node.tag == "itemizedlist":
-        markup = ""
-        for item in node.findall("listitem"):
-            assert len(item) == 1
-            markup += "\n- %s" % dox_to_rst(index, lang, item[0])
-
-        return markup
-
-    if node.tag == "para":
-        markup = node.text if node.text is not None else ""
-        for child in node:
-            markup += dox_to_rst(index, lang, child)
-            markup += child.tail if child.tail is not None else ""
-
-        return markup.strip() + "\n\n"
-
-    if node.tag == "parameterlist":
-        markup = ""
-        for item in node.findall("parameteritem"):
-            name = item.find("parameternamelist/parametername")
-            description = item.find("parameterdescription")
-            assert len(description) == 1
-            markup += "\n\n:param %s:%s" % (
-                name.text,
-                field_value(dox_to_rst(index, lang, description[0])),
-            )
-
-        return markup + "\n"
-
-    if node.tag == "programlisting":
-        return "\n.. code-block:: %s\n\n%s" % (
-            lang,
-            indent(plain_text(node), 1),
-        )
-
-    if node.tag == "ref":
-        refid = node.get("refid")
-        if refid not in index:
-            sys.stderr.write("warning: Unresolved link: %s\n" % refid)
-            return node.text
-
-        assert len(node) == 0
-        assert len(link_markup(index, lang, refid)) > 0
-        return link_markup(index, lang, refid)
-
-    if node.tag == "simplesect":
-        assert len(node) == 1
-
-        if node.get("kind") == "return":
-            return "\n:returns:" + field_value(
-                dox_to_rst(index, lang, node[0])
-            )
-
-        if node.get("kind") == "see":
-            return dox_to_rst(index, lang, node[0])
-
-        raise RuntimeError("Unknown simplesect kind: %s" % node.get("kind"))
-
-    if node.tag == "ulink":
-        return "`%s <%s>`_" % (node.text, node.get("url"))
-
-    raise RuntimeError("Unknown documentation command: %s" % node.tag)
-
-
-def description_markup(index, lang, node):
-    """Return the markup for a brief or detailed description."""
-
-    assert node.tag == "briefdescription" or node.tag == "detaileddescription"
-    assert not (node.tag == "briefdescription" and len(node) > 1)
-    assert len(node.text.strip()) == 0
-
-    return "".join([dox_to_rst(index, lang, child) for child in node]).strip()
-
-
-def set_descriptions(index, lang, definition, record):
-    """Set a record's brief/detailed descriptions from the XML definition."""
-
-    for tag in ["briefdescription", "detaileddescription"]:
-        node = definition.find(tag)
-        if node is not None:
-            record[tag] = description_markup(index, lang, node)
-
-
-def set_template_params(node, record):
-    """Set a record's template_params from the XML definition."""
-
-    template_param_list = node.find("templateparamlist")
-    if template_param_list is not None:
-        params = []
-        for param in template_param_list.findall("param"):
-            if param.find("declname") is not None:
-                # Value parameter
-                type_text = plain_text(param.find("type"))
-                name_text = plain_text(param.find("declname"))
-
-                params += ["%s %s" % (type_text, name_text)]
-            else:
-                # Type parameter
-                params += ["%s" % (plain_text(param.find("type")))]
-
-        record["template_params"] = "%s" % ", ".join(params)
-
-
-def plain_text(node):
-    """
-    Return the plain text of a node with all tags ignored.
-
-    This is needed where Doxygen may include refs but Sphinx needs plain text
-    because it parses things itself to generate links.
-    """
-
-    if node.tag == "sp":
-        markup = " "
-    elif node.text is not None:
-        markup = node.text
-    else:
-        markup = ""
-
-    for child in node:
-        markup += plain_text(child)
-        markup += child.tail if child.tail is not None else ""
-
-    return markup
-
-
-def local_name(name):
-    """Return a name with all namespace prefixes stripped."""
-
-    return name[name.rindex("::") + 2 :] if "::" in name else name
-
-
-def read_definition_doc(index, lang, root):
-    """Walk a definition document and update described records in the index."""
-
-    # Set descriptions for the compound itself
-    compound = root.find("compounddef")
-    compound_record = index[compound.get("id")]
-    set_descriptions(index, lang, compound, compound_record)
-    set_template_params(compound, compound_record)
-
-    if compound.find("title") is not None:
-        compound_record["title"] = compound.find("title").text.strip()
-
-    # Set documentation for all children
-    for section in compound.findall("sectiondef"):
-        if section.get("kind").startswith("private"):
-            continue
-
-        for member in section.findall("memberdef"):
-            kind = member.get("kind")
-            record = index[member.get("id")]
-            set_descriptions(index, lang, member, record)
-            set_template_params(member, record)
-
-            if compound.get("kind") in ["class", "struct", "union"]:
-                assert kind in ["function", "typedef", "variable"]
-                record["type"] = plain_text(member.find("type"))
-
-            if kind == "define":
-                if member.find('param') is not None:
-                    param_names = []
-                    for param in member.findall('param'):
-                        defname = param.find('defname')
-                        param_names += [defname.text] if defname is not None else []
-
-                    record["prototype"] = "%s(%s)" % (record["name"], ', '.join(param_names))
-
-            elif kind == "enum":
-                for value in member.findall("enumvalue"):
-                    set_descriptions(
-                        index, lang, value, index[value.get("id")]
-                    )
-
-            elif kind == "function":
-                record["prototype"] = "%s %s%s" % (
-                    plain_text(member.find("type")),
-                    member.find("name").text,
-                    member.find("argsstring").text,
-                )
-
-            elif kind == "typedef":
-                name = local_name(record["name"])
-                args_text = member.find("argsstring").text
-                target_text = plain_text(member.find("type"))
-                if args_text is not None:  # Function pointer
-                    assert target_text[-2:] == "(*" and args_text[0] == ")"
-                    record["type"] = target_text + args_text
-                    record["definition"] = target_text + name + args_text
-                else:  # Normal named typedef
-                    assert target_text is not None
-                    record["type"] = target_text
-                    if member.find("definition").text.startswith("using"):
-                        record["definition"] = "%s = %s" % (
-                            name,
-                            target_text,
-                        )
-                    else:
-                        record["definition"] = "%s %s" % (
-                            target_text,
-                            name,
-                        )
-
-            elif kind == "variable":
-                record["definition"] = member.find("definition").text
-
-
-def declaration_string(record):
-    """
-    Return the string that describes a declaration.
-
-    This is what follows the directive, and is in C/C++ syntax, except without
-    keywords like "typedef" and "using" as expected by Sphinx.  For example,
-    "struct ThingImpl Thing" or "void run(int value)".
-    """
-
-    kind = record["kind"]
-    result = ""
-
-    if "template_params" in record:
-        result = "template <%s> " % record["template_params"]
-
-    if kind == "define" and "prototype" in record:
-        result += record["prototype"]
-    elif kind == "function":
-        result += record["prototype"]
-    elif kind == "typedef":
-        result += record["definition"]
-    elif kind == "variable":
-        if "parent" in record:
-            result += "%s %s" % (record["type"], local_name(record["name"]))
-        else:
-            result += record["definition"]
-    elif "type" in record:
-        result += "%s %s" % (record["type"], local_name(record["name"]))
-    else:
-        result += local_name(record["name"])
-
-    return result
-
-
-def document_markup(index, lang, record):
-    """Return the complete document that describes some documented entity."""
-
-    kind = record["kind"]
-    role = sphinx_role(record, lang)
-    name = record["name"]
-    markup = ""
-
-    if name != local_name(name):
-        markup += ".. cpp:namespace:: %s\n\n" % name[0 : name.rindex("::")]
-
-    # Write top-level directive
-    markup += ".. %s:: %s\n" % (role, declaration_string(record))
-
-    # Write main description blurb
-    markup += "\n" + indent(record["briefdescription"] + "\n", 1)
-    if len(record["detaileddescription"]) > 0:
-        markup += "\n" + indent(record["detaileddescription"], 1) + "\n"
-
-    assert (
-        kind in ["class", "enum", "namespace", "struct", "union"]
-        or "children" not in record
-    )
-
-    # Sphinx C++ namespaces work by setting a scope, they have no content
-    child_indent = 0 if kind == "namespace" else 1
-
-    # Write inline children if applicable
-    markup += "\n" if "children" in record else ""
-    for child_id in record.get("children", []):
-        child_record = index[child_id]
-        child_role = sphinx_role(child_record, lang)
-
-        child_header = ".. %s:: %s\n\n" % (
-            child_role,
-            declaration_string(child_record),
-        )
-
-        markup += "\n"
-        markup += indent(child_header, child_indent)
-        markup += indent(child_record["briefdescription"], child_indent + 1)
-        markup += indent(child_record["detaileddescription"], child_indent + 1)
-
-    return markup
-
-
-def symbol_filename(name):
-    """Adapt the name of a symbol to be suitable for use as a filename."""
-
-    return name.replace("::", "__")
-
-
-def emit_groups(index, lang, output_dir, force):
-    """Write a description file for every group documented in the index."""
-
-    for record in index.values():
-        if record["kind"] != "group":
-            continue
-
-        name = record["name"]
-        filename = os.path.join(output_dir, "%s.rst" % name)
-        if not force and os.path.exists(filename):
-            raise FileExistsError("File already exists: '%s'" % filename)
-
-        with open(filename, "w") as rst:
-            rst.write(heading(record["title"], 1))
-
-            # Get all child group and symbol names
-            child_groups = {}
-            child_symbols = {}
-            for child_id in record["children"]:
-                child = index[child_id]
-                if child["kind"] == "group":
-                    child_groups[child["name"]] = child
-                else:
-                    child_symbols[child["name"]] = child
-
-            # Emit description (document body)
-            if len(record["briefdescription"]) > 0:
-                rst.write(record["briefdescription"] + "\n\n")
-            if len(record["detaileddescription"]) > 0:
-                rst.write(record["detaileddescription"] + "\n\n")
-
-            if len(child_groups) > 0:
-                # Emit TOC for child groups
-                rst.write(".. toctree::\n\n")
-                for name, group in child_groups.items():
-                    rst.write(indent(group["name"], 1) + "\n")
-
-            # Emit symbols in sorted order
-            for name, symbol in child_symbols.items():
-                rst.write("\n")
-                rst.write(document_markup(index, lang, symbol))
-                rst.write("\n")
-
-
-def run(index_xml_path, output_dir, language, force):
-    """Write a directory of Sphinx files from a Doxygen XML directory."""
-
-    # Build skeleton index from index.xml
-    xml_dir = os.path.dirname(index_xml_path)
-    index = load_index(index_xml_path)
-
-    # Load all definition documents
-    definition_docs = []
-    for record in index.values():
-        if "xml_filename" in record:
-            xml_path = os.path.join(xml_dir, record["xml_filename"])
-            definition_docs += [xml.etree.ElementTree.parse(xml_path)]
-
-    # Do an initial pass of the definition documents to resolve the index
-    for root in definition_docs:
-        resolve_index(index, root)
-
-    # Finally read the documentation from definition documents
-    for root in definition_docs:
-        read_definition_doc(index, language, root)
-
-    # Create output directory
-    try:
-        os.makedirs(output_dir)
-    except OSError:
-        pass
-
-    # Emit output files
-    emit_groups(index, language, output_dir, force)
-
-
-if __name__ == "__main__":
-    ap = argparse.ArgumentParser(
-        usage="%(prog)s [OPTION]... XML_DIR OUTPUT_DIR",
-        description=__doc__,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    ap.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="overwrite files",
-    )
-
-    ap.add_argument(
-        "-l",
-        "--language",
-        default="c",
-        choices=["c", "cpp"],
-        help="language domain for output",
-    )
-
-    ap.add_argument("index_xml_path", help="path index.xml from Doxygen")
-    ap.add_argument("output_dir", help="output directory")
-
-    run(**vars(ap.parse_args(sys.argv[1:])))
diff --git a/subprojects/sphinxygen.wrap b/subprojects/sphinxygen.wrap
new file mode 100644
index 00000000..013de9cb
--- /dev/null
+++ b/subprojects/sphinxygen.wrap
@@ -0,0 +1,14 @@
+# Copyright 2022 David Robillard <d@drobilla.net>
+# SPDX-License-Identifier: 0BSD OR ISC
+
+[wrap-file]
+directory = sphinxygen-1.0.0
+source_url = https://download.drobilla.net/sphinxygen-1.0.0.tar.gz
+source_filename = sphinxygen-1.0.0.tar.gz
+source_hash = 96b19e3b37d4886dcf3e89d4ccf0b66c0deb9f2e34ac151a7a6659a421f0282d
+
+# [wrap-git]
+# url = https://gitlab.com/drobilla/sphinxygen.git
+# push-url = ssh://git@gitlab.com:drobilla/sphinxygen.git
+# revision = main
+# depth = 1