qemu

1
# coding=utf-8
2
#
3
# QEMU qapidoc QAPI file parsing extension
4
#
5
# Copyright (c) 2020 Linaro
6
#
7
# This work is licensed under the terms of the GNU GPLv2 or later.
8
# See the COPYING file in the top-level directory.
9

10
"""
11
qapidoc is a Sphinx extension that implements the qapi-doc directive
12

13
The purpose of this extension is to read the documentation comments
14
in QAPI schema files, and insert them all into the current document.
15

16
It implements one new rST directive, "qapi-doc::".
17
Each qapi-doc:: directive takes one argument, which is the
18
pathname of the schema file to process, relative to the source tree.
19

20
The docs/conf.py file must set the qapidoc_srctree config value to
21
the root of the QEMU source tree.
22

23
The Sphinx documentation on writing extensions is at:
24
https://www.sphinx-doc.org/en/master/development/index.html
25
"""
26

27
import os
28
import re
29
import sys
30
import textwrap
31
from typing import List
32

33
from docutils import nodes
34
from docutils.parsers.rst import Directive, directives
35
from docutils.statemachine import ViewList
36
from qapi.error import QAPIError, QAPISemError
37
from qapi.gen import QAPISchemaVisitor
38
from qapi.schema import QAPISchema
39

40
from sphinx import addnodes
41
from sphinx.directives.code import CodeBlock
42
from sphinx.errors import ExtensionError
43
from sphinx.util.docutils import switch_source_input
44
from sphinx.util.nodes import nested_parse_with_titles
45

46

47
__version__ = "1.0"
48

49

50
def dedent(text: str) -> str:
51
    # Adjust indentation to make description text parse as paragraph.
52

53
    lines = text.splitlines(True)
54
    if re.match(r"\s+", lines[0]):
55
        # First line is indented; description started on the line after
56
        # the name. dedent the whole block.
57
        return textwrap.dedent(text)
58

59
    # Descr started on same line. Dedent line 2+.
60
    return lines[0] + textwrap.dedent("".join(lines[1:]))
61

62

63
# Disable black auto-formatter until re-enabled:
64
# fmt: off
65

66

67
class QAPISchemaGenRSTVisitor(QAPISchemaVisitor):
68
    """A QAPI schema visitor which generates docutils/Sphinx nodes
69

70
    This class builds up a tree of docutils/Sphinx nodes corresponding
71
    to documentation for the various QAPI objects. To use it, first
72
    create a QAPISchemaGenRSTVisitor object, and call its
73
    visit_begin() method.  Then you can call one of the two methods
74
    'freeform' (to add documentation for a freeform documentation
75
    chunk) or 'symbol' (to add documentation for a QAPI symbol). These
76
    will cause the visitor to build up the tree of document
77
    nodes. Once you've added all the documentation via 'freeform' and
78
    'symbol' method calls, you can call 'get_document_nodes' to get
79
    the final list of document nodes (in a form suitable for returning
80
    from a Sphinx directive's 'run' method).
81
    """
82
    def __init__(self, sphinx_directive):
83
        self._cur_doc = None
84
        self._sphinx_directive = sphinx_directive
85
        self._top_node = nodes.section()
86
        self._active_headings = [self._top_node]
87

88
    def _make_dlitem(self, term, defn):
89
        """Return a dlitem node with the specified term and definition.
90

91
        term should be a list of Text and literal nodes.
92
        defn should be one of:
93
        - a string, which will be handed to _parse_text_into_node
94
        - a list of Text and literal nodes, which will be put into
95
          a paragraph node
96
        """
97
        dlitem = nodes.definition_list_item()
98
        dlterm = nodes.term('', '', *term)
99
        dlitem += dlterm
100
        if defn:
101
            dldef = nodes.definition()
102
            if isinstance(defn, list):
103
                dldef += nodes.paragraph('', '', *defn)
104
            else:
105
                self._parse_text_into_node(defn, dldef)
106
            dlitem += dldef
107
        return dlitem
108

109
    def _make_section(self, title):
110
        """Return a section node with optional title"""
111
        section = nodes.section(ids=[self._sphinx_directive.new_serialno()])
112
        if title:
113
            section += nodes.title(title, title)
114
        return section
115

116
    def _nodes_for_ifcond(self, ifcond, with_if=True):
117
        """Return list of Text, literal nodes for the ifcond
118

119
        Return a list which gives text like ' (If: condition)'.
120
        If with_if is False, we don't return the "(If: " and ")".
121
        """
122

123
        doc = ifcond.docgen()
124
        if not doc:
125
            return []
126
        doc = nodes.literal('', doc)
127
        if not with_if:
128
            return [doc]
129

130
        nodelist = [nodes.Text(' ('), nodes.strong('', 'If: ')]
131
        nodelist.append(doc)
132
        nodelist.append(nodes.Text(')'))
133
        return nodelist
134

135
    def _nodes_for_one_member(self, member):
136
        """Return list of Text, literal nodes for this member
137

138
        Return a list of doctree nodes which give text like
139
        'name: type (optional) (If: ...)' suitable for use as the
140
        'term' part of a definition list item.
141
        """
142
        term = [nodes.literal('', member.name)]
143
        if member.type.doc_type():
144
            term.append(nodes.Text(': '))
145
            term.append(nodes.literal('', member.type.doc_type()))
146
        if member.optional:
147
            term.append(nodes.Text(' (optional)'))
148
        if member.ifcond.is_present():
149
            term.extend(self._nodes_for_ifcond(member.ifcond))
150
        return term
151

152
    def _nodes_for_variant_when(self, branches, variant):
153
        """Return list of Text, literal nodes for variant 'when' clause
154

155
        Return a list of doctree nodes which give text like
156
        'when tagname is variant (If: ...)' suitable for use in
157
        the 'branches' part of a definition list.
158
        """
159
        term = [nodes.Text(' when '),
160
                nodes.literal('', branches.tag_member.name),
161
                nodes.Text(' is '),
162
                nodes.literal('', '"%s"' % variant.name)]
163
        if variant.ifcond.is_present():
164
            term.extend(self._nodes_for_ifcond(variant.ifcond))
165
        return term
166

167
    def _nodes_for_members(self, doc, what, base=None, branches=None):
168
        """Return list of doctree nodes for the table of members"""
169
        dlnode = nodes.definition_list()
170
        for section in doc.args.values():
171
            term = self._nodes_for_one_member(section.member)
172
            # TODO drop fallbacks when undocumented members are outlawed
173
            if section.text:
174
                defn = dedent(section.text)
175
            else:
176
                defn = [nodes.Text('Not documented')]
177

178
            dlnode += self._make_dlitem(term, defn)
179

180
        if base:
181
            dlnode += self._make_dlitem([nodes.Text('The members of '),
182
                                         nodes.literal('', base.doc_type())],
183
                                        None)
184

185
        if branches:
186
            for v in branches.variants:
187
                if v.type.name == 'q_empty':
188
                    continue
189
                assert not v.type.is_implicit()
190
                term = [nodes.Text('The members of '),
191
                        nodes.literal('', v.type.doc_type())]
192
                term.extend(self._nodes_for_variant_when(branches, v))
193
                dlnode += self._make_dlitem(term, None)
194

195
        if not dlnode.children:
196
            return []
197

198
        section = self._make_section(what)
199
        section += dlnode
200
        return [section]
201

202
    def _nodes_for_enum_values(self, doc):
203
        """Return list of doctree nodes for the table of enum values"""
204
        seen_item = False
205
        dlnode = nodes.definition_list()
206
        for section in doc.args.values():
207
            termtext = [nodes.literal('', section.member.name)]
208
            if section.member.ifcond.is_present():
209
                termtext.extend(self._nodes_for_ifcond(section.member.ifcond))
210
            # TODO drop fallbacks when undocumented members are outlawed
211
            if section.text:
212
                defn = dedent(section.text)
213
            else:
214
                defn = [nodes.Text('Not documented')]
215

216
            dlnode += self._make_dlitem(termtext, defn)
217
            seen_item = True
218

219
        if not seen_item:
220
            return []
221

222
        section = self._make_section('Values')
223
        section += dlnode
224
        return [section]
225

226
    def _nodes_for_arguments(self, doc, arg_type):
227
        """Return list of doctree nodes for the arguments section"""
228
        if arg_type and not arg_type.is_implicit():
229
            assert not doc.args
230
            section = self._make_section('Arguments')
231
            dlnode = nodes.definition_list()
232
            dlnode += self._make_dlitem(
233
                [nodes.Text('The members of '),
234
                 nodes.literal('', arg_type.name)],
235
                None)
236
            section += dlnode
237
            return [section]
238

239
        return self._nodes_for_members(doc, 'Arguments')
240

241
    def _nodes_for_features(self, doc):
242
        """Return list of doctree nodes for the table of features"""
243
        seen_item = False
244
        dlnode = nodes.definition_list()
245
        for section in doc.features.values():
246
            dlnode += self._make_dlitem(
247
                [nodes.literal('', section.member.name)], dedent(section.text))
248
            seen_item = True
249

250
        if not seen_item:
251
            return []
252

253
        section = self._make_section('Features')
254
        section += dlnode
255
        return [section]
256

257
    def _nodes_for_example(self, exampletext):
258
        """Return list of doctree nodes for a code example snippet"""
259
        return [nodes.literal_block(exampletext, exampletext)]
260

261
    def _nodes_for_sections(self, doc):
262
        """Return list of doctree nodes for additional sections"""
263
        nodelist = []
264
        for section in doc.sections:
265
            if section.tag and section.tag == 'TODO':
266
                # Hide TODO: sections
267
                continue
268

269
            if not section.tag:
270
                # Sphinx cannot handle sectionless titles;
271
                # Instead, just append the results to the prior section.
272
                container = nodes.container()
273
                self._parse_text_into_node(section.text, container)
274
                nodelist += container.children
275
                continue
276

277
            snode = self._make_section(section.tag)
278
            if section.tag.startswith('Example'):
279
                snode += self._nodes_for_example(dedent(section.text))
280
            else:
281
                self._parse_text_into_node(dedent(section.text), snode)
282
            nodelist.append(snode)
283
        return nodelist
284

285
    def _nodes_for_if_section(self, ifcond):
286
        """Return list of doctree nodes for the "If" section"""
287
        nodelist = []
288
        if ifcond.is_present():
289
            snode = self._make_section('If')
290
            snode += nodes.paragraph(
291
                '', '', *self._nodes_for_ifcond(ifcond, with_if=False)
292
            )
293
            nodelist.append(snode)
294
        return nodelist
295

296
    def _add_doc(self, typ, sections):
297
        """Add documentation for a command/object/enum...
298

299
        We assume we're documenting the thing defined in self._cur_doc.
300
        typ is the type of thing being added ("Command", "Object", etc)
301

302
        sections is a list of nodes for sections to add to the definition.
303
        """
304

305
        doc = self._cur_doc
306
        snode = nodes.section(ids=[self._sphinx_directive.new_serialno()])
307
        snode += nodes.title('', '', *[nodes.literal(doc.symbol, doc.symbol),
308
                                       nodes.Text(' (' + typ + ')')])
309
        self._parse_text_into_node(doc.body.text, snode)
310
        for s in sections:
311
            if s is not None:
312
                snode += s
313
        self._add_node_to_current_heading(snode)
314

315
    def visit_enum_type(self, name, info, ifcond, features, members, prefix):
316
        doc = self._cur_doc
317
        self._add_doc('Enum',
318
                      self._nodes_for_enum_values(doc)
319
                      + self._nodes_for_features(doc)
320
                      + self._nodes_for_sections(doc)
321
                      + self._nodes_for_if_section(ifcond))
322

323
    def visit_object_type(self, name, info, ifcond, features,
324
                          base, members, branches):
325
        doc = self._cur_doc
326
        if base and base.is_implicit():
327
            base = None
328
        self._add_doc('Object',
329
                      self._nodes_for_members(doc, 'Members', base, branches)
330
                      + self._nodes_for_features(doc)
331
                      + self._nodes_for_sections(doc)
332
                      + self._nodes_for_if_section(ifcond))
333

334
    def visit_alternate_type(self, name, info, ifcond, features,
335
                             alternatives):
336
        doc = self._cur_doc
337
        self._add_doc('Alternate',
338
                      self._nodes_for_members(doc, 'Members')
339
                      + self._nodes_for_features(doc)
340
                      + self._nodes_for_sections(doc)
341
                      + self._nodes_for_if_section(ifcond))
342

343
    def visit_command(self, name, info, ifcond, features, arg_type,
344
                      ret_type, gen, success_response, boxed, allow_oob,
345
                      allow_preconfig, coroutine):
346
        doc = self._cur_doc
347
        self._add_doc('Command',
348
                      self._nodes_for_arguments(doc, arg_type)
349
                      + self._nodes_for_features(doc)
350
                      + self._nodes_for_sections(doc)
351
                      + self._nodes_for_if_section(ifcond))
352

353
    def visit_event(self, name, info, ifcond, features, arg_type, boxed):
354
        doc = self._cur_doc
355
        self._add_doc('Event',
356
                      self._nodes_for_arguments(doc, arg_type)
357
                      + self._nodes_for_features(doc)
358
                      + self._nodes_for_sections(doc)
359
                      + self._nodes_for_if_section(ifcond))
360

361
    def symbol(self, doc, entity):
362
        """Add documentation for one symbol to the document tree
363

364
        This is the main entry point which causes us to add documentation
365
        nodes for a symbol (which could be a 'command', 'object', 'event',
366
        etc). We do this by calling 'visit' on the schema entity, which
367
        will then call back into one of our visit_* methods, depending
368
        on what kind of thing this symbol is.
369
        """
370
        self._cur_doc = doc
371
        entity.visit(self)
372
        self._cur_doc = None
373

374
    def _start_new_heading(self, heading, level):
375
        """Start a new heading at the specified heading level
376

377
        Create a new section whose title is 'heading' and which is placed
378
        in the docutils node tree as a child of the most recent level-1
379
        heading. Subsequent document sections (commands, freeform doc chunks,
380
        etc) will be placed as children of this new heading section.
381
        """
382
        if len(self._active_headings) < level:
383
            raise QAPISemError(self._cur_doc.info,
384
                               'Level %d subheading found outside a '
385
                               'level %d heading'
386
                               % (level, level - 1))
387
        snode = self._make_section(heading)
388
        self._active_headings[level - 1] += snode
389
        self._active_headings = self._active_headings[:level]
390
        self._active_headings.append(snode)
391
        return snode
392

393
    def _add_node_to_current_heading(self, node):
394
        """Add the node to whatever the current active heading is"""
395
        self._active_headings[-1] += node
396

397
    def freeform(self, doc):
398
        """Add a piece of 'freeform' documentation to the document tree
399

400
        A 'freeform' document chunk doesn't relate to any particular
401
        symbol (for instance, it could be an introduction).
402

403
        If the freeform document starts with a line of the form
404
        '= Heading text', this is a section or subsection heading, with
405
        the heading level indicated by the number of '=' signs.
406
        """
407

408
        # QAPIDoc documentation says free-form documentation blocks
409
        # must have only a body section, nothing else.
410
        assert not doc.sections
411
        assert not doc.args
412
        assert not doc.features
413
        self._cur_doc = doc
414

415
        text = doc.body.text
416
        if re.match(r'=+ ', text):
417
            # Section/subsection heading (if present, will always be
418
            # the first line of the block)
419
            (heading, _, text) = text.partition('\n')
420
            (leader, _, heading) = heading.partition(' ')
421
            node = self._start_new_heading(heading, len(leader))
422
            if text == '':
423
                return
424

425
        self._parse_text_into_node(text, node)
426
        self._cur_doc = None
427

428
    def _parse_text_into_node(self, doctext, node):
429
        """Parse a chunk of QAPI-doc-format text into the node
430

431
        The doc comment can contain most inline rST markup, including
432
        bulleted and enumerated lists.
433
        As an extra permitted piece of markup, @var will be turned
434
        into ``var``.
435
        """
436

437
        # Handle the "@var means ``var`` case
438
        doctext = re.sub(r'@([\w-]+)', r'``\1``', doctext)
439

440
        rstlist = ViewList()
441
        for line in doctext.splitlines():
442
            # The reported line number will always be that of the start line
443
            # of the doc comment, rather than the actual location of the error.
444
            # Being more precise would require overhaul of the QAPIDoc class
445
            # to track lines more exactly within all the sub-parts of the doc
446
            # comment, as well as counting lines here.
447
            rstlist.append(line, self._cur_doc.info.fname,
448
                           self._cur_doc.info.line)
449
        # Append a blank line -- in some cases rST syntax errors get
450
        # attributed to the line after one with actual text, and if there
451
        # isn't anything in the ViewList corresponding to that then Sphinx
452
        # 1.6's AutodocReporter will then misidentify the source/line location
453
        # in the error message (usually attributing it to the top-level
454
        # .rst file rather than the offending .json file). The extra blank
455
        # line won't affect the rendered output.
456
        rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line)
457
        self._sphinx_directive.do_parse(rstlist, node)
458

459
    def get_document_nodes(self):
460
        """Return the list of docutils nodes which make up the document"""
461
        return self._top_node.children
462

463

464
# Turn the black formatter on for the rest of the file.
465
# fmt: on
466

467

468
class QAPISchemaGenDepVisitor(QAPISchemaVisitor):
469
    """A QAPI schema visitor which adds Sphinx dependencies each module
470

471
    This class calls the Sphinx note_dependency() function to tell Sphinx
472
    that the generated documentation output depends on the input
473
    schema file associated with each module in the QAPI input.
474
    """
475

476
    def __init__(self, env, qapidir):
477
        self._env = env
478
        self._qapidir = qapidir
479

480
    def visit_module(self, name):
481
        if name != "./builtin":
482
            qapifile = self._qapidir + "/" + name
483
            self._env.note_dependency(os.path.abspath(qapifile))
484
        super().visit_module(name)
485

486

487
class NestedDirective(Directive):
488
    def run(self):
489
        raise NotImplementedError
490

491
    def do_parse(self, rstlist, node):
492
        """
493
        Parse rST source lines and add them to the specified node
494

495
        Take the list of rST source lines rstlist, parse them as
496
        rST, and add the resulting docutils nodes as children of node.
497
        The nodes are parsed in a way that allows them to include
498
        subheadings (titles) without confusing the rendering of
499
        anything else.
500
        """
501
        with switch_source_input(self.state, rstlist):
502
            nested_parse_with_titles(self.state, rstlist, node)
503

504

505
class QAPIDocDirective(NestedDirective):
506
    """Extract documentation from the specified QAPI .json file"""
507

508
    required_argument = 1
509
    optional_arguments = 1
510
    option_spec = {"qapifile": directives.unchanged_required}
511
    has_content = False
512

513
    def new_serialno(self):
514
        """Return a unique new ID string suitable for use as a node's ID"""
515
        env = self.state.document.settings.env
516
        return "qapidoc-%d" % env.new_serialno("qapidoc")
517

518
    def run(self):
519
        env = self.state.document.settings.env
520
        qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0]
521
        qapidir = os.path.dirname(qapifile)
522

523
        try:
524
            schema = QAPISchema(qapifile)
525

526
            # First tell Sphinx about all the schema files that the
527
            # output documentation depends on (including 'qapifile' itself)
528
            schema.visit(QAPISchemaGenDepVisitor(env, qapidir))
529

530
            vis = QAPISchemaGenRSTVisitor(self)
531
            vis.visit_begin(schema)
532
            for doc in schema.docs:
533
                if doc.symbol:
534
                    vis.symbol(doc, schema.lookup_entity(doc.symbol))
535
                else:
536
                    vis.freeform(doc)
537
            return vis.get_document_nodes()
538
        except QAPIError as err:
539
            # Launder QAPI parse errors into Sphinx extension errors
540
            # so they are displayed nicely to the user
541
            raise ExtensionError(str(err)) from err
542

543

544
class QMPExample(CodeBlock, NestedDirective):
545
    """
546
    Custom admonition for QMP code examples.
547

548
    When the :annotated: option is present, the body of this directive
549
    is parsed as normal rST, but with any '::' code blocks set to use
550
    the QMP lexer. Code blocks must be explicitly written by the user,
551
    but this allows for intermingling explanatory paragraphs with
552
    arbitrary rST syntax and code blocks for more involved examples.
553

554
    When :annotated: is absent, the directive body is treated as a
555
    simple standalone QMP code block literal.
556
    """
557

558
    required_argument = 0
559
    optional_arguments = 0
560
    has_content = True
561
    option_spec = {
562
        "annotated": directives.flag,
563
        "title": directives.unchanged,
564
    }
565

566
    def _highlightlang(self) -> addnodes.highlightlang:
567
        """Return the current highlightlang setting for the document"""
568
        node = None
569
        doc = self.state.document
570

571
        if hasattr(doc, "findall"):
572
            # docutils >= 0.18.1
573
            for node in doc.findall(addnodes.highlightlang):
574
                pass
575
        else:
576
            for elem in doc.traverse():
577
                if isinstance(elem, addnodes.highlightlang):
578
                    node = elem
579

580
        if node:
581
            return node
582

583
        # No explicit directive found, use defaults
584
        node = addnodes.highlightlang(
585
            lang=self.env.config.highlight_language,
586
            force=False,
587
            # Yes, Sphinx uses this value to effectively disable line
588
            # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯
589
            linenothreshold=sys.maxsize,
590
        )
591
        return node
592

593
    def admonition_wrap(self, *content) -> List[nodes.Node]:
594
        title = "Example:"
595
        if "title" in self.options:
596
            title = f"{title} {self.options['title']}"
597

598
        admon = nodes.admonition(
599
            "",
600
            nodes.title("", title),
601
            *content,
602
            classes=["admonition", "admonition-example"],
603
        )
604
        return [admon]
605

606
    def run_annotated(self) -> List[nodes.Node]:
607
        lang_node = self._highlightlang()
608

609
        content_node: nodes.Element = nodes.section()
610

611
        # Configure QMP highlighting for "::" blocks, if needed
612
        if lang_node["lang"] != "QMP":
613
            content_node += addnodes.highlightlang(
614
                lang="QMP",
615
                force=False,  # "True" ignores lexing errors
616
                linenothreshold=lang_node["linenothreshold"],
617
            )
618

619
        self.do_parse(self.content, content_node)
620

621
        # Restore prior language highlighting, if needed
622
        if lang_node["lang"] != "QMP":
623
            content_node += addnodes.highlightlang(**lang_node.attributes)
624

625
        return content_node.children
626

627
    def run(self) -> List[nodes.Node]:
628
        annotated = "annotated" in self.options
629

630
        if annotated:
631
            content_nodes = self.run_annotated()
632
        else:
633
            self.arguments = ["QMP"]
634
            content_nodes = super().run()
635

636
        return self.admonition_wrap(*content_nodes)
637

638

639
def setup(app):
640
    """Register qapi-doc directive with Sphinx"""
641
    app.add_config_value("qapidoc_srctree", None, "env")
642
    app.add_directive("qapi-doc", QAPIDocDirective)
643
    app.add_directive("qmp-example", QMPExample)
644

645
    return {
646
        "version": __version__,
647
        "parallel_read_safe": True,
648
        "parallel_write_safe": True,
649
    }
650