cython

1
#
2
# TreeFragments - parsing of strings to trees
3
#
4

5
"""
6
Support for parsing strings into code trees.
7
"""
8

9

10
import re
11
from io import StringIO
12

13
from .Scanning import PyrexScanner, StringSourceDescriptor
14
from .Symtab import ModuleScope
15
from . import PyrexTypes
16
from .Visitor import VisitorTransform
17
from .Nodes import Node, StatListNode
18
from .ExprNodes import NameNode
19
from . import Parsing
20
from . import Main
21
from . import UtilNodes
22

23

24
class StringParseContext(Main.Context):
25
    def __init__(self, name, include_directories=None, compiler_directives=None, cpp=False):
26
        if include_directories is None:
27
            include_directories = []
28
        if compiler_directives is None:
29
            compiler_directives = {}
30
        Main.Context.__init__(self, include_directories, compiler_directives, cpp=cpp, language_level='3')
31
        self.module_name = name
32

33
    def find_module(self, module_name, from_module=None, pos=None, need_pxd=1, absolute_fallback=True, relative_import=False):
34
        if module_name not in (self.module_name, 'cython'):
35
            raise AssertionError("Not yet supporting any cimports/includes from string code snippets")
36
        return ModuleScope(module_name, parent_module=None, context=self)
37

38

39
def parse_from_strings(name, code, pxds=None, level=None, initial_pos=None,
40
                       context=None, allow_struct_enum_decorator=False,
41
                       in_utility_code=False):
42
    """
43
    Utility method to parse a (unicode) string of code. This is mostly
44
    used for internal Cython compiler purposes (creating code snippets
45
    that transforms should emit, as well as unit testing).
46

47
    code - a unicode string containing Cython (module-level) code
48
    name - a descriptive name for the code source (to use in error messages etc.)
49
    in_utility_code - used to suppress some messages from utility code. False by default
50
                      because some generated code snippets like properties and dataclasses
51
                      probably want to see those messages.
52

53
    RETURNS
54

55
    The tree, i.e. a ModuleNode. The ModuleNode's scope attribute is
56
    set to the scope used when parsing.
57
    """
58
    if context is None:
59
        context = StringParseContext(name)
60
    # Since source files carry an encoding, it makes sense in this context
61
    # to use a unicode string so that code fragments don't have to bother
62
    # with encoding. This means that test code passed in should not have an
63
    # encoding header.
64
    assert isinstance(code, str), "unicode code snippets only please"
65
    encoding = "UTF-8"
66

67
    module_name = name
68
    if initial_pos is None:
69
        initial_pos = (name, 1, 0)
70
    code_source = StringSourceDescriptor(name, code)
71
    if in_utility_code:
72
        code_source.in_utility_code = True
73

74
    scope = context.find_module(module_name, pos=initial_pos, need_pxd=False)
75

76
    buf = StringIO(code)
77

78
    scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
79
                     scope = scope, context = context, initial_pos = initial_pos)
80
    ctx = Parsing.Ctx(allow_struct_enum_decorator=allow_struct_enum_decorator)
81

82
    if level is None:
83
        tree = Parsing.p_module(scanner, 0, module_name, ctx=ctx)
84
        tree.scope = scope
85
        tree.is_pxd = False
86
    else:
87
        tree = Parsing.p_code(scanner, level=level, ctx=ctx)
88

89
    tree.scope = scope
90
    return tree
91

92

93
class TreeCopier(VisitorTransform):
94
    def visit_Node(self, node):
95
        if node is None:
96
            return node
97
        else:
98
            c = node.clone_node()
99
            self.visitchildren(c)
100
            return c
101

102

103
class ApplyPositionAndCopy(TreeCopier):
104
    def __init__(self, pos):
105
        super().__init__()
106
        self.pos = pos
107

108
    def visit_Node(self, node):
109
        copy = super().visit_Node(node)
110
        copy.pos = self.pos
111
        return copy
112

113

114
class TemplateTransform(VisitorTransform):
115
    """
116
    Makes a copy of a template tree while doing substitutions.
117

118
    A dictionary "substitutions" should be passed in when calling
119
    the transform; mapping names to replacement nodes. Then replacement
120
    happens like this:
121
     - If an ExprStatNode contains a single NameNode, whose name is
122
       a key in the substitutions dictionary, the ExprStatNode is
123
       replaced with a copy of the tree given in the dictionary.
124
       It is the responsibility of the caller that the replacement
125
       node is a valid statement.
126
     - If a single NameNode is otherwise encountered, it is replaced
127
       if its name is listed in the substitutions dictionary in the
128
       same way. It is the responsibility of the caller to make sure
129
       that the replacement nodes is a valid expression.
130

131
    Also a list "temps" should be passed. Any names listed will
132
    be transformed into anonymous, temporary names.
133

134
    Currently supported for tempnames is:
135
    NameNode
136
    (various function and class definition nodes etc. should be added to this)
137

138
    Each replacement node gets the position of the substituted node
139
    recursively applied to every member node.
140
    """
141

142
    temp_name_counter = 0
143

144
    def __call__(self, node, substitutions, temps, pos):
145
        self.substitutions = substitutions
146
        self.pos = pos
147
        tempmap = {}
148
        temphandles = []
149
        for temp in temps:
150
            TemplateTransform.temp_name_counter += 1
151
            handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)
152
            tempmap[temp] = handle
153
            temphandles.append(handle)
154
        self.tempmap = tempmap
155
        result = super().__call__(node)
156
        if temps:
157
            result = UtilNodes.TempsBlockNode(self.get_pos(node),
158
                                              temps=temphandles,
159
                                              body=result)
160
        return result
161

162
    def get_pos(self, node):
163
        if self.pos:
164
            return self.pos
165
        else:
166
            return node.pos
167

168
    def visit_Node(self, node):
169
        if node is None:
170
            return None
171
        else:
172
            c = node.clone_node()
173
            if self.pos is not None:
174
                c.pos = self.pos
175
            self.visitchildren(c)
176
            return c
177

178
    def try_substitution(self, node, key):
179
        sub = self.substitutions.get(key)
180
        if sub is not None:
181
            pos = self.pos
182
            if pos is None: pos = node.pos
183
            return ApplyPositionAndCopy(pos)(sub)
184
        else:
185
            return self.visit_Node(node)  # make copy as usual
186

187
    def visit_NameNode(self, node):
188
        temphandle = self.tempmap.get(node.name)
189
        if temphandle:
190
            # Replace name with temporary
191
            return temphandle.ref(self.get_pos(node))
192
        else:
193
            return self.try_substitution(node, node.name)
194

195
    def visit_ExprStatNode(self, node):
196
        # If an expression-as-statement consists of only a replaceable
197
        # NameNode, we replace the entire statement, not only the NameNode
198
        if isinstance(node.expr, NameNode):
199
            return self.try_substitution(node, node.expr.name)
200
        else:
201
            return self.visit_Node(node)
202

203

204
def copy_code_tree(node):
205
    return TreeCopier()(node)
206

207

208
_match_indent = re.compile("^ *").match
209

210

211
def strip_common_indent(lines):
212
    """Strips empty lines and common indentation from the list of strings given in lines"""
213
    # TODO: Facilitate textwrap.indent instead
214
    lines = [x for x in lines if x.strip() != ""]
215
    if lines:
216
        minindent = min([len(_match_indent(x).group(0)) for x in lines])
217
        lines = [x[minindent:] for x in lines]
218
    return lines
219

220

221
class TreeFragment:
222
    def __init__(self, code, name=None, pxds=None, temps=None, pipeline=None, level=None, initial_pos=None):
223
        if pxds is None:
224
            pxds = {}
225
        if temps is None:
226
            temps = []
227
        if pipeline is None:
228
            pipeline = []
229
        if not name:
230
            name = "(tree fragment)"
231

232
        if isinstance(code, str):
233
            def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n")))
234

235
            fmt_code = fmt(code)
236
            fmt_pxds = {}
237
            for key, value in pxds.items():
238
                fmt_pxds[key] = fmt(value)
239
            mod = t = parse_from_strings(name, fmt_code, fmt_pxds, level=level, initial_pos=initial_pos)
240
            if level is None:
241
                t = t.body  # Make sure a StatListNode is at the top
242
            if not isinstance(t, StatListNode):
243
                t = StatListNode(pos=mod.pos, stats=[t])
244
            for transform in pipeline:
245
                if transform is None:
246
                    continue
247
                t = transform(t)
248
            self.root = t
249
        elif isinstance(code, Node):
250
            if pxds:
251
                raise NotImplementedError()
252
            self.root = code
253
        else:
254
            raise ValueError("Unrecognized code format (accepts unicode and Node)")
255
        self.temps = temps
256

257
    def copy(self):
258
        return copy_code_tree(self.root)
259

260
    def substitute(self, nodes=None, temps=None, pos = None):
261
        if nodes is None:
262
            nodes = {}
263
        if temps is None:
264
            temps = []
265
        return TemplateTransform()(self.root,
266
                                   substitutions = nodes,
267
                                   temps = self.temps + temps, pos = pos)
268

269

270
class SetPosTransform(VisitorTransform):
271
    def __init__(self, pos):
272
        super().__init__()
273
        self.pos = pos
274

275
    def visit_Node(self, node):
276
        node.pos = self.pos
277
        self.visitchildren(node)
278
        return node
279