cython

1
"""
2
A Cython plugin for coverage.py
3

4
Requires the coverage package at least in version 4.0 (which added the plugin API).
5

6
This plugin requires the generated C sources to be available, next to the extension module.
7
It parses the C file and reads the original source files from it, which are stored in C comments.
8
It then reports a source file to coverage.py when it hits one of its lines during line tracing.
9

10
Basically, Cython can (on request) emit explicit trace calls into the C code that it generates,
11
and as a general human debugging helper, it always copies the current source code line
12
(and its surrounding context) into the C files before it generates code for that line, e.g.
13

14
::
15

16
      /* "line_trace.pyx":147
17
       * def cy_add_with_nogil(a,b):
18
       *     cdef int z, x=a, y=b         # 1
19
       *     with nogil:                  # 2             # <<<<<<<<<<<<<<
20
       *         z = 0                    # 3
21
       *         z += cy_add_nogil(x, y)  # 4
22
       */
23
       __Pyx_TraceLine(147,6,1,__PYX_ERR(0, 147, __pyx_L4_error))
24
      [C code generated for file line_trace.pyx, line 147, follows here]
25

26
The crux is that multiple source files can contribute code to a single C (or C++) file
27
(and thus, to a single extension module) besides the main module source file (.py/.pyx),
28
usually shared declaration files (.pxd) but also literally included files (.pxi).
29

30
Therefore, the coverage plugin doesn't actually try to look at the file that happened
31
to contribute the current source line for the trace call, but simply looks up the single
32
.c file from which the extension was compiled (which usually lies right next to it after
33
the build, having the same name), and parses the code copy comments from that .c file
34
to recover the original source files and their code as a line-to-file mapping.
35

36
That mapping is then used to report the ``__Pyx_TraceLine()`` calls to the coverage tool.
37
The plugin also reports the line of source code that it found in the C file to the coverage
38
tool to support annotated source representations.  For this, again, it does not look at the
39
actual source files but only reports the source code that it found in the C code comments.
40

41
Apart from simplicity (read one file instead of finding and parsing many), part of the
42
reasoning here is that any line in the original sources for which there is no comment line
43
(and trace call) in the generated C code cannot count as executed, really, so the C code
44
comments are a very good source for coverage reporting.  They already filter out purely
45
declarative code lines that do not contribute executable code, and such (missing) lines
46
can then be marked as excluded from coverage analysis.
47
"""
48

49

50
import re
51
import os.path
52
import sys
53
from collections import defaultdict
54

55
from coverage.plugin import CoveragePlugin, FileTracer, FileReporter  # requires coverage.py 4.0+
56
from coverage.files import canonical_filename
57

58
from .Utils import find_root_package_dir, is_package_dir, is_cython_generated_file, open_source_file
59

60

61
from . import __version__
62

63

64
C_FILE_EXTENSIONS = ['.c', '.cpp', '.cc', '.cxx']
65
MODULE_FILE_EXTENSIONS = set(['.py', '.pyx', '.pxd'] + C_FILE_EXTENSIONS)
66

67

68
def _find_c_source(base_path):
69
    file_exists = os.path.exists
70
    for ext in C_FILE_EXTENSIONS:
71
        file_name = base_path + ext
72
        if file_exists(file_name):
73
            return file_name
74
    return None
75

76

77
def _find_dep_file_path(main_file, file_path, relative_path_search=False):
78
    abs_path = os.path.abspath(file_path)
79
    if not os.path.exists(abs_path) and (file_path.endswith('.pxi') or
80
                                         relative_path_search):
81
        # files are looked up relative to the main source file
82
        rel_file_path = os.path.join(os.path.dirname(main_file), file_path)
83
        if os.path.exists(rel_file_path):
84
            abs_path = os.path.abspath(rel_file_path)
85

86
        abs_no_ext = os.path.splitext(abs_path)[0]
87
        file_no_ext, extension = os.path.splitext(file_path)
88
        # We check if the paths match by matching the directories in reverse order.
89
        # pkg/module.pyx /long/absolute_path/bla/bla/site-packages/pkg/module.c should match.
90
        # this will match the pairs: module-module and pkg-pkg. After which there is nothing left to zip.
91
        abs_no_ext = os.path.normpath(abs_no_ext)
92
        file_no_ext = os.path.normpath(file_no_ext)
93
        matching_paths = zip(reversed(abs_no_ext.split(os.sep)), reversed(file_no_ext.split(os.sep)))
94
        for one, other in matching_paths:
95
            if one != other:
96
                break
97
        else:  # No mismatches detected
98
            matching_abs_path = os.path.splitext(main_file)[0] + extension
99
            if os.path.exists(matching_abs_path):
100
                return canonical_filename(matching_abs_path)
101

102
    # search sys.path for external locations if a valid file hasn't been found
103
    if not os.path.exists(abs_path):
104
        for sys_path in sys.path:
105
            test_path = os.path.realpath(os.path.join(sys_path, file_path))
106
            if os.path.exists(test_path):
107
                return canonical_filename(test_path)
108
    return canonical_filename(abs_path)
109

110

111
def _offset_to_line(offset):
112
    return offset >> 9
113

114

115
class Plugin(CoveragePlugin):
116
    # map from traced file paths to absolute file paths
117
    _file_path_map = None
118
    # map from traced file paths to corresponding C files
119
    _c_files_map = None
120
    # map from parsed C files to their content
121
    _parsed_c_files = None
122
    # map from traced files to lines that are excluded from coverage
123
    _excluded_lines_map = None
124
    # list of regex patterns for lines to exclude
125
    _excluded_line_patterns = ()
126

127
    def sys_info(self):
128
        return [('Cython version', __version__)]
129

130
    def configure(self, config):
131
        # Entry point for coverage "configurer".
132
        # Read the regular expressions from the coverage config that match lines to be excluded from coverage.
133
        self._excluded_line_patterns = config.get_option("report:exclude_lines")
134

135
    def file_tracer(self, filename):
136
        """
137
        Try to find a C source file for a file path found by the tracer.
138
        """
139
        if filename.startswith('<') or filename.startswith('memory:'):
140
            return None
141
        c_file = py_file = None
142
        filename = canonical_filename(os.path.abspath(filename))
143
        if self._c_files_map and filename in self._c_files_map:
144
            c_file = self._c_files_map[filename][0]
145

146
        if c_file is None:
147
            c_file, py_file = self._find_source_files(filename)
148
            if not c_file:
149
                return None  # unknown file
150

151
            # parse all source file paths and lines from C file
152
            # to learn about all relevant source files right away (pyx/pxi/pxd)
153
            # FIXME: this might already be too late if the first executed line
154
            #        is not from the main .pyx file but a file with a different
155
            #        name than the .c file (which prevents us from finding the
156
            #        .c file)
157
            _, code = self._read_source_lines(c_file, filename)
158
            if code is None:
159
                return None  # no source found
160

161
        if self._file_path_map is None:
162
            self._file_path_map = {}
163
        return CythonModuleTracer(filename, py_file, c_file, self._c_files_map, self._file_path_map)
164

165
    def file_reporter(self, filename):
166
        # TODO: let coverage.py handle .py files itself
167
        #ext = os.path.splitext(filename)[1].lower()
168
        #if ext == '.py':
169
        #    from coverage.python import PythonFileReporter
170
        #    return PythonFileReporter(filename)
171

172
        filename = canonical_filename(os.path.abspath(filename))
173
        if self._c_files_map and filename in self._c_files_map:
174
            c_file, rel_file_path, code = self._c_files_map[filename]
175
        else:
176
            c_file, _ = self._find_source_files(filename)
177
            if not c_file:
178
                return None  # unknown file
179
            rel_file_path, code = self._read_source_lines(c_file, filename)
180
            if code is None:
181
                return None  # no source found
182
        return CythonModuleReporter(
183
            c_file,
184
            filename,
185
            rel_file_path,
186
            code,
187
            self._excluded_lines_map.get(rel_file_path, frozenset())
188
        )
189

190
    def _find_source_files(self, filename):
191
        basename, ext = os.path.splitext(filename)
192
        ext = ext.lower()
193
        if ext in MODULE_FILE_EXTENSIONS:
194
            pass
195
        elif ext == '.pyd':
196
            # Windows extension module
197
            platform_suffix = re.search(r'[.]cp[0-9]+-win[_a-z0-9]*$', basename, re.I)
198
            if platform_suffix:
199
                basename = basename[:platform_suffix.start()]
200
        elif ext == '.so':
201
            # Linux/Unix/Mac extension module
202
            platform_suffix = re.search(r'[.](?:cpython|pypy)-[0-9]+[-_a-z0-9]*$', basename, re.I)
203
            if platform_suffix:
204
                basename = basename[:platform_suffix.start()]
205
        elif ext == '.pxi':
206
            # if we get here, it means that the first traced line of a Cython module was
207
            # not in the main module but in an include file, so try a little harder to
208
            # find the main source file
209
            self._find_c_source_files(os.path.dirname(filename), filename)
210
            if filename in self._c_files_map:
211
                return self._c_files_map[filename][0], None
212
        else:
213
            # none of our business
214
            return None, None
215

216
        c_file = filename if ext in C_FILE_EXTENSIONS else _find_c_source(basename)
217
        if c_file is None:
218
            # a module "pkg/mod.so" can have a source file "pkg/pkg.mod.c"
219
            package_root = find_root_package_dir.uncached(filename)
220
            package_path = os.path.relpath(basename, package_root).split(os.path.sep)
221
            if len(package_path) > 1:
222
                test_basepath = os.path.join(os.path.dirname(filename), '.'.join(package_path))
223
                c_file = _find_c_source(test_basepath)
224

225
        py_source_file = None
226
        if c_file:
227
            py_source_file = os.path.splitext(c_file)[0] + '.py'
228
            if not os.path.exists(py_source_file):
229
                py_source_file = None
230
            if not is_cython_generated_file(c_file, if_not_found=False):
231
                if py_source_file and os.path.exists(c_file):
232
                    # if we did not generate the C file,
233
                    # then we probably also shouldn't care about the .py file.
234
                    py_source_file = None
235
                c_file = None
236

237
        return c_file, py_source_file
238

239
    def _find_c_source_files(self, dir_path, source_file):
240
        """
241
        Desperately parse all C files in the directory or its package parents
242
        (not re-descending) to find the (included) source file in one of them.
243
        """
244
        if not os.path.isdir(dir_path):
245
            return
246
        splitext = os.path.splitext
247
        for filename in os.listdir(dir_path):
248
            ext = splitext(filename)[1].lower()
249
            if ext in C_FILE_EXTENSIONS:
250
                self._read_source_lines(os.path.join(dir_path, filename), source_file)
251
                if source_file in self._c_files_map:
252
                    return
253
        # not found? then try one package up
254
        if is_package_dir(dir_path):
255
            self._find_c_source_files(os.path.dirname(dir_path), source_file)
256

257
    def _read_source_lines(self, c_file, sourcefile):
258
        """
259
        Parse a Cython generated C/C++ source file and find the executable lines.
260
        Each executable line starts with a comment header that states source file
261
        and line number, as well as the surrounding range of source code lines.
262
        """
263
        if self._parsed_c_files is None:
264
            self._parsed_c_files = {}
265
        if c_file in self._parsed_c_files:
266
            code_lines = self._parsed_c_files[c_file]
267
        else:
268
            code_lines = self._parse_cfile_lines(c_file)
269
            self._parsed_c_files[c_file] = code_lines
270

271
        if self._c_files_map is None:
272
            self._c_files_map = {}
273

274
        for filename, code in code_lines.items():
275
            abs_path = _find_dep_file_path(c_file, filename,
276
                                           relative_path_search=True)
277
            self._c_files_map[abs_path] = (c_file, filename, code)
278

279
        if sourcefile not in self._c_files_map:
280
            return (None,) * 2  # e.g. shared library file
281
        return self._c_files_map[sourcefile][1:]
282

283
    def _parse_cfile_lines(self, c_file):
284
        """
285
        Parse a C file and extract all source file lines that generated executable code.
286
        """
287
        match_source_path_line = re.compile(r' */[*] +"(.*)":([0-9]+)$').match
288
        match_current_code_line = re.compile(r' *[*] (.*) # <<<<<<+$').match
289
        match_comment_end = re.compile(r' *[*]/$').match
290
        match_trace_line = re.compile(r' *__Pyx_TraceLine\(([0-9]+),').match
291
        not_executable = re.compile(
292
            r'\s*c(?:type)?def\s+'
293
            r'(?:(?:public|external)\s+)?'
294
            r'(?:struct|union|enum|class)'
295
            r'(\s+[^:]+|)\s*:'
296
        ).match
297
        if self._excluded_line_patterns:
298
            line_is_excluded = re.compile("|".join(["(?:%s)" % regex for regex in self._excluded_line_patterns])).search
299
        else:
300
            line_is_excluded = lambda line: False
301

302
        code_lines = defaultdict(dict)
303
        executable_lines = defaultdict(set)
304
        current_filename = None
305
        if self._excluded_lines_map is None:
306
            self._excluded_lines_map = defaultdict(set)
307

308
        with open(c_file, encoding='utf8') as lines:
309
            lines = iter(lines)
310
            for line in lines:
311
                match = match_source_path_line(line)
312
                if not match:
313
                    if '__Pyx_TraceLine(' in line and current_filename is not None:
314
                        trace_line = match_trace_line(line)
315
                        if trace_line:
316
                            lineno = int(trace_line.group(1))
317
                            executable_lines[current_filename].add(lineno)
318
                    continue
319
                filename, lineno = match.groups()
320
                current_filename = filename
321
                lineno = int(lineno)
322
                for comment_line in lines:
323
                    match = match_current_code_line(comment_line)
324
                    if match:
325
                        code_line = match.group(1).rstrip()
326
                        if not_executable(code_line):
327
                            break
328
                        if line_is_excluded(code_line):
329
                            self._excluded_lines_map[filename].add(lineno)
330
                            break
331
                        code_lines[filename][lineno] = code_line
332
                        break
333
                    elif match_comment_end(comment_line):
334
                        # unexpected comment format - false positive?
335
                        break
336

337
        # Remove lines that generated code but are not traceable.
338
        for filename, lines in code_lines.items():
339
            dead_lines = set(lines).difference(executable_lines.get(filename, ()))
340
            for lineno in dead_lines:
341
                del lines[lineno]
342
        return code_lines
343

344

345
class CythonModuleTracer(FileTracer):
346
    """
347
    Find the Python/Cython source file for a Cython module.
348
    """
349
    def __init__(self, module_file, py_file, c_file, c_files_map, file_path_map):
350
        super().__init__()
351
        self.module_file = module_file
352
        self.py_file = py_file
353
        self.c_file = c_file
354
        self._c_files_map = c_files_map
355
        self._file_path_map = file_path_map
356

357
    def has_dynamic_source_filename(self):
358
        return True
359

360
    def dynamic_source_filename(self, filename, frame):
361
        """
362
        Determine source file path.  Called by the function call tracer.
363
        """
364
        source_file = frame.f_code.co_filename
365
        try:
366
            return self._file_path_map[source_file]
367
        except KeyError:
368
            pass
369
        abs_path = _find_dep_file_path(filename, source_file)
370

371
        if self.py_file and source_file[-3:].lower() == '.py':
372
            # always let coverage.py handle this case itself
373
            self._file_path_map[source_file] = self.py_file
374
            return self.py_file
375

376
        assert self._c_files_map is not None
377
        if abs_path not in self._c_files_map:
378
            self._c_files_map[abs_path] = (self.c_file, source_file, None)
379
        self._file_path_map[source_file] = abs_path
380
        return abs_path
381

382

383
class CythonModuleReporter(FileReporter):
384
    """
385
    Provide detailed trace information for one source file to coverage.py.
386
    """
387
    def __init__(self, c_file, source_file, rel_file_path, code, excluded_lines):
388
        super().__init__(source_file)
389
        self.name = rel_file_path
390
        self.c_file = c_file
391
        self._code = code
392
        self._excluded_lines = excluded_lines
393

394
    def lines(self):
395
        """
396
        Return set of line numbers that are possibly executable.
397
        """
398
        return set(self._code)
399

400
    def excluded_lines(self):
401
        """
402
        Return set of line numbers that are excluded from coverage.
403
        """
404
        return self._excluded_lines
405

406
    def _iter_source_tokens(self):
407
        current_line = 1
408
        for line_no, code_line in sorted(self._code.items()):
409
            while line_no > current_line:
410
                yield []
411
                current_line += 1
412
            yield [('txt', code_line)]
413
            current_line += 1
414

415
    def source(self):
416
        """
417
        Return the source code of the file as a string.
418
        """
419
        if os.path.exists(self.filename):
420
            with open_source_file(self.filename) as f:
421
                return f.read()
422
        else:
423
            return '\n'.join(
424
                (tokens[0][1] if tokens else '')
425
                for tokens in self._iter_source_tokens())
426

427
    def source_token_lines(self):
428
        """
429
        Iterate over the source code tokens.
430
        """
431
        if os.path.exists(self.filename):
432
            with open_source_file(self.filename) as f:
433
                for line in f:
434
                    yield [('txt', line.rstrip('\n'))]
435
        else:
436
            for line in self._iter_source_tokens():
437
                yield [('txt', line)]
438

439

440
def coverage_init(reg, options):
441
    plugin = Plugin()
442
    reg.add_configurer(plugin)
443
    reg.add_file_tracer(plugin)
444