cython

1
from dataclasses import dataclass
2
import sys
3
import os
4
import hashlib
5
import shutil
6
import subprocess
7
from ..Utils import safe_makedirs, cached_function
8
import zipfile
9
from .. import __version__
10

11
try:
12
    import zlib
13

14
    zipfile_compression_mode = zipfile.ZIP_DEFLATED
15
except ImportError:
16
    zipfile_compression_mode = zipfile.ZIP_STORED
17

18
try:
19
    import gzip
20

21
    gzip_open = gzip.open
22
    gzip_ext = ".gz"
23
except ImportError:
24
    gzip_open = open
25
    gzip_ext = ""
26

27
zip_ext = ".zip"
28

29
MAX_CACHE_SIZE = 1024 * 1024 * 100
30

31
join_path = cached_function(os.path.join)
32

33

34
@cached_function
35
def file_hash(filename):
36
    path = os.path.normpath(filename)
37
    prefix = ("%d:%s" % (len(path), path)).encode("UTF-8")
38
    m = hashlib.sha256(prefix)
39
    with open(path, "rb") as f:
40
        data = f.read(65000)
41
        while data:
42
            m.update(data)
43
            data = f.read(65000)
44
    return m.hexdigest()
45

46

47
@cached_function
48
def get_cython_cache_dir():
49
    r"""
50
    Return the base directory containing Cython's caches.
51

52
    Priority:
53

54
    1. CYTHON_CACHE_DIR
55
    2. (OS X): ~/Library/Caches/Cython
56
       (posix not OS X): XDG_CACHE_HOME/cython if XDG_CACHE_HOME defined
57
    3. ~/.cython
58

59
    """
60
    if "CYTHON_CACHE_DIR" in os.environ:
61
        return os.environ["CYTHON_CACHE_DIR"]
62

63
    parent = None
64
    if os.name == "posix":
65
        if sys.platform == "darwin":
66
            parent = os.path.expanduser("~/Library/Caches")
67
        else:
68
            # this could fallback on ~/.cache
69
            parent = os.environ.get("XDG_CACHE_HOME")
70

71
    if parent and os.path.isdir(parent):
72
        return join_path(parent, "cython")
73

74
    # last fallback: ~/.cython
75
    return os.path.expanduser(join_path("~", ".cython"))
76

77

78
@dataclass
79
class FingerprintFlags:
80
    language: str = "c"
81
    py_limited_api: bool = False
82
    np_pythran: bool = False
83

84
    def get_fingerprint(self):
85
        return str((self.language, self.py_limited_api, self.np_pythran))
86

87

88
class Cache:
89
    def __init__(self, path, cache_size=None):
90
        if path is None:
91
            self.path = join_path(get_cython_cache_dir(), "compiler")
92
        else:
93
            self.path = path
94
        self.cache_size = cache_size if cache_size is not None else MAX_CACHE_SIZE
95
        if not os.path.exists(self.path):
96
            os.makedirs(self.path)
97

98
    def transitive_fingerprint(
99
        self, filename, dependencies, compilation_options, flags=FingerprintFlags()
100
    ):
101
        r"""
102
        Return a fingerprint of a cython file that is about to be cythonized.
103

104
        Fingerprints are looked up in future compilations. If the fingerprint
105
        is found, the cythonization can be skipped. The fingerprint must
106
        incorporate everything that has an influence on the generated code.
107
        """
108
        try:
109
            m = hashlib.sha256(__version__.encode("UTF-8"))
110
            m.update(file_hash(filename).encode("UTF-8"))
111
            for x in sorted(dependencies):
112
                if os.path.splitext(x)[1] not in (".c", ".cpp", ".h"):
113
                    m.update(file_hash(x).encode("UTF-8"))
114
            # Include the module attributes that change the compilation result
115
            # in the fingerprint. We do not iterate over module.__dict__ and
116
            # include almost everything here as users might extend Extension
117
            # with arbitrary (random) attributes that would lead to cache
118
            # misses.
119
            m.update(flags.get_fingerprint().encode("UTF-8"))
120
            m.update(compilation_options.get_fingerprint().encode("UTF-8"))
121
            return m.hexdigest()
122
        except OSError:
123
            return None
124

125
    def fingerprint_file(self, cfile, fingerprint, ext):
126
        return (
127
            join_path(self.path, "%s-%s" % (os.path.basename(cfile), fingerprint)) + ext
128
        )
129

130
    def lookup_cache(self, c_file, fingerprint):
131
        # Cython-generated c files are highly compressible.
132
        # (E.g. a compression ratio of about 10 for Sage).
133
        if not os.path.exists(self.path):
134
            safe_makedirs(self.path)
135
        gz_fingerprint_file = self.fingerprint_file(c_file, fingerprint, gzip_ext)
136
        if os.path.exists(gz_fingerprint_file):
137
            return gz_fingerprint_file
138
        zip_fingerprint_file = self.fingerprint_file(c_file, fingerprint, zip_ext)
139
        if os.path.exists(zip_fingerprint_file):
140
            return zip_fingerprint_file
141
        return None
142

143
    def load_from_cache(self, c_file, cached):
144
        ext = os.path.splitext(cached)[1]
145
        if ext == gzip_ext:
146
            os.utime(cached, None)
147
            with gzip_open(cached, "rb") as g:
148
                with open(c_file, "wb") as f:
149
                    shutil.copyfileobj(g, f)
150
        elif ext == zip_ext:
151
            os.utime(cached, None)
152
            dirname = os.path.dirname(c_file)
153
            with zipfile.ZipFile(cached) as z:
154
                for artifact in z.namelist():
155
                    z.extract(artifact, join_path(dirname, artifact))
156
        else:
157
            raise ValueError(f"Unsupported cache file extension: {ext}")
158

159
    def store_to_cache(self, c_file, fingerprint, compilation_result):
160
        artifacts = compilation_result.get_generated_source_files()
161
        if len(artifacts) == 1:
162
            fingerprint_file = self.fingerprint_file(c_file, fingerprint, gzip_ext)
163
            with open(c_file, "rb") as f:
164
                with gzip_open(fingerprint_file + ".tmp", "wb") as g:
165
                    shutil.copyfileobj(f, g)
166
        else:
167
            fingerprint_file = self.fingerprint_file(c_file, fingerprint, zip_ext)
168
            with zipfile.ZipFile(
169
                fingerprint_file + ".tmp", "w", zipfile_compression_mode
170
            ) as zip:
171
                for artifact in artifacts:
172
                    zip.write(artifact, os.path.basename(artifact))
173
        os.rename(fingerprint_file + ".tmp", fingerprint_file)
174

175
    def cleanup_cache(self, ratio=0.85):
176
        try:
177
            completed_process = subprocess.run(
178
                ["du", "-s", "-k", os.path.abspath(self.path)], stdout=subprocess.PIPE
179
            )
180
            stdout = completed_process.stdout
181
            if completed_process.returncode == 0:
182
                total_size = 1024 * int(stdout.strip().split()[0])
183
                if total_size < self.cache_size:
184
                    return
185
        except (OSError, ValueError):
186
            pass
187
        total_size = 0
188
        all = []
189
        for file in os.listdir(self.path):
190
            path = join_path(self.path, file)
191
            s = os.stat(path)
192
            total_size += s.st_size
193
            all.append((s.st_atime, s.st_size, path))
194
        if total_size > self.cache_size:
195
            for time, size, file in reversed(sorted(all)):
196
                os.unlink(file)
197
                total_size -= size
198
                if total_size < self.cache_size * ratio:
199
                    break
200