git-cinnabar

1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this
3
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4

5
import hashlib
6
import json
7

8
from tasks import (
9
    Task,
10
    TaskEnvironment,
11
    bash_command,
12
    join_command,
13
)
14
from variables import TC_REPO_NAME
15

16

17
def sources_list(snapshot, sections):
18
    for idx, (archive, dist) in enumerate(sections):
19
        if not snapshot:
20
            yield "deb http://archive.debian.org/{} {} main".format(
21
                archive,
22
                dist,
23
            )
24
            continue
25
        yield "deb http://snapshot.debian.org/archive/{}/{} {} main".format(
26
            archive,
27
            snapshot,
28
            dist,
29
        )
30

31

32
LLVM_REPO = (
33
    "echo"
34
    " deb [signed-by=/usr/share/keyrings/llvm.gpg]"
35
    " https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-18 main"
36
    " > /etc/apt/sources.list.d/llvm.list"
37
)
38

39
DOCKER_IMAGES = {
40
    "base": {
41
        "from": "debian:bullseye-20220801",
42
        "commands": [
43
            "({}) > /etc/apt/sources.list".format(
44
                "; ".join(
45
                    "echo " + l
46
                    for l in sources_list(
47
                        "20220801T205040Z",
48
                        (
49
                            ("debian", "bullseye"),
50
                            ("debian", "bullseye-updates"),
51
                            ("debian-security", "bullseye-security"),
52
                        ),
53
                    )
54
                )
55
            ),
56
            "apt-get update -o Acquire::Check-Valid-Until=false",
57
            "apt-get install -y --no-install-recommends {}".format(
58
                " ".join(
59
                    [
60
                        "apt-transport-https",
61
                        "bzip2",
62
                        "ca-certificates",
63
                        "curl",
64
                        "gnupg2",
65
                        "libcurl3-gnutls",
66
                        "python-setuptools",
67
                        "python3-setuptools",
68
                        "python3-pip",
69
                        "unzip",
70
                        "xz-utils",
71
                        "zip",
72
                        "zstd",
73
                    ]
74
                )
75
            ),
76
            "apt-get clean",
77
            "curl -sO https://apt.llvm.org/llvm-snapshot.gpg.key",
78
            "gpg --no-default-keyring --keyring /usr/share/keyrings/llvm.gpg"
79
            " --import llvm-snapshot.gpg.key",
80
            "rm llvm-snapshot.gpg.key",
81
            "curl -sO http://snapshot.debian.org/archive/debian"
82
            "/20220326T025251Z/pool/main/p/python2-pip"
83
            "/python-pip_20.3.4%2Bdfsg-4_all.deb",
84
            "dpkg-deb -x python-pip*.deb /",
85
            "python2.7 -m pip install pip==20.3.4 wheel==0.37.1"
86
            " --upgrade --ignore-installed",
87
            "python3 -m pip install pip==20.3.4 wheel==0.37.1"
88
            " --upgrade --ignore-installed",
89
        ],
90
    },
91
    "build": {
92
        "from": "base",
93
        "commands": [
94
            LLVM_REPO,
95
            "apt-get update -o Acquire::Check-Valid-Until=false",
96
            "apt-get install -y --no-install-recommends {}".format(
97
                " ".join(
98
                    [
99
                        "clang-18",
100
                        "lld-18",
101
                        "git",
102
                        "make",
103
                        "patch",
104
                        "pkg-config",
105
                        "mmdebstrap",
106
                        "debian-archive-keyring",
107
                        "symlinks",
108
                        "fakechroot",
109
                        "gcc-mingw-w64-x86-64-win32",
110
                    ]
111
                )
112
            ),
113
            "for arch in amd64 arm64; do"
114
            " mmdebstrap -d"
115
            "  --architecture=$arch"
116
            "  --mode=chrootless"
117
            "  --variant=extract"
118
            "  --include=libc6-dev,libcurl4-gnutls-dev,zlib1g-dev,libgcc-6-dev"
119
            "  stretch sysroot-$arch"
120
            "  http://archive.debian.org/debian/ ;"
121
            " LD_PRELOAD=libfakechroot.so FAKECHROOT_BASE=$PWD/sysroot-$arch"
122
            "  symlinks -crv /;"
123
            "done",
124
            "apt-get clean",
125
        ],
126
    },
127
    "build-tools": {
128
        "from": "base",
129
        "commands": [
130
            "apt-get install -y --no-install-recommends {}".format(
131
                " ".join(
132
                    [
133
                        "gcc",
134
                        "git",
135
                        "libc6-dev",
136
                        "libcurl4-gnutls-dev",
137
                        "make",
138
                        "patch",
139
                        "python-dev",
140
                        "python3-dev",
141
                        "zlib1g-dev",
142
                    ]
143
                )
144
            ),
145
            "apt-get clean",
146
        ],
147
    },
148
    "codecov": {
149
        "from": "base",
150
        "commands": [
151
            "apt-get install -y --no-install-recommends {}".format(
152
                " ".join(
153
                    [
154
                        "gcc",
155
                        "git",
156
                        "python3-coverage",
157
                    ]
158
                )
159
            ),
160
            "apt-get clean",
161
            "ln -s /usr/bin/python3-coverage /usr/local/bin/coverage",
162
            "curl -o /usr/local/bin/codecov -sL {}".format(
163
                "https://github.com/codecov/uploader/releases/download"
164
                "/v0.1.0_9779/codecov-linux"
165
            ),
166
            "chmod +x /usr/local/bin/codecov",
167
            "curl -sL {} | tar -C /usr/local/bin -jxf -".format(
168
                "https://github.com/mozilla/grcov/releases/download/v0.8.7"
169
                "/grcov-x86_64-unknown-linux-gnu.tar.bz2"
170
            ),
171
        ],
172
    },
173
    "test": {
174
        "from": "base",
175
        "commands": [
176
            LLVM_REPO,
177
            "apt-get update -o Acquire::Check-Valid-Until=false",
178
            "apt-get install -y --no-install-recommends {}".format(
179
                " ".join(
180
                    [
181
                        "llvm-18",
182
                        "make",
183
                    ]
184
                )
185
            ),
186
            "apt-get clean",
187
            "pip3 install cram==0.7",
188
            "ln -s llvm-symbolizer-18 /usr/bin/llvm-symbolizer",
189
        ],
190
    },
191
}
192

193

194
class DockerImage(Task, metaclass=TaskEnvironment):
195
    PREFIX = "linux"
196
    cpu = "x86_64"
197
    os = "linux"
198

199
    def __init__(self, name):
200
        defn = DOCKER_IMAGES[name]
201
        base = defn["from"]
202
        self.name = name
203
        if ":" not in base:
204
            base = DockerImage.by_name(base)
205
        self.base = base
206
        self.definition = defn["commands"]
207

208
        Task.__init__(
209
            self,
210
            task_env=self,
211
            description="docker image: {}".format(name),
212
            index=self.index,
213
            expireIn="26 weeks",
214
            workerType="linux",
215
            image=base,
216
            dockerSave=True,
217
            command=self.definition,
218
        )
219

220
    def __str__(self):
221
        return "{}/{}:{}".format(TC_REPO_NAME, self.name, self.hexdigest)
222

223
    @property
224
    def index(self):
225
        return ".".join((self.PREFIX, self.name, self.hexdigest))
226

227
    @property
228
    def hexdigest(self):
229
        h = hashlib.sha1()
230
        h.update(str(self.base).encode())
231
        h.update(json.dumps(self.definition).encode())
232
        return h.hexdigest()
233

234
    def prepare_params(self, params):
235
        commands = ["mkdir artifacts"]
236
        image = params.pop("image", self)
237
        volumes = [
238
            kind.split(":", 1)[1]
239
            for mount in params.get("mounts", [])
240
            for kind in mount
241
            if ":" in kind
242
        ]
243
        if isinstance(image, DockerImage):
244
            params.setdefault("mounts", []).append({"file:dockerimage": image})
245
            image = "docker-archive:dockerimage"
246
        run_cmd = [
247
            "podman",
248
            "run",
249
            "--name=taskcontainer",
250
            "--volume=./artifacts:/artifacts",
251
            "--env=ARTIFACTS=/artifacts",
252
        ]
253
        if any(s.startswith("secrets:") for s in params.get("scopes", [])):
254
            # There's probably a better way, but it's simpler.
255
            run_cmd.append("--network=host")
256
        for v in volumes:
257
            run_cmd.append(f"--volume=./{v}:/{v}")
258
        for k, v in params.pop("env", {}).items():
259
            run_cmd.append(f"--env={k}={v}")
260
        for cap in params.pop("caps", []):
261
            run_cmd.append(f"--cap-add={cap}")
262
        run_cmd.append(image)
263
        run_cmd.extend(bash_command(*params["command"]))
264
        commands.append(join_command(*run_cmd))
265
        if params.pop("dockerSave", False):
266
            commands.extend(
267
                [
268
                    "exit_code=$?",
269
                    "podman commit taskcontainer taskcontainer",
270
                    "podman save taskcontainer"
271
                    " | zstd > artifacts/dockerImage.tar.zst",
272
                    "podman rm taskcontainer",
273
                    "exit $exit_code",
274
                ]
275
            )
276
            params["artifacts"] = ["dockerImage.tar.zst"]
277
        params["command"] = bash_command(*commands)
278

279
        if "artifacts" in params:
280
            params["artifacts"] = [f"artifacts/{a}" for a in params["artifacts"]]
281
        return params
282