git-cinnabar

1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this
3
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4

5
import hashlib
6
import os
7

8
import msys
9
from docker import DockerImage
10
from tasks import (
11
    Task,
12
    TaskEnvironment,
13
    Tool,
14
    parse_version,
15
)
16
from util import build_commit
17

18
MERCURIAL_VERSION = "6.8"
19
# Not using 2.46.0 because of
20
# https://lore.kernel.org/git/20240727191917.p64ul4jybpm2a7hm@glandium.org/
21
GIT_VERSION = "2.45.2"
22

23
ALL_MERCURIAL_VERSIONS = (
24
    "1.9.3",
25
    "2.0.2",
26
    "2.1.2",
27
    "2.2.3",
28
    "2.3.2",
29
    "2.4.2",
30
    "2.5.4",
31
    "2.6.3",
32
    "2.7.2",
33
    "2.8.2",
34
    "2.9.1",
35
    "3.0.1",
36
    "3.1.2",
37
    "3.2.4",
38
    "3.3.3",
39
    "3.4.2",
40
    "3.5.2",
41
    "3.6.3",
42
    "3.7.3",
43
    "3.8.4",
44
    "3.9.2",
45
    "4.0.2",
46
    "4.1.3",
47
    "4.2.2",
48
    "4.3.3",
49
    "4.4.2",
50
    "4.5.3",
51
    "4.6.2",
52
    "4.7.2",
53
    "4.8.2",
54
    "4.9.1",
55
    "5.0.2",
56
    "5.1.2",
57
    "5.2.2",
58
    "5.3.2",
59
    "5.4.2",
60
    "5.5.2",
61
    "5.6.1",
62
    "5.7.1",
63
    "5.8.1",
64
    "5.9.3",
65
    "6.0.3",
66
    "6.1.4",
67
    "6.2.3",
68
    "6.3.3",
69
    "6.4.2",
70
    "6.5.3",
71
    "6.6.3",
72
    "6.7.4",
73
    "6.8",
74
)
75

76
SOME_MERCURIAL_VERSIONS = (
77
    "1.9.3",
78
    "2.5.4",
79
    "3.4.2",
80
)
81

82
assert MERCURIAL_VERSION in ALL_MERCURIAL_VERSIONS
83
assert all(v in ALL_MERCURIAL_VERSIONS for v in SOME_MERCURIAL_VERSIONS)
84

85

86
def nproc(env):
87
    if env.os == "macos":
88
        return "sysctl -n hw.physicalcpu"
89
    return "nproc --all"
90

91

92
class Git(Task, metaclass=Tool):
93
    PREFIX = "git"
94

95
    def __init__(self, os_and_version):
96
        (os, version) = os_and_version.split(".", 1)
97
        self.os = os
98
        if os.endswith("osx"):
99
            build_image = TaskEnvironment.by_name("{}.build".format(os))
100
        else:
101
            build_image = DockerImage.by_name("build-tools")
102
        if os == "linux" or os.endswith("osx"):
103
            h = hashlib.sha1(build_image.hexdigest.encode())
104
            h.update(b"v4" if version == GIT_VERSION else b"v3")
105
            if os == "linux":
106
                description = "git v{}".format(version)
107
            else:
108
                env = build_image
109
                description = "git v{} {} {}".format(version, env.os, env.cpu)
110
            Task.__init__(
111
                self,
112
                task_env=build_image,
113
                description=description,
114
                index="{}.git.v{}".format(h.hexdigest(), version),
115
                expireIn="26 weeks",
116
                command=Task.checkout(
117
                    "git://git.kernel.org/pub/scm/git/git.git",
118
                    "v{}".format(version),
119
                    dest="git",
120
                )
121
                + Task.checkout()
122
                + (
123
                    [
124
                        "patch -d git -p1 < repo/CI/git-transport-disconnect.diff",
125
                    ]
126
                    if version == GIT_VERSION
127
                    else []
128
                )
129
                + [
130
                    "make -C git -j$({}) install prefix=/ NO_GETTEXT=1"
131
                    " NO_OPENSSL=1 NO_TCLTK=1 NO_UNCOMPRESS2=1"
132
                    " DESTDIR=$PWD/git".format(nproc(build_image)),
133
                    "tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst".format(version),
134
                ],
135
                artifact="git-{}.tar.zst".format(version),
136
            )
137
        else:
138
            env = TaskEnvironment.by_name("{}.build".format(os))
139
            raw_version = version
140
            if "windows" not in version:
141
                version = {
142
                    version: version + ".windows.1",
143
                    "2.17.1": "2.17.1.windows.2",
144
                }.get(version)
145
            if version.endswith(".windows.1"):
146
                min_ver = version[: -len(".windows.1")]
147
            else:
148
                min_ver = version.replace("windows.", "")
149
            h = hashlib.sha1(env.hexdigest.encode())
150
            h.update(b"v1")
151
            Task.__init__(
152
                self,
153
                task_env=build_image,
154
                description="git v{} {} {}".format(version, env.os, env.cpu),
155
                index="{}.git.v{}".format(h.hexdigest(), raw_version),
156
                expireIn="26 weeks",
157
                command=[
158
                    "curl -L https://github.com/git-for-windows/git/releases/"
159
                    "download/v{}/MinGit-{}-{}-bit.zip"
160
                    " -o git.zip".format(version, min_ver, msys.bits(env.cpu)),
161
                    "unzip -d git git.zip",
162
                    "curl -L https://github.com/git-for-windows/git/releases/"
163
                    "download/v{}/Git-{}-{}-bit.tar.bz2 | "
164
                    "tar -C git --no-same-owner -jx "
165
                    "{}/libexec/git-core/git-http-backend.exe".format(
166
                        version,
167
                        min_ver,
168
                        msys.bits(env.cpu),
169
                        msys.mingw(env.cpu).lower(),
170
                    ),
171
                    "tar -c git | zstd -c > $ARTIFACTS/git-{}.tar.zst".format(
172
                        raw_version
173
                    ),
174
                ],
175
                artifact="git-{}.tar.zst".format(raw_version),
176
            )
177

178
    def mount(self):
179
        return {"directory:git": self}
180

181
    def install(self):
182
        if self.os.endswith(("linux", "osx")):
183
            return [
184
                "export PATH=$PWD/git/bin:$PATH",
185
                "export GIT_EXEC_PATH=$PWD/git/libexec/git-core",
186
                "export GIT_TEMPLATE_DIR=$PWD/git/share/git-core/templates",
187
            ]
188
        else:
189
            return []
190

191

192
class Hg(Task, metaclass=Tool):
193
    PREFIX = "hg"
194

195
    def __init__(self, os_and_version):
196
        (os, version) = os_and_version.split(".", 1)
197
        (version, suffix, _) = version.partition(".py3")
198
        if (
199
            suffix
200
            or len(version) == 40
201
            or parse_version(version) >= parse_version("6.2")
202
        ):
203
            python = "python3"
204
        else:
205
            python = "python2.7"
206
        if os == "linux":
207
            env = TaskEnvironment.by_name("{}.build-tools".format(os))
208
        else:
209
            env = TaskEnvironment.by_name("{}.build".format(os))
210
        kwargs = {}
211

212
        if len(version) == 40:
213
            # Assume it's a sha1
214
            pretty_version = "r{}{}".format(version, suffix)
215
            artifact_version = "99.0"
216
            expire = "2 weeks"
217
        else:
218
            pretty_version = "v{}{}".format(version, suffix)
219
            artifact_version = version
220
            expire = "26 weeks"
221
        desc = "hg {}".format(pretty_version)
222
        if os == "linux":
223
            platform_tag = "linux_x86_64"
224
            if python == "python3":
225
                python_tag = "cp39"
226
                abi_tag = "cp39"
227
            else:
228
                python_tag = "cp27"
229
                abi_tag = "cp27mu"
230
        else:
231
            desc = "{} {} {}".format(desc, env.os, env.cpu)
232
            if os.endswith("osx"):
233
                py_host_plat = "macosx-{}-{}".format(env.os_version, env.cpu)
234
                platform_tag = py_host_plat.replace(".", "_").replace("-", "_")
235
                if python == "python3":
236
                    python_tag = "cp311" if os == "arm64-osx" else "cp39"
237
                    abi_tag = python_tag
238
                else:
239
                    python_tag = "cp27"
240
                    abi_tag = "cp27m"
241
                env_ = kwargs.setdefault("env", {})
242
                env_.setdefault("MACOSX_DEPLOYMENT_TARGET", env.os_version)
243
                env_.setdefault("ARCHFLAGS", "-arch {}".format(env.cpu))
244
                env_.setdefault("_PYTHON_HOST_PLATFORM", py_host_plat)
245
            else:
246
                if python == "python3":
247
                    platform_tag = "mingw_x86_64"
248
                    python_tag = "cp310"
249
                    abi_tag = "cp310"
250
                else:
251
                    platform_tag = "mingw"
252
                    python_tag = "cp27"
253
                    abi_tag = "cp27m"
254

255
        artifact = "mercurial-{{}}-{}-{}-{}.whl".format(
256
            python_tag,
257
            abi_tag,
258
            platform_tag,
259
        )
260

261
        pre_command = []
262
        if len(version) == 40:
263
            hg = self.by_name("{}.{}".format(os, MERCURIAL_VERSION))
264
            kwargs.setdefault("mounts", []).append(hg.mount())
265
            pre_command.extend(hg.install())
266
            pre_command.extend(
267
                [
268
                    "hg clone https://www.mercurial-scm.org/repo/hg"
269
                    " -r {} mercurial-{}".format(version, version),
270
                    "rm -rf mercurial-{}/.hg".format(version),
271
                    "echo tag: {} > mercurial-{}/.hg_archival.txt".format(
272
                        artifact_version, version
273
                    ),
274
                ]
275
            )
276
        # 2.6.2 is the first version available on pypi
277
        elif parse_version("2.6.2") <= parse_version(version) and parse_version(
278
            version
279
        ) < parse_version("6.2"):
280
            # pip download does more than download, and while it runs setup.py
281
            # for version 6.2, a DistutilsPlatformError exception is thrown on
282
            # Windows.
283
            pre_command.append(
284
                "{} -m pip download --no-binary mercurial --no-deps"
285
                " --progress-bar off mercurial=={}".format(python, version)
286
            )
287
        else:
288
            url = "https://mercurial-scm.org/release/mercurial-{}.tar.gz"
289
            pre_command.append("curl -sLO {}".format(url.format(version)))
290

291
        if len(version) != 40:
292
            pre_command.append("tar -zxf mercurial-{}.tar.gz".format(version))
293

294
        if os.startswith("mingw"):
295
            # Work around https://bz.mercurial-scm.org/show_bug.cgi?id=6654
296
            # and https://bz.mercurial-scm.org/show_bug.cgi?id=6757
297
            pre_command.append(
298
                'sed -i "s/, output_dir=self.build_temp/'
299
                ", output_dir=self.build_temp, extra_postargs=[$EXTRA_FLAGS]/;"
300
                "/self.addlongpathsmanifest/d;"
301
                '" mercurial-{}/setup.py'.format(version)
302
            )
303
            if python == "python3":
304
                kwargs.setdefault("env", {}).setdefault("EXTRA_FLAGS", '"-municode"')
305
            pre_command.append(
306
                'sed -i "s/ifdef __GNUC__/if 0/"'
307
                " mercurial-{}/mercurial/exewrapper.c".format(version)
308
            )
309

310
        h = hashlib.sha1(env.hexdigest.encode())
311
        h.update(artifact.encode())
312
        if os.endswith("osx"):
313
            h.update(b"v2")
314
        elif os.startswith("mingw"):
315
            h.update(b"v4")
316
        else:
317
            h.update(b"v1")
318

319
        Task.__init__(
320
            self,
321
            task_env=env,
322
            description=desc,
323
            index="{}.hg.{}".format(h.hexdigest(), pretty_version),
324
            expireIn=expire,
325
            command=pre_command
326
            + [
327
                # pyproject.toml enables PEP 517, which can't be disabled.
328
                # pip wheel doesn't accept --build-option when PEP 517 is
329
                # enabled. --build-option is necessary on msys2 because
330
                # of problems with the bdist-dir otherwise.
331
                "rm -f mercurial-{}/pyproject.toml".format(version),
332
                "{} -m pip wheel -v --build-option -b --build-option"
333
                " $PWD/wheel -w $ARTIFACTS ./mercurial-{}".format(python, version),
334
            ],
335
            artifact=artifact.format(artifact_version),
336
            **kwargs,
337
        )
338

339
    def mount(self):
340
        return {f"file:{os.path.basename(self.artifacts[0])}": self}
341

342
    def install(self):
343
        filename = os.path.basename(self.artifacts[0])
344
        if "cp3" in filename:
345
            python = "python3"
346
        else:
347
            python = "python2.7"
348
        return ["{} -m pip install {}".format(python, filename)]
349

350

351
def install_rust(version="1.80.0", target="x86_64-unknown-linux-gnu"):
352
    rustup_opts = "-y --default-toolchain none"
353
    cargo_dir = "$HOME/.cargo/bin/"
354
    rustup = cargo_dir + "rustup"
355
    rust_install = [
356
        "curl -o rustup.sh https://sh.rustup.rs",
357
        "sh rustup.sh {rustup_opts}",
358
        "{rustup} install {version} --profile minimal",
359
        "{rustup} default {version}",
360
        "PATH={cargo_dir}:$PATH",
361
        "{rustup} target add {target}",
362
    ]
363
    loc = locals()
364
    return [r.format(**loc) for r in rust_install]
365

366

367
class Build(Task, metaclass=Tool):
368
    PREFIX = "build"
369

370
    def __init__(self, os_and_variant):
371
        os, variant = (os_and_variant.split(".", 1) + [""])[:2]
372
        env = TaskEnvironment.by_name(
373
            "{}.build".format(os.replace("arm64-linux", "linux"))
374
        )
375
        if os.startswith("mingw"):
376
            build_env = TaskEnvironment.by_name("linux.build")
377
        else:
378
            build_env = env
379

380
        artifact = "git-cinnabar"
381
        if os.startswith("mingw"):
382
            artifact += ".exe"
383
        artifacts = [artifact]
384

385
        def prefix(p, s):
386
            return p + s if s else s
387

388
        hash = None
389
        head = None
390
        desc_variant = variant
391
        extra_commands = []
392
        environ = {
393
            "WARNINGS_AS_ERRORS": "1",
394
        }
395
        cargo_flags = ["-vv", "--release"]
396
        cargo_features = ["self-update", "gitdev", "xz2/static", "bzip2/static"]
397
        rust_version = None
398
        if variant == "asan":
399
            if os.endswith("osx"):
400
                opt = "-O2"
401
            else:
402
                opt = "-Og"
403
            environ["TARGET_CFLAGS"] = " ".join(
404
                [
405
                    opt,
406
                    "-g",
407
                    "-fsanitize=address",
408
                    "-fno-omit-frame-pointer",
409
                    "-fPIC",
410
                ]
411
            )
412
            environ["RUSTFLAGS"] = " ".join(
413
                [
414
                    "-Zsanitizer=address",
415
                    "-Copt-level=1",
416
                    "-Cdebuginfo=full",
417
                    "-Cforce-frame-pointers=yes",
418
                ]
419
            )
420
        elif variant == "coverage":
421
            environ["TARGET_CFLAGS"] = " ".join(
422
                [
423
                    "-coverage",
424
                    "-fPIC",
425
                ]
426
            )
427
            artifacts += ["coverage.zip"]
428
            extra_commands = [
429
                "(cd repo && zip $ARTIFACTS/coverage.zip"
430
                ' $(find . -name "*.gcno" -not -name "build_script*"))',
431
            ]
432
            environ["RUSTFLAGS"] = " ".join(
433
                [
434
                    "-Zprofile",
435
                    "-Ccodegen-units=1",
436
                    "-Cinline-threshold=0",
437
                ]
438
            )
439
            # Build without --release
440
            cargo_flags.remove("--release")
441
            environ["CARGO_INCREMENTAL"] = "0"
442
        elif variant.startswith("old:"):
443
            head = variant[4:]
444
            hash = build_commit(head)
445
            variant = ""
446
        elif variant.startswith("rust-"):
447
            rust_version = variant[5:]
448
        elif variant:
449
            raise Exception("Unknown variant: {}".format(variant))
450

451
        if "osx" not in os:
452
            environ["CC"] = "clang-18"
453
        if os in ("linux", "arm64-linux"):
454
            cargo_features.append("curl-compat")
455

456
        if os.startswith("mingw"):
457
            cpu = msys.msys_cpu(env.cpu)
458
            rust_target = "{}-pc-windows-gnu".format(cpu)
459
        elif os.startswith("osx"):
460
            rust_target = "x86_64-apple-darwin"
461
        elif os.startswith("arm64-osx"):
462
            rust_target = "aarch64-apple-darwin"
463
        elif os == "linux":
464
            rust_target = "x86_64-unknown-linux-gnu"
465
        elif os == "arm64-linux":
466
            rust_target = "aarch64-unknown-linux-gnu"
467
        if "osx" not in os:
468
            for target in dict.fromkeys(
469
                ["x86_64-unknown-linux-gnu", rust_target]
470
            ).keys():
471
                arch = {
472
                    "x86_64": "amd64",
473
                    "aarch64": "arm64",
474
                }[target.partition("-")[0]]
475
                multiarch = target.replace("unknown-", "")
476
                TARGET = target.replace("-", "_").upper()
477
                environ[f"CARGO_TARGET_{TARGET}_LINKER"] = environ["CC"]
478
                if "linux" in os:
479
                    extra_link_arg = f"--sysroot=/sysroot-{arch}"
480
                if os.startswith("mingw"):
481
                    extra_link_arg = f"-L/usr/lib/gcc/{cpu}-w64-mingw32/10-win32"
482
                environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] = (
483
                    f"-C link-arg=--target={target} "
484
                    + f"-C link-arg={extra_link_arg} "
485
                    + "-C link-arg=-fuse-ld=lld-18"
486
                )
487
                rustflags = environ.pop("RUSTFLAGS", None)
488
                if rustflags:
489
                    environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] += f" {rustflags}"
490
                if "linux" in os:
491
                    environ[f"CFLAGS_{target}"] = f"--sysroot=/sysroot-{arch}"
492
            if "linux" in os:
493
                environ["PKG_CONFIG_PATH"] = ""
494
                environ["PKG_CONFIG_SYSROOT_DIR"] = f"/sysroot-{arch}"
495
                environ["PKG_CONFIG_LIBDIR"] = ":".join(
496
                    (
497
                        f"/sysroot-{arch}/usr/lib/pkgconfig",
498
                        f"/sysroot-{arch}/usr/lib/{multiarch}/pkgconfig",
499
                        f"/sysroot-{arch}/usr/share/pkgconfig",
500
                    )
501
                )
502
        if variant in ("coverage", "asan"):
503
            environ["RUSTC_BOOTSTRAP"] = "1"
504
        if rust_version:
505
            rust_install = install_rust(rust_version, target=rust_target)
506
        else:
507
            rust_install = install_rust(target=rust_target)
508
        cargo_flags.extend(["--target", rust_target])
509
        if cargo_features:
510
            cargo_flags.extend(["--features", ",".join(cargo_features)])
511
        for key, value in list(environ.items()):
512
            # RUSTFLAGS values in the environment override builds.rustflags
513
            # from .cargo/config.toml.
514
            if "RUSTFLAGS" in key:
515
                environ[key] = value + " -Cforce-unwind-tables=yes"
516

517
        hash = hash or build_commit()
518

519
        if os.startswith("osx"):
520
            environ.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.7")
521
        if os.startswith("arm64-osx"):
522
            environ.setdefault("MACOSX_DEPLOYMENT_TARGET", "11.0")
523

524
        cpu = "arm64" if os == "arm64-linux" else env.cpu
525
        Task.__init__(
526
            self,
527
            task_env=build_env,
528
            description="build {} {}{}".format(env.os, cpu, prefix(" ", desc_variant)),
529
            index="build.{}.{}.{}{}".format(hash, env.os, cpu, prefix(".", variant)),
530
            expireIn="26 weeks",
531
            command=Task.checkout(commit=head)
532
            + rust_install
533
            + [
534
                "(cd repo ; cargo build {})".format(" ".join(cargo_flags)),
535
                "mv repo/target/{}/{}/{} $ARTIFACTS/".format(
536
                    rust_target,
537
                    "release" if "--release" in cargo_flags else "debug",
538
                    artifact,
539
                ),
540
            ]
541
            + extra_commands,
542
            artifacts=artifacts,
543
            env=environ,
544
        )
545

546
    def mount(self):
547
        return {f"file:{os.path.basename(self.artifacts[0])}": self}
548

549
    def install(self):
550
        filename = os.path.basename(self.artifacts[0])
551
        return [
552
            f"cp {filename} repo/",
553
            "chmod +x repo/{}".format(filename),
554
            "$PWD/repo/{} setup".format(filename),
555
        ]
556