git-cinnabar

1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this
3
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4

5
import hashlib
6
import json
7
import os
8
import subprocess
9
import sys
10

11
BASE_DIR = os.path.dirname(__file__)
12
sys.path.append(BASE_DIR)
13
sys.path.append(os.path.join(BASE_DIR, ".."))
14

15
from itertools import chain
16

17
import osx  # noqa: F401
18
from tasks import (
19
    Task,
20
    TaskEnvironment,
21
    Tool,
22
    action,
23
    parse_version,
24
)
25
from tools import (
26
    ALL_MERCURIAL_VERSIONS,
27
    GIT_VERSION,
28
    MERCURIAL_VERSION,
29
    SOME_MERCURIAL_VERSIONS,
30
    Build,
31
    Git,
32
    Hg,
33
    nproc,
34
)
35
from variables import *  # noqa: F403
36

37

38
def git_rev_parse(committish):
39
    return subprocess.check_output(
40
        ["git", "rev-parse", committish], text=True, cwd=os.path.join(BASE_DIR, "..")
41
    ).strip()
42

43

44
def is_old_hg(version):
45
    # `version` is a sha1 for trunk, which means it's >= 3.6
46
    if len(version) == 40:
47
        return False
48
    try:
49
        version = [int(x) for x in version.split(".")]
50
    except ValueError:
51
        # Assume that an invalid version per the conversion above is
52
        # newer.
53
        return False
54
    return version < [3, 6]
55

56

57
UPGRADE_FROM = ()  # ('0.5.0',)
58

59

60
class TestTask(Task):
61
    coverage = []
62

63
    def __init__(self, **kwargs):
64
        git = kwargs.pop("git", GIT_VERSION)
65
        hg = kwargs.pop("hg", MERCURIAL_VERSION)
66
        hg_clone = kwargs.pop("hg_clone", None)
67
        commit = kwargs.pop("commit", None)
68
        task_env = kwargs.pop("task_env", "linux")
69
        variant = kwargs.pop("variant", None)
70
        build = kwargs.pop("build", None)
71
        clone = kwargs.pop("clone", TC_COMMIT)
72
        desc = kwargs.pop("description", None)
73
        short_desc = kwargs.pop("short_desc", "test")
74
        extra_desc = kwargs.pop("extra_desc", None)
75
        pre_command = kwargs.pop("pre_command", None)
76
        if build is None:
77
            build = "{}.{}".format(task_env, variant) if variant else task_env
78
            build = Build.by_name(build)
79
            kwargs.setdefault("mounts", []).append(build.mount())
80
            build = build.install()
81
        if variant:
82
            kwargs.setdefault("env", {})["VARIANT"] = variant
83
        env = TaskEnvironment.by_name("{}.test".format(task_env))
84
        command = []
85
        if pre_command:
86
            command.extend(pre_command)
87
        if hg:
88
            hg_task = Hg.by_name("{}.{}".format(task_env, hg))
89
            kwargs.setdefault("mounts", []).append(hg_task.mount())
90
            command.extend(hg_task.install())
91
            command.append("hg --version")
92
            if is_old_hg(hg):
93
                kwargs.setdefault("env", {})["NO_CLONEBUNDLES"] = "1"
94
        if git:
95
            git_task = Git.by_name("{}.{}".format(task_env, git))
96
            kwargs.setdefault("mounts", []).append(git_task.mount())
97
            command.extend(git_task.install())
98
            command.append("git --version")
99
        command.extend(Task.checkout(commit=commit))
100
        command.extend(build)
101
        if clone:
102
            kwargs.setdefault("mounts", []).append(
103
                {"file:bundle.git": Clone.by_name(clone)}
104
            )
105
            command.extend(
106
                [
107
                    "git init repo/hg.old.git",
108
                    "git -C repo/hg.old.git fetch ../../bundle.git refs/*:refs/*",
109
                    "git -C repo/hg.old.git remote add origin hg:${REPO#https:}",
110
                    "git -C repo/hg.old.git symbolic-ref HEAD"
111
                    " refs/heads/branches/default/tip",
112
                ]
113
            )
114
            kwargs.setdefault("env", {})["REPO"] = REPO
115
        command.extend(("repo/git-cinnabar --version",))
116
        if "command" not in kwargs or hg_clone:
117
            command += [
118
                "hg init repo/hg.pure.hg",
119
                "hg -R repo/hg.pure.hg unbundle bundle.hg",
120
            ]
121
            kwargs.setdefault("mounts", []).append(
122
                {"file:bundle.hg": HgClone.by_name(MERCURIAL_VERSION)}
123
            )
124
        if "command" in kwargs:
125
            kwargs["command"] = command + kwargs["command"]
126
        else:
127
            if commit:
128
                # Always use the current CI scripts
129
                command.append(
130
                    "git -C repo -c core.autocrlf=input checkout {} CI".format(
131
                        TC_COMMIT
132
                    )
133
                )
134
            output_sync = " --output-sync=target"
135
            if env.os == "macos":
136
                output_sync = ""
137
            kwargs["command"] = command + [
138
                "make -C repo -f CI/tests.mk -j$({}){}".format(nproc(env), output_sync),
139
            ]
140

141
        if variant == "coverage":
142
            kwargs["command"].extend(
143
                [
144
                    "shopt -s nullglob",
145
                    "cd repo",
146
                    "zip $ARTIFACTS/coverage.zip" ' $(find . -name "*.gcda")',
147
                    "cd ..",
148
                    "shopt -u nullglob",
149
                ]
150
            )
151
            artifact = kwargs.pop("artifact", None)
152
            artifacts = kwargs.setdefault("artifacts", [])
153
            assert not (artifacts and artifact)
154
            if artifact:
155
                artifacts.push(artifact)
156
            artifacts.append("coverage.zip")
157
            self.coverage.append(self)
158
        elif variant == "asan" and task_env == "linux":
159
            kwargs["caps"] = ["SYS_PTRACE"]
160
        if not desc:
161
            desc = "{} w/ git-{} hg-{}".format(
162
                short_desc, git, "r" + hg if len(hg) == 40 else hg
163
            )
164
            if variant and variant != "coverage":
165
                desc = " ".join((desc, variant))
166
        if extra_desc:
167
            desc = " ".join((desc, extra_desc))
168
        if task_env != "linux":
169
            desc = " ".join((desc, env.os, env.cpu))
170
        kwargs["description"] = desc
171
        Task.__init__(self, task_env=env, **kwargs)
172

173

174
class Clone(TestTask, metaclass=Tool):
175
    PREFIX = "clone"
176

177
    def __init__(self, version):
178
        sha1 = git_rev_parse(version)
179
        expireIn = "26 weeks"
180
        kwargs = {}
181
        if version == TC_COMMIT or len(version) == 40:
182
            if version == TC_COMMIT:
183
                build = Build.by_name("linux")
184
            else:
185
                build = Build.by_name("linux.old:{}".format(version))
186
            kwargs.setdefault("mounts", []).append(build.mount())
187
            download = build.install()
188
            expireIn = "26 weeks"
189
        elif parse_version(version) < parse_version("0.6.0"):
190
            download = ["repo/git-cinnabar download"]
191
            if parse_version(version) < parse_version("0.5.7"):
192
                kwargs["git"] = "2.30.2"
193
        else:
194
            download = ["repo/download.py"]
195
        if REPO == DEFAULT_REPO:
196
            index = "bundle.{}".format(sha1)
197
        else:
198
            index = "bundle.{}.{}".format(hashlib.sha1(REPO).hexdigest(), sha1)
199
        TestTask.__init__(
200
            self,
201
            hg=MERCURIAL_VERSION,
202
            hg_clone=True,
203
            description="clone w/ {}".format(version),
204
            index=index,
205
            expireIn=expireIn,
206
            build=download,
207
            commit=sha1,
208
            clone=False,
209
            command=[
210
                "PATH=$PWD/repo:$PATH"
211
                " git -c fetch.prune=true clone -n hg::$PWD/repo/hg.pure.hg"
212
                " hg.old.git",
213
                "git -C hg.old.git bundle create $ARTIFACTS/bundle.git --all",
214
            ],
215
            artifact="bundle.git",
216
            priority="high",
217
            **kwargs,
218
        )
219

220

221
class HgClone(Task, metaclass=Tool):
222
    PREFIX = "hgclone"
223

224
    def __init__(self, version):
225
        if REPO == DEFAULT_REPO:
226
            index = "hgclone.{}".format(version)
227
        else:
228
            index = "hgclone.{}.{}".format(hashlib.sha1(REPO).hexdigest(), version)
229
        hg_task = Hg.by_name(f"linux.{version}")
230
        Task.__init__(
231
            self,
232
            task_env=TaskEnvironment.by_name("linux.test"),
233
            description=f"hg clone w/ {version}",
234
            index=index,
235
            expireIn="26 weeks",
236
            command=hg_task.install()
237
            + [
238
                "hg clone -U --stream $REPO repo",
239
                "hg -R repo bundle -t none-v1 -a $ARTIFACTS/bundle.hg",
240
            ],
241
            mounts=[hg_task.mount()],
242
            artifact="bundle.hg",
243
            env={
244
                "REPO": REPO,
245
            },
246
            priority="high",
247
        )
248

249

250
@action("decision")
251
def decision():
252
    for env in ("linux", "mingw64", "osx", "arm64-osx"):
253
        # Can't spawn osx workers from pull requests.
254
        if env.endswith("osx") and not TC_IS_PUSH:
255
            continue
256

257
        TestTask(
258
            task_env=env,
259
            variant="coverage" if env == "linux" else None,
260
        )
261

262
        task_env = TaskEnvironment.by_name("{}.test".format(env))
263
        git = Git.by_name("{}.{}".format(env, GIT_VERSION))
264
        build = Build.by_name(env)
265
        bin = os.path.basename(build.artifacts[0])
266
        Task(
267
            task_env=task_env,
268
            description="download build {} {}".format(task_env.os, task_env.cpu),
269
            command=list(
270
                chain(
271
                    git.install(),
272
                    Task.checkout(),
273
                    build.install(),
274
                    [
275
                        f"python3 repo/CI/test_download.py repo/{bin}",
276
                    ],
277
                )
278
            ),
279
            mounts=[
280
                git.mount(),
281
                build.mount(),
282
            ],
283
        )
284

285
    # Because nothing is using the arm64 linux build, we need to manually
286
    # touch it.
287
    Build.by_name("arm64-linux")
288

289
    for upgrade in UPGRADE_FROM:
290
        TestTask(
291
            short_desc="upgrade tests",
292
            extra_desc="from-{}".format(upgrade),
293
            variant="coverage",
294
            clone=upgrade,
295
            env={
296
                "UPGRADE_FROM": upgrade,
297
            },
298
            hg="5.4.2",
299
        )
300

301
    for git in ("1.8.5", "2.7.4"):
302
        TestTask(git=git, env={"GIT_OLD_VERSION": "1"})
303

304
    for hg in SOME_MERCURIAL_VERSIONS:
305
        if hg != MERCURIAL_VERSION:
306
            do_hg_version(hg)
307

308
    TestTask(
309
        task_env="linux",
310
        variant="asan",
311
    )
312

313
    for env in ("linux", "mingw64", "osx", "arm64-osx"):
314
        # Can't spawn osx workers from pull requests.
315
        if env.endswith("osx") and not TC_IS_PUSH:
316
            continue
317

318
        TestTask(
319
            task_env=env,
320
            variant="coverage" if env == "linux" else None,
321
            short_desc="graft tests",
322
            env={
323
                "GRAFT": "1",
324
            },
325
        )
326

327
    for env, variant in (
328
        ("linux", "coverage"),
329
        ("linux", "asan"),
330
        ("osx", None),
331
        ("arm64-osx", None),
332
    ):
333
        # Can't spawn osx workers from pull requests.
334
        if env.endswith("osx") and not TC_IS_PUSH:
335
            continue
336

337
        pre_command = []
338
        if env != "linux":
339
            pre_command.append("pip install cram==0.7")
340

341
        TestTask(
342
            task_env=env,
343
            variant=variant,
344
            short_desc="cram",
345
            clone=False,
346
            command=pre_command
347
            + [
348
                "cram --verbose repo/tests",
349
            ],
350
            env={
351
                "GIT_CINNABAR_CHECK": "no-version-check",
352
            },
353
        )
354

355

356
def do_hg_version(hg):
357
    TestTask(hg=hg)
358
    # Don't run cram tests for version < 3.6, which would need
359
    # different tests because of server-side changes in behavior
360
    # wrt bookmarks.
361
    if not is_old_hg(hg):
362
        TestTask(
363
            short_desc="cram",
364
            clone=False,
365
            hg=hg,
366
            command=[
367
                "cram --verbose repo/tests",
368
            ],
369
            env={
370
                "GIT_CINNABAR_CHECK": "no-version-check",
371
            },
372
        )
373

374

375
@action(
376
    "more-hg-versions",
377
    title="More hg versions",
378
    description="Trigger tests against more mercurial versions",
379
)
380
def more_hg_versions():
381
    for hg in ALL_MERCURIAL_VERSIONS:
382
        if hg != MERCURIAL_VERSION and hg not in SOME_MERCURIAL_VERSIONS:
383
            do_hg_version(hg)
384

385

386
@action(
387
    "hg-trunk",
388
    title="Test w/ hg trunk",
389
    description="Trigger tests against current mercurial trunk",
390
)
391
def hg_trunk():
392
    import requests
393

394
    r = requests.get("https://www.mercurial-scm.org/repo/hg/?cmd=branchmap")
395
    trunk = None
396
    for l in r.text.splitlines():
397
        fields = l.split()
398
        if fields[0] == "default":
399
            trunk = fields[-1]
400
    if not trunk:
401
        raise Exception("Cannot find mercurial trunk changeset")
402
    do_hg_version(trunk)
403

404

405
def main():
406
    try:
407
        func = action.by_name[TC_ACTION or "decision"].func
408
    except AttributeError:
409
        raise Exception("Unsupported action: %s", TC_ACTION or "decision")
410

411
    func()
412

413
    merge_coverage = []
414

415
    if TestTask.coverage and TC_IS_PUSH and TC_BRANCH:
416
        coverage_mounts = [
417
            {f"file:cov-{task.id}.zip": task} for task in TestTask.coverage
418
        ]
419
        task = Build.by_name("linux.coverage")
420
        coverage_mounts.append(
421
            {
422
                "file:gcno-build.zip": {
423
                    "artifact": task.artifacts[1],
424
                    "taskId": task.id,
425
                }
426
            }
427
        )
428

429
        merge_coverage.extend(
430
            [
431
                "grcov -s repo -t lcov -o repo/coverage.lcov gcno-build.zip "
432
                + " ".join(f"cov-{task.id}.zip" for task in TestTask.coverage),
433
            ]
434
        )
435

436
    if merge_coverage:
437
        Task(
438
            task_env=TaskEnvironment.by_name("linux.codecov"),
439
            description="upload coverage",
440
            scopes=["secrets:get:project/git-cinnabar/codecov"],
441
            mounts=coverage_mounts,
442
            command=list(
443
                chain(
444
                    Task.checkout(),
445
                    [
446
                        "set +x",
447
                        (
448
                            "export CODECOV_TOKEN=$(curl -sL "
449
                            f"{PROXY_URL}/api/secrets/v1/secret/project/git-cinnabar"
450
                            "/codecov | python3"
451
                            ' -c "import json, sys; print(json.load(sys.stdin)'
452
                            '[\\"secret\\"][\\"token\\"])")'
453
                        ),
454
                        "set -x",
455
                    ],
456
                    merge_coverage,
457
                    [
458
                        "cd repo",
459
                        'codecov -Z --name "taskcluster" -C {} -B {}'.format(
460
                            TC_COMMIT, TC_BRANCH
461
                        ),
462
                    ],
463
                )
464
            ),
465
        )
466

467
    for t in Task.by_id.values():
468
        t.submit()
469

470
    if not TC_ACTION and "TC_GROUP_ID" in os.environ:
471
        actions = {
472
            "version": 1,
473
            "actions": [],
474
            "variables": {
475
                "e": dict(TC_DATA, decision_id=""),
476
                "tasks_for": "action",
477
            },
478
        }
479
        for name, a in action.by_name.items():
480
            if name != "decision":
481
                actions["actions"].append(
482
                    {
483
                        "kind": "task",
484
                        "name": a.name,
485
                        "title": a.title,
486
                        "description": a.description,
487
                        "context": [],
488
                        "task": a.task,
489
                    }
490
                )
491

492
        with open("actions.json", "w") as out:
493
            out.write(json.dumps(actions, indent=True))
494

495

496
if __name__ == "__main__":
497
    main()
498