firecracker

1
#!/usr/bin/env python3
2
# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
# SPDX-License-Identifier: Apache-2.0
4

5
"""Generate Buildkite pipelines dynamically"""
6

7
from common import (
8
    COMMON_PARSER,
9
    devtool_test,
10
    get_changed_files,
11
    group,
12
    overlay_dict,
13
    pipeline_to_json,
14
    run_all_tests,
15
)
16

17
# Buildkite default job priority is 0. Setting this to 1 prioritizes PRs over
18
# scheduled jobs and other batch jobs.
19
DEFAULT_PRIORITY = 1
20

21

22
args = COMMON_PARSER.parse_args()
23

24
step_style = {
25
    "command": "./tools/devtool -y test -- ../tests/integration_tests/style/",
26
    "label": "🪶 Style",
27
    "priority": DEFAULT_PRIORITY,
28
}
29

30
defaults = {
31
    "instances": args.instances,
32
    "platforms": args.platforms,
33
    # buildkite step parameters
34
    "priority": DEFAULT_PRIORITY,
35
    "timeout_in_minutes": 45,
36
    "artifacts": ["./test_results/**/*"],
37
}
38
defaults = overlay_dict(defaults, args.step_param)
39

40
defaults_once_per_architecture = defaults.copy()
41
defaults_once_per_architecture["instances"] = ["m6i.metal", "m7g.metal"]
42
defaults_once_per_architecture["platforms"] = [("al2", "linux_5.10")]
43

44

45
devctr_grp = group(
46
    "🐋 Dev Container Sanity Build",
47
    "./tools/devtool -y build_devctr",
48
    **defaults_once_per_architecture,
49
)
50

51
release_grp = group(
52
    "📦 Release Sanity Build",
53
    "./tools/devtool -y make_release",
54
    **defaults_once_per_architecture,
55
)
56

57
build_grp = group(
58
    "📦 Build",
59
    "./tools/devtool -y test -- ../tests/integration_tests/build/",
60
    **defaults,
61
)
62

63
functional_grp = group(
64
    "⚙ Functional and security 🔒",
65
    devtool_test(
66
        pytest_opts="-n 8 --dist worksteal integration_tests/{{functional,security}}",
67
        binary_dir=args.binary_dir,
68
    ),
69
    **defaults,
70
)
71

72
defaults_for_performance = overlay_dict(
73
    defaults,
74
    {
75
        # We specify higher priority so the ag=1 jobs get picked up before the ag=n
76
        # jobs in ag=1 agents
77
        "priority": DEFAULT_PRIORITY + 1,
78
        "agents": {"ag": 1},
79
    },
80
)
81

82
performance_grp = group(
83
    "⏱ Performance",
84
    devtool_test(
85
        devtool_opts="--performance -c 1-10 -m 0",
86
        pytest_opts="../tests/integration_tests/performance/",
87
        binary_dir=args.binary_dir,
88
    ),
89
    **defaults_for_performance,
90
)
91

92
defaults_for_kani = overlay_dict(
93
    defaults_for_performance,
94
    {
95
        # Kani runs fastest on m6i.metal
96
        "instances": ["m6a.metal"],
97
        "platforms": [("al2", "linux_5.10")],
98
        "timeout_in_minutes": 300,
99
    },
100
)
101

102
kani_grp = group(
103
    "🔍 Kani",
104
    "./tools/devtool -y test -- ../tests/integration_tests/test_kani.py -n auto",
105
    **defaults_for_kani,
106
)
107
for step in kani_grp["steps"]:
108
    step["label"] = "🔍 Kani"
109

110
steps = [step_style]
111
changed_files = get_changed_files("main")
112

113
# run sanity build of devtool if Dockerfile is changed
114
if any(x.name == "Dockerfile" for x in changed_files):
115
    steps.append(devctr_grp)
116

117
if any(
118
    x.parent.name == "tools" and ("release" in x.name or x.name == "devtool")
119
    for x in changed_files
120
):
121
    steps.append(release_grp)
122

123
if not changed_files or any(
124
    x.suffix in [".rs", ".toml", ".lock"] for x in changed_files
125
):
126
    steps.append(kani_grp)
127

128
if run_all_tests(changed_files):
129
    steps += [
130
        build_grp,
131
        functional_grp,
132
        performance_grp,
133
    ]
134

135
pipeline = {"steps": steps}
136
print(pipeline_to_json(pipeline))
137