kuma

1
version: 2.1 # Adds support for executors, parameterized jobs, etc
2
orbs:
3
  gh: circleci/github-cli@2.0
4
parameters:
5
  # These parameters are not meant to be changed they are more constants for the build change these in mk/dev.mk
6
  go_version:
7
    type: string
8
    default: "1.22.0"
9
  first_k8s_version:
10
    type: string
11
    default: "v1.23.17-k3s1"
12
  last_k8s_version:
13
    type: string
14
    default: "v1.29.1-k3s2"
15
  ubuntu_image:
16
    type: string
17
    default: "ubuntu-2204:2022.10.2"
18
  gh_action_build_artifact_name:
19
    type: string
20
    default: ""
21
  gh_action_run_id:
22
    type: string
23
    default: ""
24
  e2e_param_k8sVersion:
25
    type: string
26
    default: "v1.28.1-k3s1"
27
  e2e_param_arch:
28
    type: string
29
    default: "amd64"
30
  e2e_param_parallelism:
31
    type: integer
32
    default: 1
33
  e2e_param_target:
34
    type: string
35
    default: ""
36
  e2e_param_cniNetworkPlugin:
37
    type: string
38
    default: flannel
39
# See https://circleci.com/docs/2.0/configuration-reference/#commands-requires-version-21.
40
commands:
41
  install_build_tools:
42
    description: "Install an upstream Go release to $HOME/go"
43
    parameters:
44
      go_arch:
45
        type: string
46
        default: amd64
47
    steps:
48
      - run:
49
          # `unzip`    is necessary to install `protoc`
50
          # `gcc`      is necessary to run `go test -race`
51
          # `git`      is necessary because the CircleCI version is different somehow ¯\_(ツ)_/¯
52
          # `xz-utils` is necessary to decompress xz files
53
          name: "Install basic tools"
54
          command: |
55
            if [ -r /etc/os-release ]; then source /etc/os-release; fi
56
            case "$ID" in
57
            ubuntu)
58
              if ! command -v sudo 2>&1 >/dev/null; then
59
                apt update
60
                apt install -y sudo
61
              fi
62

63
              sudo apt update
64
              sudo env DEBIAN_FRONTEND=noninteractive apt install -y curl git make unzip gcc xz-utils
65
              ;;
66
            esac
67
      - run:
68
          name: "Install Go"
69
          # See https://golang.org/doc/install#tarball
70
          command: |
71
            curl -s --fail --location https://dl.google.com/go/go<<pipeline.parameters.go_version>>.linux-<<parameters.go_arch>>.tar.gz | tar -xz -C $HOME
72
            echo 'export PATH=$HOME/go/bin:$PATH' >> $BASH_ENV
73
            # if GOPATH is not set, `golang-ci` fails with an obscure message
74
            # "ERROR Running error: context loading failed: failed to load program with go/packages: could not determine GOARCH and Go compiler"
75
            echo 'export GOPATH=$HOME/go' >> $BASH_ENV
76
  halt_non_priority_job:
77
    description: "don't run following steps if in PR and doesn't have a specific label"
78
    parameters:
79
      label:
80
        type: string
81
        description: a label to for running this (if empty no label override is allowed)
82
        default: "ci/run-full-matrix"
83
    steps:
84
      - run:
85
          name: maybe-halt
86
          command: |
87
            if [[ "<< pipeline.git.branch >>" == "master" || "<< pipeline.git.branch >>" == "release-"*  ]]; then
88
              echo "on a main branch so don't halt job"
89
              exit 0
90
            fi
91
            if [[ "<< pipeline.git.tag >>" != "" ]]; then
92
              echo "on a tag to don't halt job"
93
              exit 0
94
            fi
95
            if [[ "<<parameters.label>>" != "" && $(${KUMA_DIR}/tools/ci/has_label.sh "<<parameters.label>>") == "true" ]]; then
96
              echo "<<parameters.label>> label present on PR so don't halt job"
97
              exit 0
98
            fi
99
            echo "halt running job"
100
            circleci-agent step halt
101
            exit 0
102
  setenv_depending_on_priority:
103
    description: "don't run following steps if in PR and doesn't have a specific label"
104
    parameters:
105
      label:
106
        type: string
107
        description: a label to for running this (if empty no label override is allowed)
108
      env:
109
        type: string
110
        description: a set of env var to set if it's a priority job
111
    steps:
112
      - run:
113
          name: maybe-add-env
114
          command: |
115
            function add_env() {
116
              echo 'export << parameters.env >>' >> "$BASH_ENV"
117
              exit 0
118
            }
119
            if [[ "<< pipeline.git.branch >>" == "master" || "<< pipeline.git.branch >>" == "release-"*  ]]; then
120
              echo "on a main branch so run everything"
121
              add_env
122
            fi
123
            if [[ "<< pipeline.git.tag >>" != "" ]]; then
124
              echo "on a tag so run everything"
125
              add_env
126
            fi
127
            if [[ "<<parameters.label>>" != "" && $(${KUMA_DIR}/tools/ci/has_label.sh "<<parameters.label>>") == "true" ]]; then
128
              echo "<<parameters.label>> label present on PR so run everything"
129
              add_env
130
            fi
131
            exit 0
132
  halt_job_if_labeled:
133
    description: "don't run following steps if PR has a specific label"
134
    parameters:
135
      label:
136
        type: string
137
        description: a label to for running this (if empty no label override is allowed)
138
        default: ""
139
    steps:
140
      - run:
141
          name: maybe-halt
142
          command: |
143
            if [[ "<<parameters.label>>" != "" && $(${KUMA_DIR}/tools/ci/has_label.sh "<<parameters.label>>") == "true" ]]; then
144
              echo "Not running as the PR has label: <<parameters.label>>"
145
              circleci-agent step halt
146
              exit 0
147
            fi
148
            echo "PR doesn't have label <<parameters.label>> keep running job"
149
            exit 0
150
executors:
151
  golang-amd64:
152
    resource_class: xlarge
153
    docker:
154
      - image: "cimg/go:<< pipeline.parameters.go_version >>"
155
    environment:
156
      KUMA_DIR: .
157
      GO_VERSION: << pipeline.parameters.go_version >>
158
  golang-arm64:
159
    resource_class: arm.xlarge
160
    docker:
161
      - image: "cimg/go:<< pipeline.parameters.go_version >>"
162
    environment:
163
      KUMA_DIR: .
164
      GO_VERSION: << pipeline.parameters.go_version >>
165
  vm-xlarge-amd64:
166
    resource_class: xlarge
167
    machine:
168
      image: << pipeline.parameters.ubuntu_image >>
169
    environment:
170
      KUMA_DIR: .
171
      GO_VERSION: << pipeline.parameters.go_version >>
172
  vm-amd64:
173
    resource_class: large
174
    machine:
175
      image: << pipeline.parameters.ubuntu_image >>
176
    environment:
177
      KUMA_DIR: .
178
      GO_VERSION: << pipeline.parameters.go_version >>
179
  vm-arm64:
180
    resource_class: arm.large
181
    machine:
182
      image: << pipeline.parameters.ubuntu_image >>
183
    environment:
184
      KUMA_DIR: .
185
      GO_VERSION: << pipeline.parameters.go_version >>
186
jobs:
187
  go_cache:
188
    executor: golang-<< parameters.arch >>
189
    parameters:
190
      arch:
191
        description: the executor to run on
192
        type: string
193
        default: golang
194
    steps:
195
      - checkout
196
      - restore_cache:
197
          key: vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
198
      - run:
199
          command: make dev/tools
200
      - run:
201
          name: "Download Go modules"
202
          command: |
203
            go mod download -x
204
      - save_cache:
205
          key: vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
206
          paths:
207
            - "/home/circleci/go/pkg/mod"
208
            - "/home/circleci/.kuma-dev"
209
  test:
210
    parameters:
211
      target:
212
        description: The test make target.
213
        type: string
214
        default: test
215
      arch:
216
        description: The golang arch.
217
        type: string
218
        default: amd64
219
    executor:
220
      name: vm-<< parameters.arch >>
221
    steps:
222
      - checkout
223
      - when:
224
          condition: {equal: [arm64, << parameters.arch >>]}
225
          steps:
226
            - halt_non_priority_job
227
      - halt_job_if_labeled:
228
          label: "ci/skip-test"
229
      - install_build_tools:
230
          go_arch: << parameters.arch >>
231
      - restore_cache:
232
          keys:
233
            - vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
234
      - run:
235
          name: "Run tests"
236
          command: |
237
            make << parameters.target >> TEST_REPORTS=1
238
      - store_artifacts:
239
          path: build/reports
240
          destination: /reports
241
      - store_test_results:
242
          path: build/reports
243
  e2e:
244
    parameters:
245
      k8sVersion:
246
        description: version of k3s to use or "kind" and "kindIpv6"
247
        type: string
248
        default: << pipeline.parameters.last_k8s_version >>
249
      parallelism:
250
        description: level of parallelization
251
        type: integer
252
        default: 1
253
      target:
254
        description: makefile target without test/e2e prefix
255
        type: string
256
        default: ""
257
      arch:
258
        description: The golang arch
259
        type: string
260
        default: amd64
261
      cniNetworkPlugin:
262
        description: The CNI networking plugin to use [flannel | calico]
263
        type: string
264
        default: flannel
265
    executor:
266
      name: vm-<< parameters.arch >>
267
    parallelism: << parameters.parallelism >>
268
    steps:
269
      - checkout
270
      - halt_job_if_labeled:
271
          label: "ci/skip-test"
272
      - halt_job_if_labeled:
273
          label: "ci/skip-e2e-test"
274
      - when:
275
          condition:
276
            or:
277
              - {equal: ["", << parameters.target >>]}
278
              - {equal: [calico, << parameters.cniNetworkPlugin >>]}
279
              - {equal: [kindIpv6, << parameters.k8sVersion >>]}
280
              - {equal: [arm64, << parameters.arch >>]}
281
              - {equal: [<< pipeline.parameters.first_k8s_version >>, << parameters.k8sVersion >>]}
282
          steps:
283
            - halt_non_priority_job
284
      - run:
285
          # This works around circleci limitation by skipping tests for combinations that don't make sense
286
          # See: https://discuss.circleci.com/t/matrix-exclude-entire-subset/43879
287
          name: skip_invalid_parameter_combinations
288
          command: |
289
            echo "Running with: \
290
              k8s:<< parameters.k8sVersion >> \
291
              target:<< parameters.target >> \
292
              parallelism:<< parameters.parallelism >> \
293
              arch:<< parameters.arch >> \
294
              cniNetworkPlugin:<< parameters.cniNetworkPlugin >> \
295
            "
296
            function skip() {
297
              echo "Non valid job combination halting job reason: $1"
298
              circleci-agent step halt
299
              exit 0
300
            }
301

302
            # Handle invalid test combinations
303
            if [[ "<< parameters.k8sVersion >>" == "kind" && "<< parameters.target >>" != "universal" ]]; then
304
              skip "kind should only be used when testing ipv6 or with e2e-universal"
305
            fi
306
            if [[ "<< parameters.k8sVersion >>" != kind* && "<< parameters.target >>" == "universal" ]]; then
307
              skip "universal only runs on kind"
308
            fi
309
            echo "Continuing tests"
310
      - install_build_tools:
311
          go_arch: << parameters.arch >>
312
      - restore_cache:
313
          keys:
314
            - vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
315
      # Mount files from the upstream jobs
316
      - attach_workspace:
317
          at: build
318
      - run:
319
          name: "Setup Helm"
320
          command: |
321
            make dev/set-kuma-helm-repo
322
      - when: # CircleCI's DNS on IPV6 prevents resolving inside Kind. When change to 8.8.8.8 and remove "search" section (. removes it), resolving works again
323
          condition:
324
            equal: [<< parameters.k8sVersion >>, "kindIpv6"]
325
          steps:
326
            - run:
327
                name: Enable IPV6 and change DNS
328
                command: |
329
                  cat \<<'EOF' | sudo tee /etc/docker/daemon.json
330
                  {
331
                    "ipv6": true,
332
                    "fixed-cidr-v6": "2001:db8:1::/64",
333
                    "dns": ["8.8.8.8"],
334
                    "dns-search": ["."]
335
                  }
336
                  EOF
337
                  sudo service docker restart
338
      - run:
339
          name: "Run E2E tests"
340
          command: |
341
            if [[ "<< parameters.k8sVersion >>" == "kindIpv6" ]]; then
342
              export IPV6=true
343
              export K8S_CLUSTER_TOOL=kind
344
              export KUMA_DEFAULT_RETRIES=60
345
              export KUMA_DEFAULT_TIMEOUT="6s"
346
            fi
347
            if [[ "<< parameters.k8sVersion >>" != "kind"* ]]; then
348
              export CI_K3S_VERSION=<< parameters.k8sVersion >>
349
              export K3D_NETWORK_CNI=<< parameters.cniNetworkPlugin >>
350
            fi
351
            if [[ "<< parameters.arch >>" == "arm64" ]]; then
352
              export MAKE_PARAMETERS="-j1"
353
            else
354
              export MAKE_PARAMETERS="-j2"
355
            fi
356

357
            if [[ "<< parameters.target >>" == "" ]]; then
358
              export GINKGO_E2E_LABEL_FILTERS="job-$CIRCLE_NODE_INDEX"
359
            fi
360
            env
361
            if [[ "<< parameters.target >>" != "" ]]; then
362
              target="test/e2e-<< parameters.target >>"
363
            else
364
              target="test/e2e"
365
            fi
366
            make ${MAKE_PARAMETERS} CI=true "${target}"
367
      - store_test_results:
368
          path: build/reports
369
      - store_artifacts:
370
          name: "Store logs"
371
          path: /tmp/e2e
372
  e2e_testing:
373
    parameters:
374
      k8sVersion:
375
        description: version of k3s to use or "kind" and "kindIpv6"
376
        type: string
377
        default: << pipeline.parameters.last_k8s_version >>
378
      parallelism:
379
        description: level of parallelization
380
        type: integer
381
        default: 1
382
      target:
383
        description: makefile target without test/e2e prefix
384
        type: string
385
        default: ""
386
      arch:
387
        description: The golang arch
388
        type: string
389
        default: amd64
390
      cniNetworkPlugin:
391
        description: The CNI networking plugin to use [flannel | calico]
392
        type: string
393
        default: flannel
394
    executor:
395
      name: vm-<< parameters.arch >>
396
    parallelism: << parameters.parallelism >>
397
    steps:
398
      - checkout
399
      - install_build_tools:
400
          go_arch: << parameters.arch >>
401
      - run:
402
          command: make dev/tools
403
      - run:
404
          name: "Download Go modules"
405
          command: |
406
            go mod download -x
407
      - run:
408
          name: "Build"
409
          command: |
410
            make build
411
      - run:
412
          name: "Distributions"
413
          command: |
414
            make -j build/distributions
415
      - run:
416
          name: "Images"
417
          command: |
418
            make -j images
419
            make -j docker/save
420
      - run:
421
          name: "Setup Helm"
422
          command: |
423
            make dev/set-kuma-helm-repo
424
      - when: # CircleCI's DNS on IPV6 prevents resolving inside Kind. When change to 8.8.8.8 and remove "search" section (. removes it), resolving works again
425
          condition:
426
            equal: [<< parameters.k8sVersion >>, "kindIpv6"]
427
          steps:
428
            - run:
429
                name: Enable IPV6 and change DNS
430
                command: |
431
                  cat \<<'EOF' | sudo tee /etc/docker/daemon.json
432
                  {
433
                    "ipv6": true,
434
                    "fixed-cidr-v6": "2001:db8:1::/64",
435
                    "dns": ["8.8.8.8"],
436
                    "dns-search": ["."]
437
                  }
438
                  EOF
439
                  sudo service docker restart
440
      - run:
441
          name: "Run E2E tests"
442
          command: |
443
            if [[ "<< parameters.k8sVersion >>" == "kindIpv6" ]]; then
444
              export IPV6=true
445
              export K8S_CLUSTER_TOOL=kind
446
              export KUMA_DEFAULT_RETRIES=60
447
              export KUMA_DEFAULT_TIMEOUT="6s"
448
            fi
449
            if [[ "<< parameters.k8sVersion >>" != "kind"* ]]; then
450
              export CI_K3S_VERSION=<< parameters.k8sVersion >>
451
              export K3D_NETWORK_CNI=<< parameters.cniNetworkPlugin >>
452
            fi
453
            if [[ "<< parameters.arch >>" == "arm64" ]]; then
454
              export MAKE_PARAMETERS="-j1"
455
            else
456
              export MAKE_PARAMETERS="-j2"
457
            fi
458

459
            if [[ "<< parameters.target >>" == "" ]]; then
460
              export GINKGO_E2E_LABEL_FILTERS="job-$CIRCLE_NODE_INDEX"
461
            fi
462
            env
463
            if [[ "<< parameters.target >>" != "" ]]; then
464
              target="test/e2e-<< parameters.target >>"
465
            else
466
              target="test/e2e"
467
            fi
468
            make ${MAKE_PARAMETERS} CI=true "${target}"
469
  build:
470
    executor: vm-xlarge-amd64
471
    steps:
472
      - run: # required to build arm64 images
473
          command: |
474
            sudo apt-get update
475
            sudo apt-get install -y qemu-user-static binfmt-support
476
      - install_build_tools
477
      - checkout
478
      - setenv_depending_on_priority:
479
          label: "ci/run-full-matrix"
480
          env: ENABLED_GOARCHES="arm64 amd64" ENABLED_GOOSES="linux darwin"
481
      - restore_cache:
482
          keys:
483
            - build_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
484
      - run:
485
          command: make dev/tools
486
      - run:
487
          command: make clean
488
      - run:
489
          command: make check
490
      - run:
491
          command: make build
492
      - run:
493
          command: make -j build/distributions
494
      - run:
495
          command: make -j images
496
      - run:
497
          command: make -j docker/save
498
      - save_cache:
499
          key: build_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
500
          paths:
501
            - "/home/circleci/go/pkg/mod"
502
            - "/home/circleci/.kuma-dev"
503
      - persist_to_workspace:
504
          root: build
505
          paths:
506
            - distributions/out
507
            - docker
508
            - artifacts-linux-amd64
509
            - artifacts-linux-arm64
510
            - ebpf-amd64
511
            - ebpf-arm64
512
  distributions:
513
    executor: vm-amd64
514
    steps:
515
      - install_build_tools
516
      - checkout
517
      - setenv_depending_on_priority:
518
          label: ci/force-publish
519
          env: ALLOW_PUSH=true
520
      - setenv_depending_on_priority:
521
          label: ci/run-full-matrix
522
          env: ENABLED_GOARCHES="arm64 amd64" ENABLED_GOOSES="linux darwin"
523
      # Mount files from the upstream jobs
524
      - restore_cache:
525
          keys:
526
            - vm-amd64_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
527
      - attach_workspace:
528
          at: build
529
      - run:
530
          name: inspect created tars
531
          command: for i in build/distributions/out/*.tar.gz; do echo $i; tar -tvf $i; done
532
      - run:
533
          name: Publish distributions to Pulp
534
          command: make publish/pulp
535
      - run:
536
          name: load images
537
          command: make docker/load
538
      - run:
539
          name: publish images
540
          command: |
541
            make docker/login
542
            # ensure we always logout
543
            function on_exit() {
544
              make docker/logout
545
            }
546
            trap on_exit EXIT
547
            make docker/push
548
            make docker/manifest
549
  container-structure:
550
    executor: vm-amd64
551
    steps:
552
      - checkout
553
      - halt_job_if_labeled:
554
          label: "ci/skip-container-structure-test"
555
      - halt_job_if_labeled:
556
          label: "ci/skip-test"
557
      - setenv_depending_on_priority:
558
          label: "ci/run-full-matrix"
559
          env: ENABLED_GOARCHES="arm64 amd64"
560
      - run:
561
          command: sudo apt-get update; sudo apt-get install -y qemu-user-static binfmt-support
562
      - restore_cache:
563
          key: vm-amd64_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
564
      - attach_workspace:
565
          at: build
566
      - run:
567
          command: make test/container-structure
568
workflows:
569
  version: 2
570
  kuma-commit:
571
    when: false # Stop running for the moment (likely we'll remove this in the future.
572
    # equal: ["", << pipeline.parameters.gh_action_build_artifact_name >>]
573
    jobs:
574
      - go_cache:
575
          name: go_cache-<< matrix.arch >>
576
          matrix:
577
            alias: go_cache
578
            parameters:
579
              arch: [amd64, arm64]
580
      - build:
581
          name: build
582
      - test:
583
          name: test-<< matrix.arch >>
584
          matrix:
585
            alias: test
586
            parameters:
587
              arch: [amd64, arm64]
588
          requires: [build, go_cache-<< matrix.arch >>]
589
      - e2e:
590
          name: legacy-k8s:<< matrix.arch >>-<< matrix.k8sVersion >>
591
          matrix:
592
            alias: legacy
593
            parameters:
594
              k8sVersion: [<< pipeline.parameters.first_k8s_version >>, << pipeline.parameters.last_k8s_version >>, kind, kindIpv6]
595
              arch: [amd64, arm64]
596
          parallelism: 3
597
          target: ""
598
          requires: [build, go_cache-<< matrix.arch >>]
599
      - e2e:
600
          name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>
601
          matrix:
602
            alias: e2e
603
            parameters:
604
              k8sVersion: [<< pipeline.parameters.first_k8s_version >>, << pipeline.parameters.last_k8s_version >>, kind, kindIpv6]
605
              target: [kubernetes, universal, multizone]
606
              arch: [amd64, arm64]
607
          requires: [build, go_cache-<< matrix.arch >>]
608
      - e2e:
609
          name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>-calico
610
          matrix:
611
            alias: calico
612
            parameters:
613
              k8sVersion: [<< pipeline.parameters.last_k8s_version >>]
614
              target: [multizone]
615
              arch: [amd64]
616
              cniNetworkPlugin: [calico]
617
          requires: [build, go_cache-amd64]
618
      - container-structure:
619
          name: container-structure
620
          requires: [build]
621
      - distributions:
622
          requires: [test, container-structure]
623
  manual-e2e:
624
    when:
625
      not:
626
        equal: ["", << pipeline.parameters.gh_action_build_artifact_name >>]
627
    jobs:
628
      - e2e_testing:
629
          filters:
630
            tags:
631
              only: /.*/
632
          k8sVersion: << pipeline.parameters.e2e_param_k8sVersion >>
633
          target: << pipeline.parameters.e2e_param_target >>
634
          arch: << pipeline.parameters.e2e_param_arch >>
635
          cniNetworkPlugin: << pipeline.parameters.e2e_param_cniNetworkPlugin >>
636
          parallelism: << pipeline.parameters.e2e_param_parallelism >>
637