cilium

1
name: Network performance GKE
2

3
# Any change in triggers needs to be reflected in the concurrency group.
4
on:
5
  schedule:
6
    - cron: '39 0 * * 1-5'
7
# For testing uncomment following lines:
8
#  push:
9
#    branches:
10
#      - your_branch_name
11

12
# By specifying the access of one of the scopes, all of those that are not
13
# specified are set to 'none'.
14
permissions:
15
  # To be able to access the repository with actions/checkout
16
  contents: read
17
  # To be able to request the JWT from GitHub's OIDC provider
18
  id-token: write
19

20
concurrency:
21
  # Structure:
22
  # - Workflow name
23
  # - Event type
24
  # - A unique identifier depending on event type:
25
  #   - schedule: SHA
26
  #   - workflow_dispatch: PR number
27
  #
28
  # This structure ensures a unique concurrency group name is generated for each
29
  # type of testing, such that re-runs will cancel the previous run.
30
  group: |
31
    ${{ github.workflow }}
32
    ${{ github.event_name }}
33
    ${{
34
      (github.event_name == 'schedule' && github.sha) ||
35
      (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
36
    }}
37
  cancel-in-progress: true
38

39
env:
40
  clusterName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}
41
  cilium_cli_ci_version:
42
  CILIUM_CLI_MODE: helm
43
  test_name: gke-perf
44
  check_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
45
  USE_GKE_GCLOUD_AUTH_PLUGIN: True
46
  gcp_zone: us-east5-a
47
  k8s_version: 1.28
48

49
jobs:
50
  installation-and-perf:
51
    name: Installation and Perf Test
52
    runs-on: ubuntu-latest
53
    timeout-minutes: 60
54
    env:
55
      job_name: "Installation and Perf Test"
56
    strategy:
57
      fail-fast: false
58
      matrix:
59
        include:
60
          - index: 1
61
            name: "native"
62
            mode: "gke"
63
            encryption: "none"
64
            hubble: "false"
65

66
          - index: 2
67
            name: "tunnel"
68
            mode: "tunnel"
69
            encryption: "none"
70
            hubble: "false"
71

72
          - index: 3
73
            name: "native-ipsec"
74
            mode: "gke"
75
            encryption: "ipsec"
76
            hubble: "false"
77

78
          - index: 4
79
            name: "tunnel-ipsec"
80
            mode: "tunnel"
81
            encryption: "ipsec"
82
            hubble: "false"
83

84
          - index: 5
85
            name: "native-hubble"
86
            mode: "gke"
87
            encryption: "none"
88
            hubble: "true"
89

90
          - index: 6
91
            name: "tunnel-hubble"
92
            mode: "tunnel"
93
            encryption: "none"
94
            hubble: "true"
95

96
          - index: 7
97
            name: "native-ipsec-hubble"
98
            mode: "gke"
99
            encryption: "ipsec"
100
            hubble: "true"
101

102
          - index: 8
103
            name: "tunnel-ipsec-hubble"
104
            mode: "tunnel"
105
            encryption: "ipsec"
106
            hubble: "true"
107

108
    steps:
109
      - name: Checkout context ref (trusted)
110
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
111
        with:
112
          ref: ${{ inputs.context-ref || github.sha }}
113
          persist-credentials: false
114

115
      - name: Set Environment Variables
116
        uses: ./.github/actions/set-env-variables
117

118
      - name: Set up job variables
119
        id: vars
120
        run: |
121
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ ${{ github.event.pull_request }} ] ; then
122
            SHA="${{ inputs.image-tag }}"
123
            OWNER="${{ inputs.PR-number }}"
124
          else
125
            SHA="${{ github.sha }}"
126
            OWNER="${{ github.ref_name }}"
127
            OWNER="${OWNER/./-}"
128
          fi
129

130
          CILIUM_INSTALL_DEFAULTS="--chart-directory=install/kubernetes/cilium \
131
            --cluster-name=${{ env.clusterName }}-${{ matrix.index }} \
132
            --helm-set=agentNotReadyTaintKey=ignore-taint.cluster-autoscaler.kubernetes.io/cilium-agent-not-ready \
133
            --helm-set=debug.enabled=false \
134
            --helm-set=bpf.monitorAggregation=maximum \
135
            --helm-set=hubble.enabled=${{ matrix.hubble == 'true' }} \
136
            --wait=false"
137

138
          # only add SHA to the image tags if it was set
139
          if [ -n "${SHA}" ]; then
140
            echo sha=${SHA} >> $GITHUB_OUTPUT
141
            CILIUM_INSTALL_DEFAULTS+=" --helm-set=image.repository=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/cilium-ci \
142
            --helm-set=image.useDigest=false \
143
            --helm-set=image.tag=${SHA} \
144
            --helm-set=operator.image.repository=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/operator \
145
            --helm-set=operator.image.suffix=-ci \
146
            --helm-set=operator.image.tag=${SHA} \
147
            --helm-set=operator.image.useDigest=false \
148
            --helm-set=clustermesh.apiserver.image.repository=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/clustermesh-apiserver-ci \
149
            --helm-set=clustermesh.apiserver.image.tag=${SHA} \
150
            --helm-set=clustermesh.apiserver.image.useDigest=false \
151
            --helm-set=hubble.relay.image.repository=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/hubble-relay-ci \
152
            --helm-set=hubble.relay.image.tag=${SHA} \
153
            --helm-set=hubble.relay.image.useDigest=false"
154
          fi
155

156
          CILIUM_INSTALL_DEFAULTS+=" --datapath-mode=${{ matrix.mode }}"
157

158
          if [ "${{ matrix.encryption }}" = "ipsec" ] ; then
159
            CILIUM_INSTALL_DEFAULTS+=" --helm-set=encryption.enabled=true --helm-set=encryption.type=ipsec"
160
          fi
161

162
          echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
163
          echo owner=${OWNER} >> $GITHUB_OUTPUT
164

165
      - name: Install Cilium CLI
166
        uses: cilium/cilium-cli@7306e3cdc6caee738157f08e3e1ba26179f104e5 # v0.15.23
167
        with:
168
          repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
169
          release-version: ${{ env.CILIUM_CLI_VERSION }}
170
          ci-version: ${{ env.cilium_cli_ci_version }}
171

172
      - name: Set up gcloud credentials
173
        id: 'auth'
174
        uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
175
        with:
176
          workload_identity_provider: ${{ secrets.GCP_PERF_WORKLOAD_IDENTITY_PROVIDER }}
177
          service_account: ${{ secrets.GCP_PERF_SA }}
178
          create_credentials_file: true
179
          export_environment_variables: true
180

181
      - name: Set up gcloud CLI
182
        uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
183
        with:
184
          project_id: ${{ secrets.GCP_PERF_PROJECT_ID }}
185
          version: "405.0.0"
186

187
      - name: Install gke-gcloud-auth-plugin
188
        run: |
189
          gcloud components install gke-gcloud-auth-plugin
190

191
      - name: Display gcloud CLI info
192
        run: |
193
          gcloud info
194

195
      - name: Create GKE cluster
196
        run: |
197
          gcloud container clusters create ${{ env.clusterName }}-${{ matrix.index }} \
198
            --labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \
199
            --zone ${{ env.gcp_zone }} \
200
            --cluster-version ${{ env.k8s_version }} \
201
            --enable-ip-alias \
202
            --create-subnetwork="range=/26" \
203
            --cluster-ipv4-cidr="/21" \
204
            --services-ipv4-cidr="/24" \
205
            --image-type COS_CONTAINERD \
206
            --num-nodes 2 \
207
            --machine-type n2-standard-2 \
208
            --disk-type pd-standard \
209
            --disk-size 20GB \
210
            --node-taints ignore-taint.cluster-autoscaler.kubernetes.io/cilium-agent-not-ready=true:NoExecute
211

212
      - name: Get cluster credentials
213
        run: |
214
          gcloud container clusters get-credentials ${{ env.clusterName }}-${{ matrix.index }} --zone ${{ env.gcp_zone }}
215

216
      - name: Wait for images to be available
217
        timeout-minutes: 30
218
        shell: bash
219
        run: |
220
          for image in cilium-ci operator-generic-ci hubble-relay-ci ; do
221
            until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
222
          done
223

224
      - name: Create custom IPsec secret
225
        if: ${{ matrix.encryption == 'ipsec' }}
226
        run: |
227
          kubectl create -n kube-system secret generic cilium-ipsec-keys --from-literal=keys="15 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"
228

229
      - name: Install Cilium
230
        id: install-cilium
231
        run: |
232
          cilium install --dry-run-helm-values ${{ steps.vars.outputs.cilium_install_defaults }}
233
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }}
234

235
      - name: Wait for Cilium to be ready
236
        run: |
237
          cilium status --wait --wait-duration=10m
238
          kubectl get pods -n kube-system
239
          kubectl -n kube-system exec daemonset/cilium -- cilium-dbg status
240

241
      - name: Run perf test (${{ matrix.name }})
242
        id: run-perf
243
        run: |
244
          mkdir output
245
          cilium connectivity perf --duration=30s --host-net=true --pod-net=true --report-dir=./output
246

247
      - name: Get sysdump
248
        if: ${{ always() && steps.run-perf.outcome != 'skipped' && steps.run-perf.outcome != 'cancelled' }}
249
        run: |
250
          cilium status
251
          cilium sysdump --output-filename cilium-sysdump-final
252

253
      - name: Clean up GKE
254
        if: ${{ always() }}
255
        run: |
256
          while [ "$(gcloud container operations list --zone ${{ env.gcp_zone }} --filter="status=RUNNING AND targetLink~${{ env.clusterName }}-${{ matrix.index }}" --format="value(name)")" ];do
257
            echo "cluster has an ongoing operation, waiting for all operations to finish"; sleep 15
258
          done
259
          gcloud container clusters delete ${{ env.clusterName }}-${{ matrix.index }} --zone ${{ env.gcp_zone }} --quiet --async
260
        shell: bash {0} # Disable default fail-fast behavior so that all commands run independently
261

262
      - name: Export results and sysdump to GS bucket
263
        if: ${{ always() && steps.run-perf.outcome != 'skipped' && steps.run-perf.outcome != 'cancelled' }}
264
        uses: cilium/scale-tests-action/export-results@238d773bd07754bfd693a6b22c94eddf3a12778d # main
265
        with:
266
          test_name: ${{ env.test_name }}-${{ matrix.name }}
267
          results_bucket: ${{ env.GCP_PERF_RESULTS_BUCKET }}
268
          artifacts: ./output/*
269
          other_files: cilium-sysdump-final.zip
270