/
krepost
/
deckhouse
Обзор
Документация
Войти
/
krepost
/
deckhouse
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
main
.github/workflows/e2e-openstack.yml
6 179 строк
288 KB
Maksim Khimchenko
Update werf version to v2.57.1 (#18562)
19 мар 2026, 10:54
Не верифицирован
19 мар 2026, 10:54
99eb9d5
Код
Авторство
О чём код?
# # THIS FILE IS GENERATED, PLEASE DO NOT EDIT. # # Copyright 2022 Flant JSC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # <template: e2e_workflow_template> name: 'e2e: OpenStack' on: workflow_dispatch: inputs: issue_id: description: 'ID of issue where label was set' required: false issue_number: description: 'Number of issue where label was set' required: false comment_id: description: 'ID of comment in issue where to put workflow run status' required: false ci_commit_ref_name: description: 'Git ref name for image tags' required: false pull_request_ref: description: 'Git ref for checkout PR sources' required: false pull_request_sha: description: 'Git SHA for restoring artifacts from cache' required: false pull_request_head_label: description: 'Head label of pull request. e.g. my_repo:my_feature_branch' required: false test_config: description: 'JSON string of parameters of current test' required: false default: '{"cri":"ContainerdV2","ver":"1.32","edition":"FE"}' initial_ref_slug: description: 'An image tag to install first and then switch to workflow context ref' required: false autodelete: description: 'Should the cluster be deleted regardless of the test result' required: true default: false type: boolean env: # <template: werf_envs> WERF_VERSION: "v2.57.1" WERF_ENV: "FE" WERF_TELEMETRY_EXTRA_ATTRIBUTE_TEAM: "flantTeam=deckhouse/deckhouse" TEST_TIMEOUT: "15m" # Use fixed string 'sys/deckhouse-oss' for repo name. ${CI_PROJECT_PATH} is not available here in GitHub. REGISTRY_PATH: "sys/deckhouse-oss" # Registry for additional repositories used for testing Github Actions workflows. GHA_TEST_REGISTRY_PATH: "ghcr.io/${{ github.repository }}" # Need for ssh: default. DOCKER_BUILDKIT: "1" WERF_FINAL_IMAGES_ONLY: true WERF_LOG_TERMINAL_WIDTH: "200" WERF_LOG_TIME: true WERF_GIT_WORK_TREE_POOL_LIMIT: "10" GOPROXY: "${{vars.GOPROXY}}" # </template: werf_envs> # Note: no concurrency section for e2e workflows. # Usually you run e2e and wait until it ends. permissions: contents: read id-token: write pull-requests: read jobs: started_at: name: Save start timestamp outputs: started_at: ${{ steps.started_at.outputs.started_at }} runs-on: "regular" steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: git_info_job> git_info: name: Get git info runs-on: "regular" outputs: ci_commit_tag: ${{ steps.git_info.outputs.ci_commit_tag }} ci_commit_branch: ${{ steps.git_info.outputs.ci_commit_branch }} ci_commit_ref_name: ${{ steps.git_info.outputs.ci_commit_ref_name }} ci_commit_ref_slug: ${{ steps.git_info.outputs.ci_commit_ref_slug }} ref_full: ${{ steps.git_info.outputs.ref_full }} github_sha: ${{ steps.git_info.outputs.github_sha }} pr_number: ${{ steps.git_info.outputs.pr_number }} # Skip the CI for automation PRs, e.g. changelog, don't skip if Pull Request title contains "[run ci]". if: ${{ contains(github.event.pull_request.title, '[run ci]') || github.event.pull_request.user.login != 'deckhouse-BOaTswain' }} steps: - id: git_info name: Get tag name and SHA uses: actions/github-script@v6.4.1 with: script: | const { GITHUB_REF_TYPE, GITHUB_REF_NAME, GITHUB_REF } = process.env let refSlug = '' let refName = '' let refFull = '' let githubBranch = '' let githubTag = '' let githubSHA = '' let prNumber = '' if (context.eventName === "workflow_dispatch" && context.payload.inputs && context.payload.inputs.pull_request_ref) { // Trigger: workflow_dispatch with pull_request_ref. // Extract pull request number from 'refs/pull/<NUM>/merge' prNumber = context.payload.inputs.pull_request_ref.replace('refs/pull/', '').replace('/merge', '').replace('/head', '') refSlug = `pr${prNumber}` refName = context.payload.inputs.ci_commit_ref_name refFull = context.payload.inputs.pull_request_ref githubBranch = refName githubSHA = context.payload.inputs.pull_request_sha core.info(`workflow_dispatch event: set git info from inputs. inputs: ${JSON.stringify(context.payload.inputs)}`) } else if (context.eventName === "pull_request" || context.eventName === "pull_request_target" ) { // For PRs from forks, tag images with `prXXX` to avoid clashes between branches. const targetRepo = context.payload.repository.full_name; const prRepo = context.payload.pull_request.head.repo.full_name const prRef = context.payload.pull_request.head.ref refSlug = `pr${context.issue.number}`; refName = (prRepo === targetRepo) ? prRef : refSlug; refFull = `refs/pull/${context.issue.number}/head` githubBranch = refName githubSHA = context.payload.pull_request.head.sha core.info(`pull request event: set git info from pull_request.head. pr:${prRepo}:${prRef} target:${targetRepo}:${context.ref}`) prNumber = context.issue.number } else { // Other triggers: workflow_dispatch without pull_request_ref, schedule, push... // refName is 'main' or tag name, so slugification is not necessary. refSlug = GITHUB_REF_NAME refName = GITHUB_REF_NAME refFull = GITHUB_REF githubTag = GITHUB_REF_TYPE == "tag" ? refName : "" githubBranch = GITHUB_REF_TYPE == "branch" ? refName : "" githubSHA = context.sha core.info(`${context.eventName} event: set git info from context: ${JSON.stringify({GITHUB_REF_NAME, GITHUB_REF_TYPE, sha: context.sha })}`) } core.setCommandEcho(true) core.setOutput('ci_commit_ref_slug', refSlug) core.setOutput('ci_commit_ref_name', refName) core.setOutput(`ci_commit_tag`, githubTag) core.setOutput(`ci_commit_branch`, githubBranch) core.setOutput(`ref_full`, refFull) core.setOutput('github_sha', githubSHA) core.setOutput('pr_number', prNumber) core.setCommandEcho(false) # </template: git_info_job> # </template: block-until-image-is-not-ready> block-until-image-is-not-ready: name: Block until the docker image is not ready runs-on: "regular" permissions: contents: read actions: read pull-requests: read steps: - uses: actions/checkout@v3.5.2 with: ref: "main" - name: Block until the docker image is not ready id: block-image-dont-ready uses: actions/github-script@v6.4.1 with: script: | const githubAction = require('./.github/scripts/js/helpers/github-actions')({github, context, core}); const { isReleaseBranch } = require('./.github/scripts/js/helpers/utils'); const isTag = (branch) => branch.includes("/tags/") let branchName; let prNum; if (context.eventName === 'pull_request') { branchName = context.payload.pull_request.head.ref; prNum = context.payload.pull_request.number; } else if (context.eventName === 'workflow_dispatch') { branchName = context.payload.inputs.ci_commit_ref_name; prNum = context.payload.inputs.issue_number; } if (!branchName) { branchName = githubAction.GetBranchNameFromContext(context); } if (!branchName) { core.setFailed("Branch not found."); return false; } core.info(`${branchName} is main? ${branchName === 'main'}`) core.info(`${branchName} is release? ${isReleaseBranch(branchName)}`) core.info(`${branchName} is tag? ${isTag(branchName)}`) if (branchName === 'main' || isReleaseBranch(branchName) || isTag(branchName)) { core.info(`Skip a job because it's a tag, release or main branch: ${branchName}`); return true; } let infoText = `Check build workflow is completed for branch: ${branchName}`; if (prNum) { infoText += ` | PR: ${context.payload.repository.html_url}/pull/${prNum}`; } core.info(infoText); const { waitForJobInWorkflowIsCompletedWithSuccess } = require('./.github/scripts/js/validators/validate-job-in-workflow-is-ready')({ github, context, core }); try { let workflowName = 'Build and test for dev branches'; let jobName = 'Build FE'; await waitForJobInWorkflowIsCompletedWithSuccess(branchName, workflowName, jobName); } catch(error) { core.setFailed(error); } # </template: block-until-image-is-not-ready> # <template: check_e2e_labels_job> check_e2e_labels: name: Check e2e labels runs-on: "regular" permissions: contents: read issues: write pull-requests: write outputs: run_containerd_1_31: ${{ steps.check.outputs.run_containerd_1_31 }} run_containerd_1_32: ${{ steps.check.outputs.run_containerd_1_32 }} run_containerd_1_33: ${{ steps.check.outputs.run_containerd_1_33 }} run_containerd_1_34: ${{ steps.check.outputs.run_containerd_1_34 }} run_containerd_1_35: ${{ steps.check.outputs.run_containerd_1_35 }} run_containerd_automatic: ${{ steps.check.outputs.run_containerd_automatic }} run_containerdv2_1_31: ${{ steps.check.outputs.run_containerdv2_1_31 }} run_containerdv2_1_32: ${{ steps.check.outputs.run_containerdv2_1_32 }} run_containerdv2_1_33: ${{ steps.check.outputs.run_containerdv2_1_33 }} run_containerdv2_1_34: ${{ steps.check.outputs.run_containerdv2_1_34 }} run_containerdv2_1_35: ${{ steps.check.outputs.run_containerdv2_1_35 }} run_containerdv2_automatic: ${{ steps.check.outputs.run_containerdv2_automatic }} edition: ${{ steps.check.outputs.edition }} multimaster: ${{ steps.check.outputs.multimaster }} cis: ${{ steps.check.outputs.cis }} autoscaler: ${{ steps.check.outputs.autoscaler }} steps: # <template: checkout_step> - name: Checkout sources uses: actions/checkout@v3.5.2 # </template: checkout_step> - name: Check e2e labels id: check uses: actions/github-script@v6.4.1 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const provider = 'openstack'; const kubernetesDefaultVersion = '1.33'; const ci = require('./.github/scripts/js/ci'); return await ci.checkE2ELabels({github, context, core, provider, kubernetesDefaultVersion}); # </template: check_e2e_labels_job> # <template: e2e_run_job_template> run_containerd_1_31: name: "e2e: OpenStack, Containerd, Kubernetes 1.31" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerd_1_31 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerd;1.31" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.31" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, Containerd, Kubernetes 1.31'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/Containerd/1.31" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.31" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.31" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerd_1_31 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, Containerd, Kubernetes 1.31'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerd_1_32: name: "e2e: OpenStack, Containerd, Kubernetes 1.32" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerd_1_32 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerd;1.32" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.32" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, Containerd, Kubernetes 1.32'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/Containerd/1.32" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.32" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.32" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerd_1_32 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, Containerd, Kubernetes 1.32'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerd_1_33: name: "e2e: OpenStack, Containerd, Kubernetes 1.33" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerd_1_33 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerd;1.33" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.33" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, Containerd, Kubernetes 1.33'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/Containerd/1.33" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.33" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.33" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerd_1_33 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, Containerd, Kubernetes 1.33'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerd_1_34: name: "e2e: OpenStack, Containerd, Kubernetes 1.34" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerd_1_34 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerd;1.34" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.34" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, Containerd, Kubernetes 1.34'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/Containerd/1.34" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.34" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.34" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerd_1_34 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, Containerd, Kubernetes 1.34'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerd_1_35: name: "e2e: OpenStack, Containerd, Kubernetes 1.35" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerd_1_35 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerd;1.35" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.35" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, Containerd, Kubernetes 1.35'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/Containerd/1.35" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.35" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "1.35" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerd_1_35 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, Containerd, Kubernetes 1.35'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerd_Automatic: name: "e2e: OpenStack, Containerd, Kubernetes Automatic" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerd_Automatic == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerd;Automatic" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "Automatic" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, Containerd, Kubernetes Automatic'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/Containerd/Automatic" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "Automatic" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: Containerd LAYOUT: Standard KUBERNETES_VERSION: "Automatic" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerd_Automatic path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, Containerd, Kubernetes Automatic'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerdv2_1_31: name: "e2e: OpenStack, ContainerdV2, Kubernetes 1.31" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerdv2_1_31 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerdv2;1.31" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.31" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.31'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/ContainerdV2/1.31" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.31" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.31" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerdv2_1_31 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.31'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerdv2_1_32: name: "e2e: OpenStack, ContainerdV2, Kubernetes 1.32" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerdv2_1_32 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerdv2;1.32" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.32" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.32'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/ContainerdV2/1.32" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.32" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.32" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerdv2_1_32 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.32'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerdv2_1_33: name: "e2e: OpenStack, ContainerdV2, Kubernetes 1.33" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerdv2_1_33 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerdv2;1.33" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.33" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.33'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/ContainerdV2/1.33" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.33" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.33" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerdv2_1_33 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.33'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerdv2_1_34: name: "e2e: OpenStack, ContainerdV2, Kubernetes 1.34" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerdv2_1_34 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerdv2;1.34" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.34" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.34'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/ContainerdV2/1.34" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.34" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.34" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerdv2_1_34 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.34'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerdv2_1_35: name: "e2e: OpenStack, ContainerdV2, Kubernetes 1.35" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerdv2_1_35 == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerdv2;1.35" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.35" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.35'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/ContainerdV2/1.35" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.35" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "1.35" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerdv2_1_35 path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, ContainerdV2, Kubernetes 1.35'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> # <template: e2e_run_job_template> run_containerdv2_Automatic: name: "e2e: OpenStack, ContainerdV2, Kubernetes Automatic" needs: - check_e2e_labels - git_info - block-until-image-is-not-ready if: needs.check_e2e_labels.outputs.run_containerdv2_Automatic == 'true' outputs: ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} run_id: ${{ github.run_id }} # need for find state in artifact cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} ran_for: "openstack;Standard;containerdv2;Automatic" failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} issue_number: ${{ inputs.issue_number }} install_image_path: ${{ steps.setup.outputs.install-image-path }} env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "Automatic" EVENT_LABEL: ${{ github.event.label.name }} WERF_ENV: "${{ needs.check_e2e_labels.outputs.edition || fromJson(inputs.test_config).edition }}" runs-on: [self-hosted, regular] steps: # <template: started_at_output> - name: Job started timestamp id: started_at run: | unixTimestamp=$(date +%s) echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT # </template: started_at_output> # <template: checkout_from_event_ref_step> - name: Checkout sources uses: actions/checkout@v3.5.2 with: ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} fetch-depth: 0 # </template: checkout_from_event_ref_step> # <template: update_comment_on_start> - name: Update comment on start if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const name = 'e2e: OpenStack, ContainerdV2, Kubernetes Automatic'; const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnStart({github, context, core, name}) # </template: update_comment_on_start> # <template: import_secrets> - name: Split repository name id: split env: REPO: ${{ github.repository }} run: echo "name=${REPO##*/}" >> $GITHUB_OUTPUT - name: Import secrets id: secrets uses: hashicorp/vault-action@v2 with: url: https://seguro.flant.com path: github role: "${{ steps.split.outputs.name }}" method: jwt jwtGithubAudience: github-access-aud secrets: | projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_DEV_REGISTRY_HOST | DECKHOUSE_DEV_REGISTRY_HOST ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken login | DECKHOUSE_DEV_REGISTRY_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/dev-registry/writetoken password | DECKHOUSE_DEV_REGISTRY_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/registry_host DECKHOUSE_REGISTRY_STAGE_HOST | DECKHOUSE_REGISTRY_STAGE_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/LAYOUT_STAGE_DECKHOUSE_DOCKERCFG DOCKERCFG | LAYOUT_STAGE_DECKHOUSE_DOCKERCFG ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/github/fox_registry_secret FOX_DOCKERCFG | LAYOUT_FOX_DOCKERCFG ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken login | DECKHOUSE_REGISTRY_STAGE_USER ; projects/data/101ceaca-97cd-462f-aed5-070d9b9de175/stage-registry/writetoken password | DECKHOUSE_REGISTRY_STAGE_PASSWORD ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-id-rsa id_rsa_b64 | LAYOUT_SSH_KEY ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_TOKEN | E2E_COMMANDER_TOKEN ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/commander/environment_variables E2E_NEW_COMMANDER_HOST | E2E_COMMANDER_HOST ; projects/data/6db2f1ee-9b6f-4f4f-8381-2fb43060478a/e2e/e2e-os-password password | LAYOUT_OS_PASSWORD ; # </template: import_secrets> # <template: login_dev_registry_step> - name: Check dev registry credentials id: check_dev_registry env: HOST: ${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to dev registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_PASSWORD }} logout: false # </template: login_dev_registry_step> # <template: login_stage_registry_step> - name: Check stage registry credentials id: check_stage_registry env: HOST: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_STAGE_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to stage registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_stage_registry.outputs.has_credentials == 'true' }} with: registry: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_HOST }} username: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_USER }} password: ${{ steps.secrets.outputs.DECKHOUSE_REGISTRY_STAGE_PASSWORD }} logout: false # </template: login_stage_registry_step> # <template: login_rw_registry_step> - name: Check rw registry credentials id: check_rw_registry env: HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} run: | if [[ -n $HOST ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT fi - name: Login to rw registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} with: registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} logout: false - name: Login to Github Container Registry uses: docker/login-action@v2.1.0 if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} logout: false # </template: login_rw_registry_step> - name: Setup id: setup env: DECKHOUSE_REGISTRY_HOST: ${{ steps.secrets.outputs.DECKHOUSE_DEV_REGISTRY_HOST }} CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} REF_FULL: ${{needs.git_info.outputs.ref_full}} INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} MANUAL_RUN: "true" MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }} run: | # Calculate unique prefix for e2e test. # GITHUB_RUN_ID is a unique number for each workflow run. # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. # Add CRI and KUBERNETES_VERSION to create unique directory for each job. # CRI and PROVIDER values are trimmed to reduce prefix length. if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then KUBERNETES_VERSION_SUF="auto" else KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} fi DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") if [[ "${MANUAL_RUN}" == "false" ]] ; then # for jobs which run multiple providers concurrency (daily e2e, for example) # add provider suffix to prevent "directory already exists" error DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" fi # converts to DNS-like (all letters in lower case and replace all dots to dash) # because it prefix will use for k8s resources names (nodes, for example) DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') ## Source: ci_templates/build.yml # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. REPO_SUFFIX= fi # Use rw-registry for Git tags. SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" if [[ "$CI_COMMIT_REF_SLUG" =~ ^release-[0-9]+\.[0-9]+ ]]; then # Use stage-registry for release branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_REGISTRY_STAGE_HOST}/${REGISTRY_PATH}" else # Use dev-registry for Git branches. BRANCH_REGISTRY_PATH="${DECKHOUSE_DEV_REGISTRY_HOST}/${REGISTRY_PATH}" fi if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. # Use dev-regisry for branches and Github Container Registry for semver tags. SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" fi # Add edition name for non-FE tests on branch if [[ -n ${WERF_ENV} && ${WERF_ENV,,} != "fe" ]]; then IMAGE_EDITION=${WERF_ENV,,} fi # Prepare initial image tag for deploy/deckhouse to test switching from previous release. INITIAL_IMAGE_TAG= if [[ -n ${INITIAL_REF_SLUG} ]] ; then INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} fi # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. # Use it as image tag. Add suffix to not overlap with PRs in main repo. IMAGE_TAG=${CI_COMMIT_REF_SLUG}${IMAGE_EDITION:+-${IMAGE_EDITION}}${REPO_SUFFIX:+-${REPO_SUFFIX}} INSTALL_IMAGE_NAME= if [[ -n ${CI_COMMIT_BRANCH} ]]; then # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} fi if [[ -n ${CI_COMMIT_TAG} ]] ; then REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} fi if [[ -n ${INITIAL_REF_SLUG} ]] ; then INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} git fetch origin ${INITIAL_REF_SLUG} git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts fi SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" if [ "${MULTIMASTER}" == true ] ; then MASTERS_COUNT=3 else MASTERS_COUNT=1 fi echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}" IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" echo '::echo::on' echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT echo '::echo::off' - name: "Run e2e test: OpenStack/ContainerdV2/Automatic" id: e2e_test_run timeout-minutes: 80 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "Automatic" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} TEST_AUTOSCALER_ENABLED: ${{ needs.check_e2e_labels.outputs.autoscaler }} CIS_ENABLED: ${{ needs.check_e2e_labels.outputs.cis }} run: | echo "Execute 'script-commander.sh run-test' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} CIS_ENABLED=${CIS_ENABLED} TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export TEST_AUTOSCALER_ENABLED=${TEST_AUTOSCALER_ENABLED:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export CIS_ENABLED=${CIS_ENABLED} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh run-test # </template: e2e_run_template> # </template: e2e_tests_create_debug_logs> - name: Create Deckhouse debug logs id: e2e_tests_create_debug_logs if: failure() uses: ./.github/actions/upload-d8-debug-logs with: sshloginfile: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} sshkey: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} # </template: e2e_tests_create_debug_logs> - name: Read connection string if: ${{ failure() || cancelled() }} id: check_stay_failed_cluster uses: actions/github-script@v6.4.1 env: SSH_CONNECT_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_connection_str_file }} SSH_BASTION_STR_FILE: ${{ steps.e2e_test_run.outputs.ssh_bastion_str_file }} with: # it sets `should_run` output var if e2e/failed/stay label script: | const e2e_cleanup = require('./.github/scripts/js/e2e/cleanup'); await e2e_cleanup.readConnectionScript({core, context, github}); - name: Label pr if e2e failed if: ${{ (failure() || cancelled()) && needs.git_info.outputs.pr_number }} uses: actions-ecosystem/action-add-labels@v1 with: github_token: ${{ secrets.BOATSWAIN_GITHUB_TOKEN }} number: ${{ needs.git_info.outputs.pr_number }} labels: "e2e/cluster/failed" - name: Check pause label if: ${{ success() && needs.git_info.outputs.pr_number }} uses: actions/github-script@v6.4.1 env: PR_NUMBER: ${{needs.git_info.outputs.pr_number}} with: script: | const waitPauseRemove = require('./.github/scripts/js/e2e/wait-remove-pause-label.js')({ github, context, core }); await waitPauseRemove(); - name: Cleanup bootstrapped cluster if: inputs.autodelete||success() id: cleanup_cluster timeout-minutes: 60 env: PROVIDER: OpenStack CRI: ContainerdV2 LAYOUT: Standard KUBERNETES_VERSION: "Automatic" LAYOUT_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_DECKHOUSE_DOCKERCFG }} LAYOUT_STAGE_DECKHOUSE_DOCKERCFG: ${{ steps.secrets.outputs.LAYOUT_STAGE_DECKHOUSE_DOCKERCFG }} LAYOUT_SSH_KEY: ${{ steps.secrets.outputs.LAYOUT_SSH_KEY }} PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }} INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} # <template: e2e_run_template> TEMPLATE_ID: "19ae3748-08bf-4ad9-9102-9edbf576dc8f" LAYOUT_OS_PASSWORD: ${{ steps.secrets.outputs.LAYOUT_OS_PASSWORD }} COMMENT_ID: ${{ inputs.comment_id }} GITHUB_API_SERVER: ${{ github.api_url }} REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} COMMANDER_TOKEN: ${{ steps.secrets.outputs.E2E_COMMANDER_TOKEN }} COMMANDER_HOST: ${{ steps.secrets.outputs.E2E_COMMANDER_HOST }} run: | echo "Execute 'script-commander.sh cleanup' via 'shell', using environment: INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} PREFIX=${PREFIX} PROVIDER=${PROVIDER} CRI=${CRI} LAYOUT=${LAYOUT} KUBERNETES_VERSION=${KUBERNETES_VERSION} TMP_DIR_PATH=${TMP_DIR_PATH} MASTERS_COUNT=${MASTERS_COUNT} " export DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} export STAGE_DECKHOUSE_DOCKERCFG=${LAYOUT_STAGE_DECKHOUSE_DOCKERCFG} export PROVIDER=${PROVIDER:-not_provided} export MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} export LAYOUT=${LAYOUT:-not_provided} export SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} export LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} export FOX_DOCKERCFG=${LAYOUT_FOX_DOCKERCFG} # for logs upload ssh_connect_str_file="ssh-connect_str-${PREFIX}" echo "ssh_connection_str_file=${ssh_connect_str_file}" >> $GITHUB_OUTPUT bash $(pwd)/testing/cloud_layouts/script-commander.sh cleanup # </template: e2e_run_template> - name: Save test results if: ${{ steps.setup.outputs.dhctl-log-file }} uses: actions/upload-artifact@v4.4.0 with: name: test_output_openstack_containerdv2_Automatic path: | ${{ steps.setup.outputs.dhctl-log-file}}* ${{ steps.setup.outputs.tmp-dir-path}}/logs testing/cloud_layouts/ !testing/cloud_layouts/**/sshkey - name: Cleanup temp directory if: always() env: TMPPATH: ${{ steps.setup.outputs.tmppath}} run: | echo "Remove temporary directory '${TMPPATH}' ..." if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then rm -rf "${TMPPATH}" else echo Not a directory. fi # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'job,separate'; const name = 'e2e: OpenStack, ContainerdV2, Kubernetes Automatic'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # </template: e2e_run_job_template> last_comment: name: Update comment on finish needs: ["started_at","git_info","run_containerd_1_31","run_containerd_1_32","run_containerd_1_33","run_containerd_1_34","run_containerd_1_35","run_containerd_Automatic","run_containerdv2_1_31","run_containerdv2_1_32","run_containerdv2_1_33","run_containerdv2_1_34","run_containerdv2_1_35","run_containerdv2_Automatic"] if: ${{ always() }} runs-on: "regular" env: JOB_NAMES: | {"run_containerd_1_31":"e2e: OpenStack, Containerd, Kubernetes 1.31","run_containerd_1_32":"e2e: OpenStack, Containerd, Kubernetes 1.32","run_containerd_1_33":"e2e: OpenStack, Containerd, Kubernetes 1.33","run_containerd_1_34":"e2e: OpenStack, Containerd, Kubernetes 1.34","run_containerd_1_35":"e2e: OpenStack, Containerd, Kubernetes 1.35","run_containerd_Automatic":"e2e: OpenStack, Containerd, Kubernetes Automatic","run_containerdv2_1_31":"e2e: OpenStack, ContainerdV2, Kubernetes 1.31","run_containerdv2_1_32":"e2e: OpenStack, ContainerdV2, Kubernetes 1.32","run_containerdv2_1_33":"e2e: OpenStack, ContainerdV2, Kubernetes 1.33","run_containerdv2_1_34":"e2e: OpenStack, ContainerdV2, Kubernetes 1.34","run_containerdv2_1_35":"e2e: OpenStack, ContainerdV2, Kubernetes 1.35","run_containerdv2_Automatic":"e2e: OpenStack, ContainerdV2, Kubernetes Automatic"} steps: # <template: checkout_step> - name: Checkout sources uses: actions/checkout@v3.5.2 # </template: checkout_step> # <template: update_comment_on_finish> - name: Update comment on finish id: update_comment_on_finish if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }} env: NEEDS_CONTEXT: ${{ toJSON(needs) }} JOB_CONTEXT: ${{ toJSON(job) }} STEPS_CONTEXT: ${{ toJSON(steps) }} uses: actions/github-script@v6.4.1 with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} retries: 3 script: | const statusConfig = 'workflow,final,no-skipped,restore-separate'; const name = 'e2e: OpenStack'; const needsContext = JSON.parse(process.env.NEEDS_CONTEXT); const jobContext = JSON.parse(process.env.JOB_CONTEXT); const stepsContext = JSON.parse(process.env.STEPS_CONTEXT); let jobNames = null if (process.env.JOB_NAMES) { jobNames = JSON.parse(process.env.JOB_NAMES); } core.info(`needsContext: ${JSON.stringify(needsContext)}`); core.info(`jobContext: ${JSON.stringify(jobContext)}`); core.info(`stepsContext: ${JSON.stringify(stepsContext)}`); core.info(`jobNames: ${JSON.stringify(jobNames)}`); const ci = require('./.github/scripts/js/ci'); return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames}); # </template: update_comment_on_finish> # <template: set_e2e_requirement_status> - name: Set commit status after e2e run id: set_e2e_requirement_status if: ${{ always() }} uses: actions/github-script@v6.4.1 env: JOB_STATUS: ${{ job.status }} STATUS_TARGET_COMMIT: ${{needs.git_info.outputs.github_sha}} with: github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} script: | const e2eStatus = require('./.github/scripts/js/e2e-commit-status'); await e2eStatus.setStatusAfterE2eRun({github, context, core}); # </template: set_e2e_requirement_status> # </template: e2e_workflow_template>