.github/workflows/packer.yaml - main - githubmirror/nbs

.github/workflows/packer.yaml
343 строки14 KB
Nikita Menkovich
fix: extract instance id and disk id from packer build and use them for cleanup in case of cancellation (#6539)
04 авг 2026, 17:45
04 авг 2026, 17:45a3330ab
name: Build VM Image
run-name: Build VM Image ${{ github.event.workflow_run.event }} update_image_id:${{ inputs.update_image_id }} pooled:${{ inputs.update_pooled_image_ids }} runner_version:${{ inputs.update_runner_version }}
  image_id:${{ inputs.image_id }} build_image:${{ inputs.build_image }} check_vm_creation:${{ inputs.check_vm_creation }}
on:
  # schedule:
  #   - cron: "0 5 * * *"
  workflow_call:
    inputs:
      image_id:
        description: "Image id to use, otherwise latest image will be used (or built if build_image is true)"
        required: false
        type: string
      build_image:
        description: "Build image instead of getting latest"
        required: false
        default: false
        type: boolean
      check_vm_creation:
        description: "Check VM creation"
        required: false
        default: false
        type: boolean
      update_image_id:
        description: "Update non pooled image id"
        required: false
        default: false
        type: boolean
      update_pooled_image_ids:
        description: "Update pooled image ids"
        required: false
        default: false
        type: boolean
      update_runner_version:
        description: "Update GitHub runner version repository variable"
        required: false
        default: false
        type: boolean
      images_to_keep:
        description: "Number of latest images to keep during cleanup"
        required: false
        default: 14
        type: number
      cleanup_built_image:
        description: "Delete the image created by this workflow after checks finish"
        required: false
        default: false
        type: boolean
  workflow_dispatch:
    inputs:
      image_id:
        description: "Image id to use for runner VM, if not set latest image will be used (or built if build_image is true)"
        required: false
        default: ""
        type: string
      build_image:
        description: "Build image instead of getting latest"
        required: false
        default: false
        type: boolean
      check_vm_creation:
        description: "Check VM creation"
        required: false
        default: false
        type: boolean
      update_image_id:
        description: "Update non pooled image id"
        required: false
        default: false
        type: boolean
      update_pooled_image_ids:
        description: "Update pooled image ids"
        required: false
        default: false
        type: boolean
      update_runner_version:
        description: "Update GitHub runner version repository variable"
        required: false
        default: false
        type: boolean
      images_to_keep:
        description: "Number of latest images to keep during cleanup"
        required: false
        default: 14
        type: number
      cleanup_built_image:
        description: "Delete the image created by this workflow after checks finish"
        required: false
        default: false
        type: boolean

defaults:
  run:
    shell: bash --noprofile --norc -eo pipefail -x {0}

concurrency:
  group: packer-build-vm-image
  cancel-in-progress: false

jobs:
  build:
    runs-on: [ self-hosted, "self-hosted", "runner_light" ]
    steps:
      - name: Checkout PR
        uses: actions/checkout@v7
        with:
          sparse-checkout: ".github"
          clean: true
      - name: disable sparse checkout (workaround for actions/checkout/issues/2249)
        if: ${{ always() }}
        run: git config core.sparseCheckout false
        continue-on-error: true

      - name: install dependencies
        run: |
          pip install -r .github/scripts/requirements.txt
          echo "PATH=$PATH:$HOME/nebius-cloud/bin:$HOME/.nebius/bin" >> $GITHUB_ENV
      - name: set up nebius cli
        id: cli
        uses: ./.github/actions/nebius_cli
        with:
          config_yaml: ${{ secrets.POOLED_LIGHT_NEBIUS_CONFIG_YAML }}

      - name: Resolve GitHub runner release
        id: runner-release
        if: inputs.image_id == '' && inputs.build_image == true
        env:
          GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          RUNNER_VERSION: "latest"
        run: |
          export PYTHONPATH=$PYTHONPATH:$GITHUB_WORKSPACE/.github
          python3 -m scripts.github_runner_release --version "${RUNNER_VERSION}"
      - name: set up packer
        id: setup-packer
        if: inputs.image_id == '' && inputs.build_image == true
        # it is pinned to commit because fix only exist there ans is not tagges
        # and in general it is good practice to do so
        uses: hashicorp/setup-packer@ce93c3c08a6c2ff2275bf4b54ff0d9a75f6c9789
        with:
          version: 1.15.3

      - name: init packer
        id: init-packer
        if: inputs.image_id == '' && inputs.build_image == true
        run: |
          cd .github/packer || exit
          packer init github-runner.pkr.hcl
      - name: build image
        if: inputs.image_id == '' && inputs.build_image == true
        id: build-image
        timeout-minutes: 60
        env:
          PASSWORD_HASH: ${{ secrets.VM_USER_PASSWD }}
          NEBIUS_IAM_TOKEN: ${{ steps.cli.outputs.NEBIUS_IAM_TOKEN }}
          RUNNER_VERSION: ${{ steps.runner-release.outputs.RUNNER_VERSION }}
          RUNNER_SHA256_X64: ${{ steps.runner-release.outputs.RUNNER_SHA256_X64 }}
          GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          PARENT_ID: ${{ vars.POOLED_HEAVY_PARENT_ID }}
          SUBNET_ID: ${{ vars.POOLED_HEAVY_SUBNET_ID }}
        run: |
          cd .github/packer || exit
          set +e
          PACKER_NO_COLOR=1 packer build \
              -var "RUNNER_VERSION=${RUNNER_VERSION}" \
              -var "RUNNER_SHA256_X64=${RUNNER_SHA256_X64}" \
              github-runner.pkr.hcl 2>&1 | awk -v github_output="${GITHUB_OUTPUT}" '
                  {
                      print
                      fflush()
                      for (i = 1; i < NF; i++) {
                          if ($i == "disk" && $(i + 1) ~ /^computedisk-[[:alnum:]]+$/) {
                              print "PACKER_DISK_ID=" $(i + 1) >> github_output
                              close(github_output)
                          }
                          if ($i == "instance" && $(i + 1) ~ /^computeinstance-[[:alnum:]]+$/) {
                              print "PACKER_INSTANCE_ID=" $(i + 1) >> github_output
                              close(github_output)
                          }
                      }
                  }
              '
          PACKER_EXIT_CODE=${PIPESTATUS[0]}
          set -e
          if ((PACKER_EXIT_CODE != 0)); then
              exit "${PACKER_EXIT_CODE}"
          fi
          cat reports/manifest.json
          echo "IMAGE_ID_2204=$(jq -r '.builds[0].artifact_id' reports/manifest.json)" >> "${GITHUB_OUTPUT}"
      - name: Show Packer resource IDs
        if: always()
        env:
          PACKER_INSTANCE_ID: ${{ steps.build-image.outputs.PACKER_INSTANCE_ID }}
          PACKER_DISK_ID: ${{ steps.build-image.outputs.PACKER_DISK_ID }}
        run: |
          echo "PACKER_INSTANCE_ID=${PACKER_INSTANCE_ID}"
          echo "PACKER_DISK_ID=${PACKER_DISK_ID}"
      - name: Remove Nebius resources by IDs
        if: >-
          always() &&
          (steps.build-image.outcome == 'failure' ||
          steps.build-image.outcome == 'cancelled') &&
          (steps.build-image.outputs.PACKER_INSTANCE_ID != '' ||
          steps.build-image.outputs.PACKER_DISK_ID != '')
        uses: ./.github/actions/nebius_remove_by_ids
        timeout-minutes: 30
        with:
          nebius_config_yaml: ${{ secrets.POOLED_LIGHT_NEBIUS_CONFIG_YAML }}
          instance_id: ${{ steps.build-image.outputs.PACKER_INSTANCE_ID }}
          disk_id: ${{ steps.build-image.outputs.PACKER_DISK_ID }}
      - name: Get latest image ID from Nebius
        id: get-image-id
        env:
          IMAGE_ID: ${{ inputs.image_id }}
          BUILT_IMAGE_ID: ${{ steps.build-image.outputs.IMAGE_ID_2204 }}
          PARENT_ID: ${{ vars.POOLED_HEAVY_PARENT_ID }}
        run: |
          if [ -n "${IMAGE_ID}" ]; then
              if [[ "${IMAGE_ID}" != computeimage-* ]]; then
                  echo "Provided image id is not valid: ${IMAGE_ID}"
                  exit 1
              fi
              echo "IMAGE_ID_2204=${IMAGE_ID}" >> $GITHUB_OUTPUT
              echo "Using image id provided by inputs: ${IMAGE_ID}"
              exit 0
          fi
          if [ -n "${BUILT_IMAGE_ID}" ]; then
              echo "IMAGE_ID_2204=${BUILT_IMAGE_ID}" >> $GITHUB_OUTPUT
              echo "Using image id built by this workflow: ${BUILT_IMAGE_ID}"
              exit 0
          fi
          IMAGE_ID=$(nebius compute image get-latest-by-family --image-family ubuntu22.04-nbs-github-ci --parent-id "${PARENT_ID}" --format json | jq -r .metadata.id)
          echo "IMAGE_ID_2204=$IMAGE_ID" >> $GITHUB_OUTPUT
      - name: Start runner
        id: start-runner
        if: >-
          always() &&
          inputs.check_vm_creation == true &&
          (inputs.image_id != '' || inputs.build_image != true || steps.build-image.outcome == 'success') &&
          steps.get-image-id.outputs.IMAGE_ID_2204 != '' &&
          (steps.get-image-id.outputs.IMAGE_ID_2204 != vars.NEBIUS_IMAGE_ID_2204 ||
          steps.get-image-id.outputs.IMAGE_ID_2204 != vars.POOLED_LIGHT_IMAGE_ID_2204 ||
          steps.get-image-id.outputs.IMAGE_ID_2204 != vars.POOLED_HEAVY_IMAGE_ID_2204)
        uses: ./.github/actions/nebius_runner_create
        timeout-minutes: 60
        with:
          repo_owner: ${{ github.repository_owner }}
          repo: ${{ github.event.repository.name }}
          nebius_config_yaml: ${{ secrets.POOLED_LIGHT_NEBIUS_CONFIG_YAML }}
          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          vm_parent_id: ${{ vars.POOLED_HEAVY_PARENT_ID }}
          vm_name: ${{ format('packer-run-{0}-{1}', github.run_id, github.run_attempt) }}
          vm_platform_id: ${{ vars.POOLED_HEAVY_PLATFORM_ID }}
          vm_preset: "4vcpu-16gb"
          vm_disk_type: "network-ssd-nonreplicated"
          vm_disk_size: 93
          vm_subnet: ${{ vars.POOLED_HEAVY_SUBNET_ID }}
          vm_image: ${{ steps.get-image-id.outputs.IMAGE_ID_2204 }}
          vm_labels: ${{ format('run={0}-{1},repo={2},owner={3}', github.run_id, github.run_attempt, github.event.repository.name, github.repository_owner) }}
          vm_user_passwd: ${{ secrets.VM_USER_PASSWD }}
          vm_public_ip: ${{ vars.POOLED_HEAVY_PUBLIC_IP_ENABLE == 'yes' && 'yes' || 'no' }}

      - name: Stop runner
        uses: ./.github/actions/nebius_runner_remove
        if: always() && steps.start-runner.outcome != 'skipped'
        timeout-minutes: 60
        with:
          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          repo_owner: ${{ github.repository_owner }}
          repo: ${{ github.event.repository.name }}
          nebius_config_yaml: ${{ secrets.POOLED_LIGHT_NEBIUS_CONFIG_YAML }}
          vm_parent_id: ${{ vars.POOLED_HEAVY_PARENT_ID }}
          vm_id: ${{ steps.start-runner.outputs.INSTANCE_ID }}
          vm_labels: ${{ format('run={0}-{1},repo={2},owner={3}', github.run_id, github.run_attempt, github.event.repository.name, github.repository_owner) }}

      - name: Set new image id
        if: inputs.update_image_id == true
        run: |
          export PYTHONPATH=$PYTHONPATH:$GITHUB_WORKSPACE/.github
          python3 -m scripts.nebius_manage_images \
              --new-image-id ${NEW_IMAGE_ID} \
              --parent-id ${PARENT_ID} \
              --remove-old-images \
              --images-to-keep ${IMAGES_TO_KEEP} \
              --image-variable-name NEBIUS_IMAGE_ID_2204 ${UPDATE_IMAGE_ID}
        env:
          GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          NEW_IMAGE_ID: ${{ steps.get-image-id.outputs.IMAGE_ID_2204 }}
          UPDATE_IMAGE_ID: ${{ inputs.update_image_id && '--update-image-id' || '' }}
          IMAGES_TO_KEEP: ${{ inputs.images_to_keep }}
          PARENT_ID: ${{ vars.NEBIUS_PARENT_ID }}

      - name: Set new pooled image id
        if: inputs.update_pooled_image_ids == true
        run: |
          export PYTHONPATH=$PYTHONPATH:$GITHUB_WORKSPACE/.github
          python3 -m scripts.nebius_manage_images \
              --parent-id ${LIGHT_PARENT_ID} \
              --new-image-id ${NEW_IMAGE_ID} \
              --remove-old-images \
              --images-to-keep ${IMAGES_TO_KEEP} \
              --image-variable-name POOLED_LIGHT_IMAGE_ID_2204 ${UPDATE_IMAGE_ID}

          python3 -m scripts.nebius_manage_images \
              --parent-id ${HEAVY_PARENT_ID} \
              --new-image-id ${NEW_IMAGE_ID} \
              --remove-old-images \
              --images-to-keep ${IMAGES_TO_KEEP} \
              --image-variable-name POOLED_HEAVY_IMAGE_ID_2204 ${UPDATE_IMAGE_ID}
        env:
          GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          NEW_IMAGE_ID: ${{ steps.get-image-id.outputs.IMAGE_ID_2204 }}
          UPDATE_IMAGE_ID: ${{ inputs.update_pooled_image_ids && '--update-image-id' || '' }}
          IMAGES_TO_KEEP: ${{ inputs.images_to_keep }}
          HEAVY_PARENT_ID: ${{ vars.POOLED_HEAVY_PARENT_ID }}
          LIGHT_PARENT_ID: ${{ vars.POOLED_LIGHT_PARENT_ID }}

      - name: Set runner version
        if: >-
          inputs.image_id == '' &&
          inputs.build_image == true &&
          inputs.update_runner_version == true &&
          steps.start-runner.outcome == 'success' &&
          steps.runner-release.outputs.RUNNER_VERSION != vars.RUNNER_VERSION
        env:
          GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          RUNNER_VERSION: ${{ steps.runner-release.outputs.RUNNER_VERSION }}
        run: |
          export PYTHONPATH=$PYTHONPATH:$GITHUB_WORKSPACE/.github
          python3 -m scripts.github_actions_variables \
              --variable-name RUNNER_VERSION \
              --value "${RUNNER_VERSION}"

      - name: Remove built image
        if: >-
          always() &&
          inputs.cleanup_built_image == true &&
          inputs.update_image_id != true &&
          inputs.update_pooled_image_ids != true &&
          steps.build-image.outputs.IMAGE_ID_2204 != ''
        env:
          IMAGE_ID: ${{ steps.build-image.outputs.IMAGE_ID_2204 }}
        run: |
          export PYTHONPATH=$PYTHONPATH:$GITHUB_WORKSPACE/.github
          python3 -m scripts.nebius_manage_images --delete-image-id "${IMAGE_ID}"