build_and_test_on_demand.yaml

name: Build and test NBS on demand

on:
  workflow_dispatch:
    inputs:
      build_target:
        type: string
        default: "cloud/blockstore/apps/,cloud/filestore/apps/,cloud/disk_manager/,cloud/tasks/"
        description: "limit build to specific target"
      test_target:
        type: string
        default: "cloud/blockstore/,cloud/filestore/,cloud/disk_manager/,cloud/tasks/"
        description: "limit test to specific target"
      build_preset:
        type: choice
        default: "relwithdebinfo"
        description: "Build preset"
        options:
          - debug
          - relwithdebinfo
          - release-asan
          - release-tsan
      cache_update:
        type: boolean
        default: false
        description: "Update build cache for build and cache"
      test_size:
        type: choice
        default: "small"
        description: "test size to run"
        options:
          - small
          - small,medium
          - small,medium,large
      test_type:
        type: choice
        default: "unittest"
        description: "type of tests to run"
        options:
          - unittest,clang_tidy,gtest
          - py3test,py2test,pytest,flake8,black,py2_flake8
          - go_test,gofmt
          - unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,go_test,gofmt
      sleep_after_tests:
        type: string
        default: "0"
        description: "Amount of seconds to sleep after tests"
      run_build:
        type: boolean
        default: true
        description: "run build"
      run_tests:
        type: boolean
        default: true
        description: "run tests"
  workflow_call:
    inputs:
      build_target:
        type: string
        default: "cloud/blockstore/apps/,cloud/filestore/apps/,cloud/disk_manager/,cloud/tasks/"
        description: "limit build to specific target"
      test_target:
        type: string
        default: "cloud/blockstore/,cloud/filestore/,cloud/disk_manager/,cloud/tasks/"
        description: "limit test to specific target"
      build_preset:
        type: string
        default: "relwithdebinfo"
      test_size:
        type: string
        default: "small,medium,large"
      test_type:
        type: string
        default: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,go_test,gofmt"
      run_build:
        type: boolean
        default: true
      run_tests:
        type: boolean
        default: true
      sleep_after_tests:
        type: string
        default: "0"
        description: "Amount of seconds to sleep after tests"
      cache_update_build:
        type: boolean
        default: false
        description: "Update build cache in build stage"
      cache_update_tests:
        type: boolean
        default: false
        description: "Update build cache in test stage"
      link_threads:
        type: string
        default: "24"
        description: "--link-threads in ya make"
      test_threads:
        type: string
        default: "12"
        description: "--test-threads in ya test"


jobs:
  provide-runner:
    name: Start self-hosted runner
    timeout-minutes: 60
    if: always()
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-runner.outputs.label }}
      instance-id: ${{ steps.start-runner.outputs.instance-id }}
    steps:
      - name: Start runner
        id: start-runner
        if: always()
        uses: ./.github/actions/runner_start
        timeout-minutes: 60
        with:
          org: ${{ github.repository_owner }}
          team: NBS
          repo_owner: ${{ github.repository_owner }}
          repo: ${{ github.event.repository.name }}
          service_account_key: ${{ secrets.NEBIUS_SA_JSON_CREDENTIALS }}
          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          vm_folder: bjeuq5o166dq4ukv3eec
          vm_name: ${{ github.event.pull_request.number && format('pr-{0}', github.event.pull_request.number) || format('run-{0}-{1}', github.run_id, github.run_attempt) }}
          vm_zone: eu-north1-c
          vm_cpu: 60
          vm_memory: 240
          vm_disk_type: network-ssd-nonreplicated
          vm_disk_size: 1023
          vm_subnet: f8uh0ml4rhb45nde9p75
          vm_image: ${{ vars.IMAGE_ID_2204 }}
          vm_labels: ${{ github.event.pull_request.number && format('pr={0},run={1}-{2}', github.event.pull_request.number, github.run_id, github.run_attempt) || format('run={0}-{1}', github.run_id, github.run_attempt) }}


  prepare-vm:
    name: Prepare runner [id=${{ needs.provide-runner.outputs.instance-id }}]
    needs: provide-runner
    runs-on: [ self-hosted, "${{ needs.provide-runner.outputs.label }}" ]
    outputs:
      runner_ipv4: ${{ steps.configure-ncp.outputs.runner_ipv4 }}
    steps:
      - name: Checkout PR
        uses: actions/checkout@v4
        if: github.event.pull_request.head.sha != ''
        with:
          submodules: true
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: ${{ contains(github.event.pull_request.labels.*.name, 'rebase') && 0 || 1 }}
      - name: Rebase PR
        if: ${{ github.event.pull_request.head.sha != '' && contains(github.event.pull_request.labels.*.name, 'rebase') }}
        shell: bash
        run: |
          git fetch origin main
          git rebase origin/main
      - name: Checkout
        uses: actions/checkout@v4
        if: github.event.pull_request.head.sha == ''
        with:
          submodules: true
      - id: configure-ncp
        name: Configure NCP
        uses: ./.github/actions/ncp
        with:
          instance_id: ${{ needs.provide-runner.outputs.instance-id }}
          sa_json: ${{ secrets.NEBIUS_GITHUB_USER_SA_JSON}}
      - id: configure-authorized-keys
        name: Configure authorized_keys
        uses: ./.github/actions/ssh_keys
        with:
          org: ydb-platform
          team: nbs
          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      - name: Prepare VM
        uses: ./.github/actions/prepare

  build-and-test:
    name: Build and test NBS [id=${{ needs.provide-runner.outputs.instance-id }} ip=${{ needs.prepare-vm.outputs.runner_ipv4 }}]
    uses: ./.github/workflows/build_and_test_ya.yaml
    needs:
      - provide-runner
      - prepare-vm
    with:
      runner_kind: self-hosted
      runner_label: ${{ needs.provide-runner.outputs.label }}
      runner_instance_id: ${{ needs.provide-runner.outputs.instance-id }}
      runner_ipv4: ${{ needs.prepare-vm.outputs.runner_ipv4 }}
      build_target: ${{ inputs.build_target }}
      test_target: ${{ inputs.test_target }}
      build_preset: ${{ inputs.build_preset }}
      run_build: ${{ inputs.run_build }}
      run_tests: ${{ inputs.run_tests }}
      test_size: ${{ inputs.test_size }}
      test_type: ${{ inputs.test_type }}
      link_threads: ${{ inputs.link_threads || '24' }}
      test_threads: ${{ inputs.test_threads || '12' }}
      sleep_after_tests: ${{ inputs.sleep_after_tests }}
      cache_update_build: ${{ fromJson(github.event_name == 'workflow_dispatch' && format('{0}', inputs.cache_update) || format('{0}', inputs.cache_update_build)) }}
      cache_update_tests: ${{ fromJson(github.event_name == 'workflow_dispatch' && format('{0}', inputs.cache_update) || format('{0}', inputs.cache_update_tests)) }}
    secrets: inherit

  sleep-if-needed:
    name: Sleep ${{ needs.build-and-test.outputs.sleep_after_tests || '0' }}s if build failed
    if: ${{ !cancelled() }}
    needs:
      - provide-runner # required to get output from the start-runner job
      - build-and-test # required to wait when the main job is done
    runs-on: ubuntu-latest
    steps:
      - name: Sleep ${{ needs.build-and-test.outputs.sleep_after_tests  || '0' }}s if build failed
        shell: bash
        if: ${{ needs.build-and-test.outputs.sleep_after_tests && needs.build-and-test.outputs.sleep_after_tests != '0'  }}
        run: sleep ${{ needs.build-and-test.outputs.sleep_after_tests }}

  release-runner:
    name: Release self-hosted runner ${{ needs.provide-runner.outputs.instance-id }}
    needs:
      - provide-runner  # required to get output from the start-runner job
      - build-and-test  # required to wait when the main job is done
      - sleep-if-needed # required to wait until all debug is done
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Stop runner
        uses: ./.github/actions/runner_remove
        if: always()
        timeout-minutes: 60
        with:
          service_account_key: ${{ secrets.NEBIUS_SA_JSON_CREDENTIALS }}
          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          vm_id: ${{ needs.provide-runner.outputs.instance-id }}