qa-constrained-tip-tracking.yml

name: QA - Constrained Tip tracking

on:
  schedule:
    - cron: '0 20 * * 0'  # Run on Sunday at 08:00 PM UTC
  workflow_dispatch:     # Run manually

jobs:
  constrained-tip-tracking-test:
    timeout-minutes: 600
    strategy:
      fail-fast: false
      matrix:
        include:
          - chain: mainnet
            backend: Erigon3
            cgroup_name: constrained_res_32G
          - chain: bor-mainnet
            backend: Polygon
            cgroup_name: constrained_res_64G
    runs-on: [ self-hosted, "${{ matrix.backend }}" ]
    env:
      ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir
      ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
      ERIGON_QA_PATH: /home/qarunner/erigon-qa
      TRACKING_TIME_SECONDS: 14400 # 4 hours
      TOTAL_TIME_SECONDS: 28800 # 8 hours
      CHAIN: ${{ matrix.chain }}
      CGROUP_NAME: ${{ matrix.cgroup_name }}

    steps:
    - name: Check out repository
      uses: actions/checkout@v4

    - name: Clean Erigon Build Directory
      run: |
        make clean

    - name: Build Erigon
      run: |
        make erigon
      working-directory: ${{ github.workspace }}

    - name: Pause the Erigon instance dedicated to db maintenance
      run: |
        python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true

    - name: Restore Erigon Testbed Data Directory
      run: |
        rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/

    - name: Run Erigon, wait sync and check ability to maintain sync
      id: test_step
      run: |
        set +e # Disable exit on error
        
        # Run Erigon under memory constraints, wait sync and check ability to maintain sync
        cgexec -g memory:$CGROUP_NAME python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
          ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon3 $CHAIN standard_node statistics
  
        # Capture monitoring script exit status
        test_exit_status=$?
        
        # Save the subsection reached status
        echo "test_executed=true" >> $GITHUB_OUTPUT
        
        # Clean up Erigon process if it's still running
        if kill -0 $ERIGON_PID 2> /dev/null; then
          echo "Terminating Erigon"
          kill $ERIGON_PID
          wait $ERIGON_PID
        fi

        # Check test runner script exit status
        if [ $test_exit_status -eq 0 ]; then
          echo "Tests completed successfully"
          echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
        else
          echo "Error detected during tests"
          echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
        fi

    - name: Save test results
      if: steps.test_step.outputs.test_executed == 'true'
      env:
        TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }}
      run: |
        db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit)
        if [ -z "$db_version" ]; then
          db_version="no-version"
        fi
        
        python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
          --repo erigon \
          --commit $(git rev-parse HEAD) \
          --branch ${{ github.ref_name }} \
          --test_name constrained-tip-tracking \
          --chain $CHAIN \
          --runner ${{ runner.name }} \
          --db_version $db_version \
          --outcome $TEST_RESULT \
          --result_file ${{ github.workspace }}/result-$CHAIN.json

    - name: Upload test results
      if: steps.test_step.outputs.test_executed == 'true'
      uses: actions/upload-artifact@v4
      with:
        name: test-results-${{ env.CHAIN }}
        path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json

    - name: Upload Erigon full log
      if: steps.test_step.outputs.test_executed == 'true'
      uses: actions/upload-artifact@v4
      with:
        name: erigon-log-${{ env.CHAIN }}
        path: |
          ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/erigon.log
          ${{ env.ERIGON_TESTBED_DATA_DIR }}/proc_stat.log

    - name: Delete Erigon Testbed Data Directory
      if: always()
      run: |
        rm -rf $ERIGON_TESTBED_DATA_DIR

    - name: Resume the Erigon instance dedicated to db maintenance
      run: |
        python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true

    - name: Action for Success
      if: steps.test_step.outputs.TEST_RESULT == 'success'
      run: echo "::notice::Tests completed successfully"

    - name: Action for Not Success
      if: steps.test_step.outputs.TEST_RESULT != 'success'
      run: |
        echo "::error::Error detected during tests"
        exit 1