From aa8691abfff37f9c8639499d714e8c1cd27173e4 Mon Sep 17 00:00:00 2001 From: Nikita Menkovich Date: Tue, 10 Dec 2024 15:39:41 +0100 Subject: [PATCH 1/8] feat: add shell extractor script that will extract contents of run sections in actions --- .github/scripts/shell-extractor.py | 158 +++++++++++++++++++++++ .github/workflows/pr-github-actions.yaml | 6 + 2 files changed, 164 insertions(+) create mode 100644 .github/scripts/shell-extractor.py diff --git a/.github/scripts/shell-extractor.py b/.github/scripts/shell-extractor.py new file mode 100644 index 00000000000..3501989f6ef --- /dev/null +++ b/.github/scripts/shell-extractor.py @@ -0,0 +1,158 @@ +import os +import yaml +import uuid + +ACTIONS_DIR = ".github/actions" +WORKFLOWS_DIR = ".github/workflows" +TEMP_ACTIONS_DIR = ".github/temporary/actions" +TEMP_WORKFLOWS_DIR = ".github/temporary/workflows" + +os.makedirs(TEMP_ACTIONS_DIR, exist_ok=True) +os.makedirs(TEMP_WORKFLOWS_DIR, exist_ok=True) + +def load_yaml_file(filepath): + with open(filepath, 'r') as f: + return yaml.safe_load(f) + +def extract_runs_from_workflow(data): + """Extract run commands from a workflow YAML structure.""" + runs = [] + if not data or 'jobs' not in data: + return runs + + for job_id, job_data in data['jobs'].items(): + steps = job_data.get('steps', []) + if steps and isinstance(steps, list): + for step in steps: + if 'run' in step: + runs.append(step['run']) + return runs + +def extract_runs_from_action(data): + """Extract run commands from a composite action YAML structure.""" + runs = [] + if not data or 'runs' not in data: + return runs + + runs_data = data['runs'] + if runs_data.get('using') == 'composite': + steps = runs_data.get('steps', []) + if steps and isinstance(steps, list): + for step in steps: + if 'run' in step: + runs.append(step['run']) + return runs + +def parse_command_blocks(run_content): + """ + Parse the run content into command blocks. + + Rules: + - If a line ends with a backslash, it continues the same command block. + - If we encounter a line with '< Date: Tue, 10 Dec 2024 15:43:49 +0100 Subject: [PATCH 2/8] black --- .github/scripts/shell-extractor.py | 40 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/scripts/shell-extractor.py b/.github/scripts/shell-extractor.py index 3501989f6ef..86ce68ab465 100644 --- a/.github/scripts/shell-extractor.py +++ b/.github/scripts/shell-extractor.py @@ -10,39 +10,43 @@ os.makedirs(TEMP_ACTIONS_DIR, exist_ok=True) os.makedirs(TEMP_WORKFLOWS_DIR, exist_ok=True) + def load_yaml_file(filepath): - with open(filepath, 'r') as f: + with open(filepath, "r") as f: return yaml.safe_load(f) + def extract_runs_from_workflow(data): """Extract run commands from a workflow YAML structure.""" runs = [] - if not data or 'jobs' not in data: + if not data or "jobs" not in data: return runs - for job_id, job_data in data['jobs'].items(): - steps = job_data.get('steps', []) + for job_id, job_data in data["jobs"].items(): + steps = job_data.get("steps", []) if steps and isinstance(steps, list): for step in steps: - if 'run' in step: - runs.append(step['run']) + if "run" in step: + runs.append(step["run"]) return runs + def extract_runs_from_action(data): """Extract run commands from a composite action YAML structure.""" runs = [] - if not data or 'runs' not in data: + if not data or "runs" not in data: return runs - runs_data = data['runs'] - if runs_data.get('using') == 'composite': - steps = runs_data.get('steps', []) + runs_data = data["runs"] + if runs_data.get("using") == "composite": + steps = runs_data.get("steps", []) if steps and isinstance(steps, list): for step in steps: - if 'run' in step: - runs.append(step['run']) + if "run" in step: + runs.append(step["run"]) return runs + def parse_command_blocks(run_content): """ Parse the run content into command blocks. @@ -87,7 +91,7 @@ def parse_command_blocks(run_content): # If not heredoc, check if line ends with a backslash stripped = line.strip() current_block.append(line) - if not stripped.endswith('\\'): + if not stripped.endswith("\\"): # This block ends here command_blocks.append(current_block) current_block = [] @@ -98,6 +102,7 @@ def parse_command_blocks(run_content): return command_blocks + def write_runs_to_files(runs, output_dir, prefix): """ Write each run command to a unique .sh file in the given output_dir. @@ -113,7 +118,7 @@ def write_runs_to_files(runs, output_dir, prefix): command_blocks = parse_command_blocks(run_content) - with open(filepath, 'w') as f: + with open(filepath, "w") as f: f.write("#!/usr/bin/env bash\n\n") for block in command_blocks: # Check if this block contains GitHub variables @@ -122,10 +127,11 @@ def write_runs_to_files(runs, output_dir, prefix): for line in block: f.write(line + "\n") + def process_workflows(): for root, dirs, files in os.walk(WORKFLOWS_DIR): for file in files: - if file.endswith(('.yml', '.yaml')): + if file.endswith((".yml", ".yaml")): filepath = os.path.join(root, file) data = load_yaml_file(filepath) runs = extract_runs_from_workflow(data) @@ -133,6 +139,7 @@ def process_workflows(): base_name = os.path.splitext(file)[0] write_runs_to_files(runs, TEMP_WORKFLOWS_DIR, base_name) + def process_actions(): # For actions, we assume each action directory under .github/actions contains an action.yml or action.yaml for action_dir in os.listdir(ACTIONS_DIR): @@ -140,7 +147,7 @@ def process_actions(): if os.path.isdir(full_path): # Look for action.yml or action.yaml action_file = None - for candidate in ['action.yml', 'action.yaml']: + for candidate in ["action.yml", "action.yaml"]: candidate_path = os.path.join(full_path, candidate) if os.path.exists(candidate_path): action_file = candidate_path @@ -152,6 +159,7 @@ def process_actions(): if runs: write_runs_to_files(runs, TEMP_ACTIONS_DIR, action_dir) + if __name__ == "__main__": process_workflows() process_actions() From 26501a25b82b886c230a3c0088970a05b30832cf Mon Sep 17 00:00:00 2001 From: Nikita Menkovich Date: Tue, 10 Dec 2024 15:59:11 +0100 Subject: [PATCH 3/8] fix --- .github/workflows/pr-github-actions.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-github-actions.yaml b/.github/workflows/pr-github-actions.yaml index ade3b80ef43..34fb778442d 100644 --- a/.github/workflows/pr-github-actions.yaml +++ b/.github/workflows/pr-github-actions.yaml @@ -48,16 +48,20 @@ jobs: run: pip install pyyaml - name: generate shellscripts from github actions - run: python .github/scripts/shell-extractor.py + run: | + python .github/scripts/shell-extractor.py + find .github/temporary/ -type f -name "*.sh"; - name: shellcheck - uses: reviewdog/action-shellcheck@v1.19.0 + uses: reviewdog/action-shellcheck@v1.28.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: ${{ steps.reporter.outputs.value }} path: | .github/ pattern: "*.sh" + level: "error" + fail_on_error: "true" - name: shfmt uses: librarian/action-shfmt@v0.0.1 From b479668e99f2d8125de5a63e7c81daa9a2252fcc Mon Sep 17 00:00:00 2001 From: Nikita Menkovich Date: Tue, 10 Dec 2024 17:05:27 +0100 Subject: [PATCH 4/8] fix shfmt issue with extracted files --- .github/scripts/shell-extractor.py | 4 ++-- .github/workflows/pr-github-actions.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/shell-extractor.py b/.github/scripts/shell-extractor.py index 86ce68ab465..a394e45e33a 100644 --- a/.github/scripts/shell-extractor.py +++ b/.github/scripts/shell-extractor.py @@ -4,8 +4,8 @@ ACTIONS_DIR = ".github/actions" WORKFLOWS_DIR = ".github/workflows" -TEMP_ACTIONS_DIR = ".github/temporary/actions" -TEMP_WORKFLOWS_DIR = ".github/temporary/workflows" +TEMP_ACTIONS_DIR = ".temporary/actions" +TEMP_WORKFLOWS_DIR = ".temporary/workflows" os.makedirs(TEMP_ACTIONS_DIR, exist_ok=True) os.makedirs(TEMP_WORKFLOWS_DIR, exist_ok=True) diff --git a/.github/workflows/pr-github-actions.yaml b/.github/workflows/pr-github-actions.yaml index 34fb778442d..ed0bc2bda7e 100644 --- a/.github/workflows/pr-github-actions.yaml +++ b/.github/workflows/pr-github-actions.yaml @@ -59,9 +59,9 @@ jobs: reporter: ${{ steps.reporter.outputs.value }} path: | .github/ + .temporary/ pattern: "*.sh" level: "error" - fail_on_error: "true" - name: shfmt uses: librarian/action-shfmt@v0.0.1 From 5ff797988b602045e0a6047802f8630255aebd4c Mon Sep 17 00:00:00 2001 From: Nikita Menkovich Date: Tue, 10 Dec 2024 17:12:45 +0100 Subject: [PATCH 5/8] remove reviewdog shellcheck, replace with another github action --- .github/scripts/shell-extractor.py | 2 +- .github/workflows/pr-github-actions.yaml | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/scripts/shell-extractor.py b/.github/scripts/shell-extractor.py index a394e45e33a..6028e222d58 100644 --- a/.github/scripts/shell-extractor.py +++ b/.github/scripts/shell-extractor.py @@ -163,4 +163,4 @@ def process_actions(): if __name__ == "__main__": process_workflows() process_actions() - print("Run commands have been extracted and saved in .github/temporary/") + print("Run commands have been extracted and saved in .temporary/") diff --git a/.github/workflows/pr-github-actions.yaml b/.github/workflows/pr-github-actions.yaml index ed0bc2bda7e..b55e9776097 100644 --- a/.github/workflows/pr-github-actions.yaml +++ b/.github/workflows/pr-github-actions.yaml @@ -50,18 +50,17 @@ jobs: - name: generate shellscripts from github actions run: | python .github/scripts/shell-extractor.py - find .github/temporary/ -type f -name "*.sh"; + find .temporary/ -type f -name "*.sh"; - - name: shellcheck - uses: reviewdog/action-shellcheck@v1.28.0 + - name: shellcheck for .github dir + uses: ludeeus/action-shellcheck@master with: - github_token: ${{ secrets.GITHUB_TOKEN }} - reporter: ${{ steps.reporter.outputs.value }} - path: | - .github/ - .temporary/ - pattern: "*.sh" - level: "error" + scandir: .github + + - name: shellcheck for github actions shell scripts in .temporary dir + uses: ludeeus/action-shellcheck@master + with: + scandir: .temporary - name: shfmt uses: librarian/action-shfmt@v0.0.1 From 1a7c08fb593e6b92f0cd5d33d18a66d87cc7e4f5 Mon Sep 17 00:00:00 2001 From: Nikita Menkovich Date: Tue, 10 Dec 2024 18:27:12 +0100 Subject: [PATCH 6/8] fix: various shellcheck errors --- .github/actions/build/action.yaml | 24 +++++--- .github/actions/build_cmake/action.yaml | 25 ++++---- .github/actions/nebius_cli/action.yaml | 10 ++-- .github/actions/prepare/action.yaml | 1 + .github/actions/s3cmd/action.yaml | 3 +- .github/actions/ssh_keys/action.yaml | 4 +- .github/actions/test/action.yaml | 57 +++++++++++-------- .github/actions/test_cmake/action.yaml | 10 ++-- .github/scripts/shell-extractor.py | 5 +- .github/workflows/github_actions_scripts.yaml | 4 +- .github/workflows/packer.yaml | 10 ++-- .github/workflows/pr-github-actions.yaml | 9 ++- .github/workflows/pr.yaml | 22 +++---- 13 files changed, 106 insertions(+), 78 deletions(-) diff --git a/.github/actions/build/action.yaml b/.github/actions/build/action.yaml index 4247ede4ae7..b98f8fea79e 100644 --- a/.github/actions/build/action.yaml +++ b/.github/actions/build/action.yaml @@ -46,6 +46,7 @@ runs: echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV rm -rf $TMP_DIR && mkdir $TMP_DIR && chown -R github:github $TMP_DIR $GITHUB_WORKSPACE + # shellcheck disable=SC2193 if [ "${{ inputs.clean_ya_dir }}" == "yes" ] && [ -d /home/github/.ya/ ]; then echo "Cleaning ya dir" rm -rf /home/github/.ya/ @@ -55,35 +56,40 @@ runs: shell: bash --noprofile --norc -eo pipefail -x {0} run: | function grep_ya_tc() { - ps aux | grep [y]a-tc || true + ps aux | grep "[y]a-tc" || true } extra_params=() - if [ ! -z "${{ inputs.build_target }}" ]; then + # shellcheck disable=SC2157 + if [ -n "${{ inputs.build_target }}" ]; then readarray -d ',' -t targets < <(printf "%s" "${{ inputs.build_target }}") for target in "${targets[@]}"; do extra_params+=(--target="${target}") done fi + # shellcheck disable=SC2157,SC2193 if [ "${{ inputs.use_network_cache }}" == "yes" ]; then - if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then + if [ -n "${{ inputs.bazel_remote_uri }}" ]; then extra_params+=(--bazel-remote-store) extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") fi - if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then + # shellcheck disable=SC2157 + if [ -n "${{ inputs.bazel_remote_username }}" ]; then extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}") extra_params+=(--add-result .o) fi + # shellcheck disable=SC2193 if [ "${{ inputs.cache_update }}" == "true" ]; then extra_params+=(--bazel-remote-put) fi fi + # shellcheck disable=SC2195 case "${{ inputs.build_preset }}" in debug) build_type=debug @@ -132,11 +138,11 @@ runs: start_time=$(date +%s) while true; do - pgrep -x "ya-tc" && { - echo "ya-tc is still running." + pgrep -x "$process_name" && { + echo "$process_name is still running." grep_ya_tc } || { - echo "ya-tc is not running." + echo "$process_name is not running." grep_ya_tc break } @@ -145,9 +151,9 @@ runs: elapsed_time=$((current_time - start_time)) if [ "$elapsed_time" -ge "$timeout" ]; then - echo "Timeout reached. ya-tc is still running. killing it" + echo "Timeout reached. $process_name is still running. killing it" grep_ya_tc - pkill -f ya-tc || true + pkill -f $process_name || true grep_ya_tc break fi diff --git a/.github/actions/build_cmake/action.yaml b/.github/actions/build_cmake/action.yaml index 8c60e5f1899..b776032e931 100644 --- a/.github/actions/build_cmake/action.yaml +++ b/.github/actions/build_cmake/action.yaml @@ -24,17 +24,19 @@ runs: run: | export TMP_DIR=$(pwd)/tmp_build export ROOT_PATH=$(pwd) - echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV + { + echo "TMP_DIR=$TMP_DIR" + echo "ROOT_PATH=$ROOT_PATH" + echo "CONAN_USER_HOME=$TMP_DIR" + echo "CCACHE_SLOPPINESS=locale" + echo "CCACHE_BASEDIR=$TMP_DIR" + echo "CCACHE_MAXSIZE=500G" + echo "CCACHE_NOREADONLY=1" + echo "CCACHE_REMOTE_ONLY=1" + echo "CCACHE_DEBUG=1" + echo "CCACHE_REMOTE_STORAGE=http://${{ inputs.bazel_remote_username }}:${{ inputs.bazel_remote_password }}@195.242.17.155:9090|layout=bazel" + } >> $GITHUB_ENV rm -rf $TMP_DIR && mkdir $TMP_DIR - echo "ROOT_PATH=$ROOT_PATH" >> $GITHUB_ENV - echo "CONAN_USER_HOME=$TMP_DIR" >> $GITHUB_ENV - echo "CCACHE_SLOPPINESS=locale" >> $GITHUB_ENV - echo "CCACHE_BASEDIR=$TMP_DIR" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=500G" >> $GITHUB_ENV - echo "CCACHE_NOREADONLY=1" >> $GITHUB_ENV - echo "CCACHE_REMOTE_ONLY=1" >> $GITHUB_ENV - echo "CCACHE_DEBUG=1" >> $GITHUB_ENV - echo "CCACHE_REMOTE_STORAGE=http://${{ inputs.bazel_remote_username }}:${{ inputs.bazel_remote_password }}@195.242.17.155:9090|layout=bazel" >> $GITHUB_ENV - name: build shell: bash --noprofile --norc -eo pipefail -x {0} @@ -42,6 +44,7 @@ runs: cd $TMP_DIR export + # shellcheck disable=SC2195 case "${{ inputs.build_preset }}" in debug) export CMAKE_BUILD_TYPE=Debug @@ -57,7 +60,7 @@ runs: exit 1 ;; esac - if [ $(git branch --show-current) == "main" ]; then + if [ "$(git branch --show-current)" == "main" ]; then cp ${ROOT_PATH}/contrib/ydb/library/yql/minikql/codegen/codegen_llvm_deps.h.txt ${ROOT_PATH}/contrib/ydb/library/yql/minikql/codegen/codegen_llvm_deps.h fi diff --git a/.github/actions/nebius_cli/action.yaml b/.github/actions/nebius_cli/action.yaml index ad9dd4e635b..6eb1e3ecc70 100644 --- a/.github/actions/nebius_cli/action.yaml +++ b/.github/actions/nebius_cli/action.yaml @@ -31,11 +31,11 @@ runs: ${sa_json} EOF - cat sa.json | jq -r '."subject-credentials"."private-key"' > private.pem - echo "::add-mask::$(cat sa.json | jq -r '."subject-credentials"."kid"')" - public_key_id=$(cat sa.json | jq -r '."subject-credentials"."kid"') - echo "::add-mask::$(cat sa.json | jq -r '."subject-credentials"."iss"')" - service_account_id=$(cat sa.json | jq -r '."subject-credentials"."iss"') + jq -r '."subject-credentials"."private-key"' sa.json > private.pem + echo "::add-mask::$(jq -r '."subject-credentials"."kid"' sa.json)" + public_key_id=$(jq -r '."subject-credentials"."kid"' sa.json) + echo "::add-mask::$(jq -r '."subject-credentials"."iss"' sa.json)" + service_account_id=$(jq -r '."subject-credentials"."iss"' sa.json) echo "::add-mask::tenant-e00en3r863f7me6wtd" nebius profile create --endpoint api.eu-north1.nebius.cloud \ --profile nbs-github-user-sa \ diff --git a/.github/actions/prepare/action.yaml b/.github/actions/prepare/action.yaml index 1a5ad2d07a0..9be8a50f1da 100644 --- a/.github/actions/prepare/action.yaml +++ b/.github/actions/prepare/action.yaml @@ -19,6 +19,7 @@ runs: distcc strace qemu-kvm qemu-utils dpkg-dev atop pigz pbzip2 xz-utils pixz gdb sudo apt-get remove -y unattended-upgrades sudo pip install https://github.com/librarian/python-sdk/releases/download/v0.1.1/nebiusai-0.1.1-py3-none-any.whl + # shellcheck disable=SC2102 sudo pip3 install conan==1.59 pytest==7.1.3 pytest-timeout pytest-xdist==3.3.1 setproctitle==1.3.2 grpcio grpcio-tools \ PyHamcrest tornado xmltodict pyarrow boto3 moto[server] psutil yandexcloud==0.258.0 PyGithub==2.2.0 pyinstaller==5.13.2 \ cryptography packaging six pyyaml rapidgzip pyOpenSSL==24.2.1 diff --git a/.github/actions/s3cmd/action.yaml b/.github/actions/s3cmd/action.yaml index 21313371608..248e985e53a 100644 --- a/.github/actions/s3cmd/action.yaml +++ b/.github/actions/s3cmd/action.yaml @@ -91,6 +91,7 @@ runs: folder="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}" + # shellcheck disable=SC2195 case "${{ inputs.build_preset }}" in relwithdebinfo|release) ;; @@ -114,7 +115,7 @@ runs: exit 1 ;; esac - + # shellcheck disable=SC2129 echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${GITHUB_WORKFLOW_NO_SPACES}/${{ github.run_id }}/${{ github.run_attempt || '1' }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${GITHUB_WORKFLOW_NO_SPACES}/${{ github.run_id }}/${{ github.run_attempt || '1' }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV echo "S3_WEBSITE_PREFIX=https://${{ inputs.s3_bucket }}.${{ inputs.s3_website_suffix }}/${{ github.repository }}/${GITHUB_WORKFLOW_NO_SPACES}/${{ github.run_id }}/${{ github.run_attempt || '1' }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV diff --git a/.github/actions/ssh_keys/action.yaml b/.github/actions/ssh_keys/action.yaml index 0b42230ed75..2139b987288 100644 --- a/.github/actions/ssh_keys/action.yaml +++ b/.github/actions/ssh_keys/action.yaml @@ -56,11 +56,11 @@ runs: - name: collect members public ssh keys shell: bash run: | - cat $LOGINS_FILE | while read login; do + cat $LOGINS_FILE | while read -r login; do curl -s -L -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $token" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/users/${login}/keys | jq -r .[].key | while read key; do + https://api.github.com/users/${login}/keys | jq -r .[].key | while read -r key; do echo $key $login; done; done | tee -a $KEYS_FILE diff --git a/.github/actions/test/action.yaml b/.github/actions/test/action.yaml index cf5e6d52b25..ffdc94cd5d6 100644 --- a/.github/actions/test/action.yaml +++ b/.github/actions/test/action.yaml @@ -65,15 +65,17 @@ runs: id: init run: | export TMP_DIR=/home/github/tmp - echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV - echo "LOG_DIR=$TMP_DIR/logs" >> $GITHUB_ENV - echo "OUT_DIR=$TMP_DIR/out" >> $GITHUB_ENV - echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" >> $GITHUB_ENV - echo "TESTS_DATA_DIR=$TMP_DIR/test_data" >> $GITHUB_ENV - echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports" >> $GITHUB_ENV - echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml" >> $GITHUB_ENV - echo "JUNIT_REPORT_PARTS=$TMP_DIR/junit-split" >> $GITHUB_ENV - echo "SUMMARY_LINKS=$(mktemp -p /home/github)" >> $GITHUB_ENV + { + echo "TMP_DIR=$TMP_DIR" + echo "LOG_DIR=$TMP_DIR/logs" + echo "OUT_DIR=$TMP_DIR/out" + echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" + echo "TESTS_DATA_DIR=$TMP_DIR/test_data" + echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports" + echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml" + echo "JUNIT_REPORT_PARTS=$TMP_DIR/junit-split" + echo "SUMMARY_LINKS=$(mktemp -p /home/github)" + } >> $GITHUB_ENV - name: prepare shell: bash --noprofile --norc -eo pipefail -x {0} @@ -83,6 +85,7 @@ runs: chown -R github:github $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS \ $REPORTS_ARTIFACTS_DIR $SUMMARY_LINKS $GITHUB_WORKSPACE \ $GITHUB_STEP_SUMMARY $TESTS_DATA_DIR + # shellcheck disable=SC2193 if [ "${{ inputs.clean_ya_dir }}" == "yes" ] && [ -d /home/github/.ya/ ]; then echo "Cleaning ya dir" rm -rf /home/github/.ya/ @@ -93,7 +96,7 @@ runs: set -x extra_params=() - # FIXME: copy-paste from build_ya + # shellcheck disable=SC2195 case "${{ inputs.build_preset }}" in debug) build_type=debug @@ -127,25 +130,29 @@ runs: ;; esac - - if [ ! -z "${{ inputs.test_target }}" ]; then + # shellcheck disable=SC2157 + if [ -n "${{ inputs.test_target }}" ]; then readarray -d ',' -t targets < <(printf "%s" "${{ inputs.test_target }}") for target in "${targets[@]}"; do extra_params+=(--target="${target}") done fi + # shellcheck disable=SC2193 if [ "${{ inputs.use_network_cache }}" == "yes" ]; then - if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then + # shellcheck disable=SC2157 + if [ -n "${{ inputs.bazel_remote_uri }}" ]; then extra_params+=(--bazel-remote-store) extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") fi - if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then + # shellcheck disable=SC2157 + if [ -n "${{ inputs.bazel_remote_username }}" ]; then extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}") fi + # shellcheck disable=SC2193 if [ "${{ inputs.cache_update }}" = "true" ]; then extra_params+=(--bazel-remote-put) fi @@ -167,7 +174,7 @@ runs: date echo "::group::ya-make-test" sudo -E -H -u github ./ya test -k --build "${build_type}" \ - ${test_size[@]/#/--test-size=} ${test_type[@]/#/--test-type=} \ + "${test_size[@]/#/--test-size=}" "${test_type[@]/#/--test-type=}" \ --test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}" \ --cache-size 512G --do-not-output-stderrs -T \ --stat --log-file "$LOG_DIR/ya_log.txt" --evlog-file "$LOG_DIR/ya_evlog.jsonl" \ @@ -200,6 +207,7 @@ runs: - name: archive unitest reports (orig) shell: bash --noprofile --norc -eo pipefail -x {0} run: | + # shellcheck disable=SC2024 sudo -E -H -u github gzip -c $JUNIT_REPORT_XML > $REPORTS_ARTIFACTS_DIR/orig_junit.xml.gz - name: postprocess junit report @@ -209,6 +217,7 @@ runs: cat .github/config/muted_ya.txt cat .github/config/muted_ya_nebius.txt MUTED_CONFIG=".github/config/muted_ya.txt" + # shellcheck disable=SC2193 [ "${{ inputs.nebius }}" == "yes" ] && MUTED_CONFIG=".github/config/muted_ya_nebius.txt" echo "::endgroup::" echo "::group::postprocess-junit" @@ -226,7 +235,7 @@ runs: - name: archive unitest reports (transformed) shell: bash --noprofile --norc -eo pipefail -x {0} run: | - sudo -E -H -u github tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz $(basename $JUNIT_REPORT_PARTS) $JUNIT_REPORT_XML + sudo -E -H -u github tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz "$(basename $JUNIT_REPORT_PARTS)" $JUNIT_REPORT_XML - name: write tests summary shell: bash --noprofile --norc -eo pipefail -x {0} @@ -260,7 +269,7 @@ runs: echo "::group::Copy-failed-tests-data" sudo -E -H -u github .github/scripts/tests/fail-checker.py "$JUNIT_REPORT_XML" --paths-only - sudo -E -H -u github .github/scripts/tests/fail-checker.py "$JUNIT_REPORT_XML" --paths-only | while read path; do + sudo -E -H -u github .github/scripts/tests/fail-checker.py "$JUNIT_REPORT_XML" --paths-only | while read -r path; do echo $path find "${GITHUB_WORKSPACE}/${path}" -print0 | xargs -0 xargs -0 cp -L -r --parents -t "$TESTS_DATA_DIR" done @@ -272,7 +281,7 @@ runs: echo "::endgroup::" echo "::group::remove-images-from-tests-data-dir" find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image - find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image -delete + find "$TESTS_DATA_DIR" \( -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image \) -delete echo "::endgroup::" echo "::group::s3-sync" if [ "$SYNC_TO_S3" = "true" ]; @@ -324,18 +333,18 @@ runs: if: success() && inputs.upload_ya_dir == 'yes' run: | function grep_ya_tc() { - ps aux | grep [y]a-tc || true + ps aux | grep "[y]a-tc" || true } process_name="ya-tc" # by default ya-tc should terminate within 5 minutes timeout=360 start_time=$(date +%s) while true; do - pgrep -x "ya-tc" && { - echo "ya-tc is still running." + pgrep -x "$process_name" && { + echo "$process_name is still running." grep_ya_tc } || { - echo "ya-tc is not running." + echo "$process_name is not running." grep_ya_tc break } @@ -344,9 +353,9 @@ runs: elapsed_time=$((current_time - start_time)) if [ "$elapsed_time" -ge "$timeout" ]; then - echo "Timeout reached. ya-tc is still running. killing it" + echo "Timeout reached. $process_name is still running. killing it" grep_ya_tc - pkill -f ya-tc || true + pkill -f $process_name || true grep_ya_tc break fi diff --git a/.github/actions/test_cmake/action.yaml b/.github/actions/test_cmake/action.yaml index dd92d18b1f1..a3edc2b4460 100644 --- a/.github/actions/test_cmake/action.yaml +++ b/.github/actions/test_cmake/action.yaml @@ -7,10 +7,12 @@ runs: - name: prepare shell: bash --noprofile --norc -eo pipefail -x {0} run: | - echo "SHELLOPTS=$SHELLOPTS:xtrace" >> $GITHUB_ENV - echo "ARTIFACTS_DIR=${TMP_DIR}/artifacts" >> $GITHUB_ENV - echo "SUMMARY_DIR=${ARTIFACTS_DIR}/summary" >> $GITHUB_ENV - echo "SUMMARY_LINKS=$(mktemp)" >> $GITHUB_ENV + { + echo "SHELLOPTS=$SHELLOPTS:xtrace" + echo "ARTIFACTS_DIR=${TMP_DIR}/artifacts" + echo "SUMMARY_DIR=${ARTIFACTS_DIR}/summary" + echo "SUMMARY_LINKS=$(mktemp)" + } >> $GITHUB_ENV - name: create dirs shell: bash --noprofile --norc -eo pipefail -x {0} diff --git a/.github/scripts/shell-extractor.py b/.github/scripts/shell-extractor.py index 6028e222d58..d4ea22baab5 100644 --- a/.github/scripts/shell-extractor.py +++ b/.github/scripts/shell-extractor.py @@ -1,6 +1,5 @@ import os import yaml -import uuid ACTIONS_DIR = ".github/actions" WORKFLOWS_DIR = ".github/workflows" @@ -113,7 +112,7 @@ def write_runs_to_files(runs, output_dir, prefix): - If any block contains '${{ ... }}', insert '# shellcheck disable=SC2296' before that block. """ for i, run_content in enumerate(runs, 1): - file_id = f"{prefix}-{i}-{uuid.uuid4().hex[:8]}.sh" + file_id = f"{prefix}-{i}.sh" filepath = os.path.join(output_dir, file_id) command_blocks = parse_command_blocks(run_content) @@ -123,7 +122,7 @@ def write_runs_to_files(runs, output_dir, prefix): for block in command_blocks: # Check if this block contains GitHub variables if any("${{" in line for line in block): - f.write("# shellcheck disable=SC2296\n") + f.write("# shellcheck disable=SC2296,SC1083\n") for line in block: f.write(line + "\n") diff --git a/.github/workflows/github_actions_scripts.yaml b/.github/workflows/github_actions_scripts.yaml index f16c250c474..b38a41dff2f 100644 --- a/.github/workflows/github_actions_scripts.yaml +++ b/.github/workflows/github_actions_scripts.yaml @@ -44,7 +44,7 @@ jobs: find . -type f -exec md5sum {} + | sort -k2,2 | tee ${GITHUB_WORKSPACE}/.github/scripts/tests/test-data/MD5SUMS cp $TMP_DIR/{fail-checker,summary_env,ya-test.html} ${GITHUB_WORKSPACE}/.github/scripts/tests/test-data/ cd ${GITHUB_WORKSPACE}/.github/scripts/tests/test-data/ - export NEW_VERSION=$(($ACTIONS_TEST_DATA_VERSION + 1)) + export NEW_VERSION=$((ACTIONS_TEST_DATA_VERSION + 1)) tar zcvf cloud-v${NEW_VERSION}.tar.gz cloud/ MD5SUMS fail-checker junit.xml summary_env ya-test.html muted_ya.txt Upload resulting cloud-v${NEW_VERSION}.tar.gz to the storage.ai.nebius.cloud bucket github-actions-test-data @@ -148,7 +148,7 @@ jobs: shell: bash run: | cd $TMP_DIR - grep FAILED RESULT | awk -F: '{print $1}' | while read f; + grep FAILED RESULT | awk -F: '{print $1}' | while read -r f; do echo "==========" echo $f diff --git a/.github/workflows/packer.yaml b/.github/workflows/packer.yaml index b888dff2c99..73da7d20318 100644 --- a/.github/workflows/packer.yaml +++ b/.github/workflows/packer.yaml @@ -162,11 +162,11 @@ jobs: ${sa_json} EOF - cat sa.json | jq -r '."subject-credentials"."private-key"' > private.pem - echo "::add-mask::$(cat sa.json | jq -r '."subject-credentials"."kid"')" - public_key_id=$(cat sa.json | jq -r '."subject-credentials"."kid"') - echo "::add-mask::$(cat sa.json | jq -r '."subject-credentials"."iss"')" - service_account_id=$(cat sa.json | jq -r '."subject-credentials"."iss"') + jq -r '."subject-credentials"."private-key"' sa.json> private.pem + echo "::add-mask::$(jq -r '."subject-credentials"."kid"' sa.json)" + public_key_id=$(jq -r '."subject-credentials"."kid"' sa.json) + echo "::add-mask::$(jq -r '."subject-credentials"."iss"' sa.json)" + service_account_id=$(jq -r '."subject-credentials"."iss" sa.json') echo "::add-mask::tenant-e00en3r863f7me6wtd" nebius profile create --endpoint api.eu-north1.nebius.cloud \ --profile nbs-github-user-sa \ diff --git a/.github/workflows/pr-github-actions.yaml b/.github/workflows/pr-github-actions.yaml index b55e9776097..7c55fb4a2a4 100644 --- a/.github/workflows/pr-github-actions.yaml +++ b/.github/workflows/pr-github-actions.yaml @@ -56,9 +56,14 @@ jobs: uses: ludeeus/action-shellcheck@master with: scandir: .github + env: + SHELL_CHECK_OPTS: "-e -e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" + - name: shellcheck for github actions shell scripts in .temporary dir uses: ludeeus/action-shellcheck@master + env: + SHELL_CHECK_OPTS: "-e -e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" with: scandir: .temporary @@ -130,8 +135,8 @@ jobs: run: | set -x export TMP_OUT=$(mktemp) - find .github/workflows -type f \( -iname \*.yaml -o -iname \*.yml \) \ - | xargs -I {} action-validator --verbose {} > $TMP_OUT + find .github/workflows -type f \( -iname \*.yaml -o -iname \*.yml \) -print0 \ + | xargs -0 -I {} action-validator --verbose {} > $TMP_OUT echo "WORKFLOW_LINT=$(cat $TMP_OUT | awk -v ORS='\\n' 1)" cat $TMP_OUT >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8fd4ac0b152..5be87b7f13c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -203,17 +203,19 @@ jobs: fi # Output to GitHub environment file - echo "build_target=\"$build_target\"" >> $GITHUB_OUTPUT - echo "build_target_asan=\"$build_target_asan\"" >> $GITHUB_OUTPUT - echo "build_target_tsan=\"$build_target_tsan\"" >> $GITHUB_OUTPUT - echo "build_target_msan=\"$build_target_msan\"" >> $GITHUB_OUTPUT - echo "build_target_ubsan=\"$build_target_ubsan\"" >> $GITHUB_OUTPUT + { + echo "build_target=\"$build_target\"" + echo "build_target_asan=\"$build_target_asan\"" + echo "build_target_tsan=\"$build_target_tsan\"" + echo "build_target_msan=\"$build_target_msan\"" + echo "build_target_ubsan=\"$build_target_ubsan\"" - echo "test_target=\"$test_target\"" >> $GITHUB_OUTPUT - echo "test_target_asan=\"$test_target_asan\"" >> $GITHUB_OUTPUT - echo "test_target_tsan=\"$test_target_tsan\"" >> $GITHUB_OUTPUT - echo "test_target_msan=\"$test_target_msan\"" >> $GITHUB_OUTPUT - echo "test_target_ubsan=\"$test_target_ubsan\"" >> $GITHUB_OUTPUT + echo "test_target=\"$test_target\"" + echo "test_target_asan=\"$test_target_asan\"" + echo "test_target_tsan=\"$test_target_tsan\"" + echo "test_target_msan=\"$test_target_msan\"" + echo "test_target_ubsan=\"$test_target_ubsan\"" + } >> $GITHUB_OUTPUT env: contains_blockstore: ${{ contains(github.event.pull_request.labels.*.name, 'blockstore') && 'true' || 'false' }} From acc43c6f89d6ff97fd6a2b3a8db4beecd8caf586 Mon Sep 17 00:00:00 2001 From: Nikita Menkovich Date: Tue, 10 Dec 2024 18:28:42 +0100 Subject: [PATCH 7/8] fix: various shellcheck errors --- .github/workflows/pr-github-actions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-github-actions.yaml b/.github/workflows/pr-github-actions.yaml index 7c55fb4a2a4..fc0d6ab8c9d 100644 --- a/.github/workflows/pr-github-actions.yaml +++ b/.github/workflows/pr-github-actions.yaml @@ -57,13 +57,13 @@ jobs: with: scandir: .github env: - SHELL_CHECK_OPTS: "-e -e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" + SHELL_CHECK_OPTS: "-e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" - name: shellcheck for github actions shell scripts in .temporary dir uses: ludeeus/action-shellcheck@master env: - SHELL_CHECK_OPTS: "-e -e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" + SHELL_CHECK_OPTS: "-e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" with: scandir: .temporary From f3cdaeca0434304e10f82bf0a6e599fc551ce6b7 Mon Sep 17 00:00:00 2001 From: Nikita Menkovich Date: Tue, 10 Dec 2024 18:35:30 +0100 Subject: [PATCH 8/8] fix: various shellcheck errors --- .github/workflows/pr-github-actions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-github-actions.yaml b/.github/workflows/pr-github-actions.yaml index fc0d6ab8c9d..4f22e6ea5b4 100644 --- a/.github/workflows/pr-github-actions.yaml +++ b/.github/workflows/pr-github-actions.yaml @@ -57,13 +57,13 @@ jobs: with: scandir: .github env: - SHELL_CHECK_OPTS: "-e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" + SHELLCHECK_OPTS: "-e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" - name: shellcheck for github actions shell scripts in .temporary dir uses: ludeeus/action-shellcheck@master env: - SHELL_CHECK_OPTS: "-e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" + SHELLCHECK_OPTS: "-e SC2155,SC2086,SC2154,SC2164,SC2009,SC2015" with: scandir: .temporary