Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add shell extractor script that will extract contents of run sections in actions #2668

Closed
wants to merge 8 commits into from
Closed

feat: add shell extractor script that will extract contents of run sections in actions #2668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
aa8691a
feat: add shell extractor script that will extract contents of run se…
librarian Dec 10, 2024
8f66f6d
black
librarian Dec 10, 2024
26501a2
fix
librarian Dec 10, 2024
b479668
fix shfmt issue with extracted files
librarian Dec 10, 2024
5ff7979
remove reviewdog shellcheck, replace with another github action
librarian Dec 10, 2024
1a7c08f
fix: various shellcheck errors
librarian Dec 10, 2024
acc43c6
fix: various shellcheck errors
librarian Dec 10, 2024
f3cdaec
fix: various shellcheck errors
librarian Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions .github/actions/build/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ runs:
echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
rm -rf $TMP_DIR && mkdir $TMP_DIR && chown -R github:github $TMP_DIR $GITHUB_WORKSPACE

# shellcheck disable=SC2193
if [ "${{ inputs.clean_ya_dir }}" == "yes" ] && [ -d /home/github/.ya/ ]; then
echo "Cleaning ya dir"
rm -rf /home/github/.ya/
Expand All @@ -55,35 +56,40 @@ runs:
shell: bash --noprofile --norc -eo pipefail -x {0}
run: |
function grep_ya_tc() {
ps aux | grep [y]a-tc || true
ps aux | grep "[y]a-tc" || true
}

extra_params=()

if [ ! -z "${{ inputs.build_target }}" ]; then
# shellcheck disable=SC2157
if [ -n "${{ inputs.build_target }}" ]; then
readarray -d ',' -t targets < <(printf "%s" "${{ inputs.build_target }}")
for target in "${targets[@]}"; do
extra_params+=(--target="${target}")
done
fi

# shellcheck disable=SC2157,SC2193
if [ "${{ inputs.use_network_cache }}" == "yes" ]; then
if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then
if [ -n "${{ inputs.bazel_remote_uri }}" ]; then
extra_params+=(--bazel-remote-store)
extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
fi

if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then
# shellcheck disable=SC2157
if [ -n "${{ inputs.bazel_remote_username }}" ]; then
extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}")
extra_params+=(--add-result .o)
fi

# shellcheck disable=SC2193
if [ "${{ inputs.cache_update }}" == "true" ]; then
extra_params+=(--bazel-remote-put)
fi
fi

# shellcheck disable=SC2195
case "${{ inputs.build_preset }}" in
debug)
build_type=debug
Expand Down Expand Up @@ -132,11 +138,11 @@ runs:
start_time=$(date +%s)

while true; do
pgrep -x "ya-tc" && {
echo "ya-tc is still running."
pgrep -x "$process_name" && {
echo "$process_name is still running."
grep_ya_tc
} || {
echo "ya-tc is not running."
echo "$process_name is not running."
grep_ya_tc
break
}
Expand All @@ -145,9 +151,9 @@ runs:
elapsed_time=$((current_time - start_time))

if [ "$elapsed_time" -ge "$timeout" ]; then
echo "Timeout reached. ya-tc is still running. killing it"
echo "Timeout reached. $process_name is still running. killing it"
grep_ya_tc
pkill -f ya-tc || true
pkill -f $process_name || true
grep_ya_tc
break
fi
Expand Down
25 changes: 14 additions & 11 deletions .github/actions/build_cmake/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,27 @@ runs:
run: |
export TMP_DIR=$(pwd)/tmp_build
export ROOT_PATH=$(pwd)
echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
{
echo "TMP_DIR=$TMP_DIR"
echo "ROOT_PATH=$ROOT_PATH"
echo "CONAN_USER_HOME=$TMP_DIR"
echo "CCACHE_SLOPPINESS=locale"
echo "CCACHE_BASEDIR=$TMP_DIR"
echo "CCACHE_MAXSIZE=500G"
echo "CCACHE_NOREADONLY=1"
echo "CCACHE_REMOTE_ONLY=1"
echo "CCACHE_DEBUG=1"
echo "CCACHE_REMOTE_STORAGE=http://${{ inputs.bazel_remote_username }}:${{ inputs.bazel_remote_password }}@195.242.17.155:9090|layout=bazel"
} >> $GITHUB_ENV
rm -rf $TMP_DIR && mkdir $TMP_DIR
echo "ROOT_PATH=$ROOT_PATH" >> $GITHUB_ENV
echo "CONAN_USER_HOME=$TMP_DIR" >> $GITHUB_ENV
echo "CCACHE_SLOPPINESS=locale" >> $GITHUB_ENV
echo "CCACHE_BASEDIR=$TMP_DIR" >> $GITHUB_ENV
echo "CCACHE_MAXSIZE=500G" >> $GITHUB_ENV
echo "CCACHE_NOREADONLY=1" >> $GITHUB_ENV
echo "CCACHE_REMOTE_ONLY=1" >> $GITHUB_ENV
echo "CCACHE_DEBUG=1" >> $GITHUB_ENV
echo "CCACHE_REMOTE_STORAGE=http://${{ inputs.bazel_remote_username }}:${{ inputs.bazel_remote_password }}@195.242.17.155:9090|layout=bazel" >> $GITHUB_ENV

- name: build
shell: bash --noprofile --norc -eo pipefail -x {0}
run: |
cd $TMP_DIR
export

# shellcheck disable=SC2195
case "${{ inputs.build_preset }}" in
debug)
export CMAKE_BUILD_TYPE=Debug
Expand All @@ -57,7 +60,7 @@ runs:
exit 1
;;
esac
if [ $(git branch --show-current) == "main" ]; then
if [ "$(git branch --show-current)" == "main" ]; then
cp ${ROOT_PATH}/contrib/ydb/library/yql/minikql/codegen/codegen_llvm_deps.h.txt ${ROOT_PATH}/contrib/ydb/library/yql/minikql/codegen/codegen_llvm_deps.h
fi

Expand Down
10 changes: 5 additions & 5 deletions .github/actions/nebius_cli/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ runs:
${sa_json}
EOF

cat sa.json | jq -r '."subject-credentials"."private-key"' > private.pem
echo "::add-mask::$(cat sa.json | jq -r '."subject-credentials"."kid"')"
public_key_id=$(cat sa.json | jq -r '."subject-credentials"."kid"')
echo "::add-mask::$(cat sa.json | jq -r '."subject-credentials"."iss"')"
service_account_id=$(cat sa.json | jq -r '."subject-credentials"."iss"')
jq -r '."subject-credentials"."private-key"' sa.json > private.pem
echo "::add-mask::$(jq -r '."subject-credentials"."kid"' sa.json)"
public_key_id=$(jq -r '."subject-credentials"."kid"' sa.json)
echo "::add-mask::$(jq -r '."subject-credentials"."iss"' sa.json)"
service_account_id=$(jq -r '."subject-credentials"."iss"' sa.json)
echo "::add-mask::tenant-e00en3r863f7me6wtd"
nebius profile create --endpoint api.eu-north1.nebius.cloud \
--profile nbs-github-user-sa \
Expand Down
1 change: 1 addition & 0 deletions .github/actions/prepare/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ runs:
distcc strace qemu-kvm qemu-utils dpkg-dev atop pigz pbzip2 xz-utils pixz gdb
sudo apt-get remove -y unattended-upgrades
sudo pip install https://github.com/librarian/python-sdk/releases/download/v0.1.1/nebiusai-0.1.1-py3-none-any.whl
# shellcheck disable=SC2102
sudo pip3 install conan==1.59 pytest==7.1.3 pytest-timeout pytest-xdist==3.3.1 setproctitle==1.3.2 grpcio grpcio-tools \
PyHamcrest tornado xmltodict pyarrow boto3 moto[server] psutil yandexcloud==0.258.0 PyGithub==2.2.0 pyinstaller==5.13.2 \
cryptography packaging six pyyaml rapidgzip pyOpenSSL==24.2.1
Expand Down
3 changes: 2 additions & 1 deletion .github/actions/s3cmd/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ runs:

folder="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"

# shellcheck disable=SC2195
case "${{ inputs.build_preset }}" in
relwithdebinfo|release)
;;
Expand All @@ -114,7 +115,7 @@ runs:
exit 1
;;
esac

# shellcheck disable=SC2129
echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${GITHUB_WORKFLOW_NO_SPACES}/${{ github.run_id }}/${{ github.run_attempt || '1' }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${GITHUB_WORKFLOW_NO_SPACES}/${{ github.run_id }}/${{ github.run_attempt || '1' }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
echo "S3_WEBSITE_PREFIX=https://${{ inputs.s3_bucket }}.${{ inputs.s3_website_suffix }}/${{ github.repository }}/${GITHUB_WORKFLOW_NO_SPACES}/${{ github.run_id }}/${{ github.run_attempt || '1' }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/ssh_keys/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,11 @@ runs:
- name: collect members public ssh keys
shell: bash
run: |
cat $LOGINS_FILE | while read login; do
cat $LOGINS_FILE | while read -r login; do
curl -s -L -H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $token" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/users/${login}/keys | jq -r .[].key | while read key; do
https://api.github.com/users/${login}/keys | jq -r .[].key | while read -r key; do
echo $key $login;
done;
done | tee -a $KEYS_FILE
Expand Down
57 changes: 33 additions & 24 deletions .github/actions/test/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,17 @@ runs:
id: init
run: |
export TMP_DIR=/home/github/tmp
echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
echo "LOG_DIR=$TMP_DIR/logs" >> $GITHUB_ENV
echo "OUT_DIR=$TMP_DIR/out" >> $GITHUB_ENV
echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" >> $GITHUB_ENV
echo "TESTS_DATA_DIR=$TMP_DIR/test_data" >> $GITHUB_ENV
echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports" >> $GITHUB_ENV
echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml" >> $GITHUB_ENV
echo "JUNIT_REPORT_PARTS=$TMP_DIR/junit-split" >> $GITHUB_ENV
echo "SUMMARY_LINKS=$(mktemp -p /home/github)" >> $GITHUB_ENV
{
echo "TMP_DIR=$TMP_DIR"
echo "LOG_DIR=$TMP_DIR/logs"
echo "OUT_DIR=$TMP_DIR/out"
echo "ARTIFACTS_DIR=$TMP_DIR/artifacts"
echo "TESTS_DATA_DIR=$TMP_DIR/test_data"
echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports"
echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml"
echo "JUNIT_REPORT_PARTS=$TMP_DIR/junit-split"
echo "SUMMARY_LINKS=$(mktemp -p /home/github)"
} >> $GITHUB_ENV

- name: prepare
shell: bash --noprofile --norc -eo pipefail -x {0}
Expand All @@ -83,6 +85,7 @@ runs:
chown -R github:github $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS \
$REPORTS_ARTIFACTS_DIR $SUMMARY_LINKS $GITHUB_WORKSPACE \
$GITHUB_STEP_SUMMARY $TESTS_DATA_DIR
# shellcheck disable=SC2193
if [ "${{ inputs.clean_ya_dir }}" == "yes" ] && [ -d /home/github/.ya/ ]; then
echo "Cleaning ya dir"
rm -rf /home/github/.ya/
Expand All @@ -93,7 +96,7 @@ runs:
set -x
extra_params=()

# FIXME: copy-paste from build_ya
# shellcheck disable=SC2195
case "${{ inputs.build_preset }}" in
debug)
build_type=debug
Expand Down Expand Up @@ -127,25 +130,29 @@ runs:
;;
esac


if [ ! -z "${{ inputs.test_target }}" ]; then
# shellcheck disable=SC2157
if [ -n "${{ inputs.test_target }}" ]; then
readarray -d ',' -t targets < <(printf "%s" "${{ inputs.test_target }}")
for target in "${targets[@]}"; do
extra_params+=(--target="${target}")
done
fi

# shellcheck disable=SC2193
if [ "${{ inputs.use_network_cache }}" == "yes" ]; then
if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then
# shellcheck disable=SC2157
if [ -n "${{ inputs.bazel_remote_uri }}" ]; then
extra_params+=(--bazel-remote-store)
extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
fi

if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then
# shellcheck disable=SC2157
if [ -n "${{ inputs.bazel_remote_username }}" ]; then
extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}")
fi

# shellcheck disable=SC2193
if [ "${{ inputs.cache_update }}" = "true" ]; then
extra_params+=(--bazel-remote-put)
fi
Expand All @@ -167,7 +174,7 @@ runs:
date
echo "::group::ya-make-test"
sudo -E -H -u github ./ya test -k --build "${build_type}" \
${test_size[@]/#/--test-size=} ${test_type[@]/#/--test-type=} \
"${test_size[@]/#/--test-size=}" "${test_type[@]/#/--test-type=}" \
--test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}" \
--cache-size 512G --do-not-output-stderrs -T \
--stat --log-file "$LOG_DIR/ya_log.txt" --evlog-file "$LOG_DIR/ya_evlog.jsonl" \
Expand Down Expand Up @@ -200,6 +207,7 @@ runs:
- name: archive unitest reports (orig)
shell: bash --noprofile --norc -eo pipefail -x {0}
run: |
# shellcheck disable=SC2024
sudo -E -H -u github gzip -c $JUNIT_REPORT_XML > $REPORTS_ARTIFACTS_DIR/orig_junit.xml.gz

- name: postprocess junit report
Expand All @@ -209,6 +217,7 @@ runs:
cat .github/config/muted_ya.txt
cat .github/config/muted_ya_nebius.txt
MUTED_CONFIG=".github/config/muted_ya.txt"
# shellcheck disable=SC2193
[ "${{ inputs.nebius }}" == "yes" ] && MUTED_CONFIG=".github/config/muted_ya_nebius.txt"
echo "::endgroup::"
echo "::group::postprocess-junit"
Expand All @@ -226,7 +235,7 @@ runs:
- name: archive unitest reports (transformed)
shell: bash --noprofile --norc -eo pipefail -x {0}
run: |
sudo -E -H -u github tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz $(basename $JUNIT_REPORT_PARTS) $JUNIT_REPORT_XML
sudo -E -H -u github tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz "$(basename $JUNIT_REPORT_PARTS)" $JUNIT_REPORT_XML

- name: write tests summary
shell: bash --noprofile --norc -eo pipefail -x {0}
Expand Down Expand Up @@ -260,7 +269,7 @@ runs:

echo "::group::Copy-failed-tests-data"
sudo -E -H -u github .github/scripts/tests/fail-checker.py "$JUNIT_REPORT_XML" --paths-only
sudo -E -H -u github .github/scripts/tests/fail-checker.py "$JUNIT_REPORT_XML" --paths-only | while read path; do
sudo -E -H -u github .github/scripts/tests/fail-checker.py "$JUNIT_REPORT_XML" --paths-only | while read -r path; do
echo $path
find "${GITHUB_WORKSPACE}/${path}" -print0 | xargs -0 xargs -0 cp -L -r --parents -t "$TESTS_DATA_DIR"
done
Expand All @@ -272,7 +281,7 @@ runs:
echo "::endgroup::"
echo "::group::remove-images-from-tests-data-dir"
find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image
find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image -delete
find "$TESTS_DATA_DIR" \( -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image \) -delete
echo "::endgroup::"
echo "::group::s3-sync"
if [ "$SYNC_TO_S3" = "true" ];
Expand Down Expand Up @@ -324,18 +333,18 @@ runs:
if: success() && inputs.upload_ya_dir == 'yes'
run: |
function grep_ya_tc() {
ps aux | grep [y]a-tc || true
ps aux | grep "[y]a-tc" || true
}
process_name="ya-tc"
# by default ya-tc should terminate within 5 minutes
timeout=360
start_time=$(date +%s)
while true; do
pgrep -x "ya-tc" && {
echo "ya-tc is still running."
pgrep -x "$process_name" && {
echo "$process_name is still running."
grep_ya_tc
} || {
echo "ya-tc is not running."
echo "$process_name is not running."
grep_ya_tc
break
}
Expand All @@ -344,9 +353,9 @@ runs:
elapsed_time=$((current_time - start_time))

if [ "$elapsed_time" -ge "$timeout" ]; then
echo "Timeout reached. ya-tc is still running. killing it"
echo "Timeout reached. $process_name is still running. killing it"
grep_ya_tc
pkill -f ya-tc || true
pkill -f $process_name || true
grep_ya_tc
break
fi
Expand Down
10 changes: 6 additions & 4 deletions .github/actions/test_cmake/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ runs:
- name: prepare
shell: bash --noprofile --norc -eo pipefail -x {0}
run: |
echo "SHELLOPTS=$SHELLOPTS:xtrace" >> $GITHUB_ENV
echo "ARTIFACTS_DIR=${TMP_DIR}/artifacts" >> $GITHUB_ENV
echo "SUMMARY_DIR=${ARTIFACTS_DIR}/summary" >> $GITHUB_ENV
echo "SUMMARY_LINKS=$(mktemp)" >> $GITHUB_ENV
{
echo "SHELLOPTS=$SHELLOPTS:xtrace"
echo "ARTIFACTS_DIR=${TMP_DIR}/artifacts"
echo "SUMMARY_DIR=${ARTIFACTS_DIR}/summary"
echo "SUMMARY_LINKS=$(mktemp)"
} >> $GITHUB_ENV

- name: create dirs
shell: bash --noprofile --norc -eo pipefail -x {0}
Expand Down
Loading
Loading