Merge remote-tracking branch 'upstream/main' into issues-1733-replay-…

…grpc

.github/actions/nebius_cli/action.yaml

@@ -23,7 +23,7 @@ runs:
     - name: install nebius cli
       shell: bash
       run: |
-        curl -sSL https://storage.ai.nebius.cloud/nebius/install.sh | bash
+        curl -sSL https://storage.eu-north1.nebius.cloud/cli/install.sh | bash
     - name: configure nebius cli
       shell: bash
       run: |

.github/actions/nebius_threads_calculator/action.yaml

@@ -34,19 +34,19 @@ runs:
         if [[ $tests_size == *"large"* ]]; then
           case "$vm_preset" in
             "80vcpu-320gb")
-                test_threads=20
+                test_threads=16
                 build_threads=80
             ;;
             "64vcpu-256gb")
-                test_threads=18
+                test_threads=14
                 build_threads=64
             ;;
             "48vcpu-192gb")
-                test_threads=16
+                test_threads=12
                 build_threads=48
             ;;
             "32vcpu-128gb")
-                test_threads=12
+                test_threads=10
                 build_threads=32
             ;;
             "16vcpu-64gb")
@@ -73,11 +73,11 @@ runs:
         else
           case "$vm_preset" in
             "80vcpu-320gb")
-                test_threads=32
+                test_threads=24
                 build_threads=80
             ;;
             "64vcpu-256gb")
-                test_threads=24
+                test_threads=20
                 build_threads=64
             ;;
             "48vcpu-192gb")

.github/actions/test/action.yaml

@@ -56,6 +56,10 @@ inputs:
     required: false
     default: 'no'
     description: 'Use nebius'
+  truncate_enabled:
+    required: false
+    default: 'yes'
+    description: 'Truncate err files'
 
 runs:
   using: composite
@@ -90,6 +94,9 @@ runs:
           echo "Cleaning ya dir"
           rm -rf /home/github/.ya/
         fi
+        # checking /etc/hosts to see if vm was created correctly
+        cat /etc/hosts
+
     - name: ya test
       shell: bash --noprofile --norc -eo pipefail -x {0}
       run: |
@@ -280,8 +287,21 @@ runs:
           find "$TESTS_DATA_DIR" -type f -print0 | xargs -0 -n 10 file -i  | grep "application/x-executable" | awk -F: '{print $1}' | xargs rm
           echo "::endgroup::"
           echo "::group::remove-images-from-tests-data-dir"
-          find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image
-          find "$TESTS_DATA_DIR" \( -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image \) -delete
+          find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image -o -name NVMENBS01 -o -name generated_other_big_raw_image
+          find "$TESTS_DATA_DIR" \( -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image -o -name NVMENBS01 -o -name generated_other_big_raw_image \) -delete
+          echo "::endgroup::"
+          echo "::group::truncate-err-files"
+          find "$TESTS_DATA_DIR" -type f -name "*.err" -size +1G -print0 | while IFS= read -r -d '' file; do
+              orig_size=$(du -h "$file" | cut -f1)
+              echo "$file - $orig_size"
+              # shellcheck disable=SC2193
+              if [ "${{ inputs.truncate_enabled }}" == "yes" ]; then
+                truncate -s 1G "$file"
+                echo "... truncated (original size was $orig_size) ..." >> "$file"
+              else
+                echo "not truncated"
+              fi
+          done
           echo "::endgroup::"
           echo "::group::s3-sync"
           if [ "$SYNC_TO_S3" = "true" ];

.github/config/muted_ya_nebius.txt

@@ -0,0 +1,2 @@
+cloud/filestore/tests/fio_index/mount-kikimr-test *
+cloud/filestore/tests/fio_index/mount-local-test *

.github/scripts/github-runner.sh

@@ -92,6 +92,9 @@ sudo usermod -a -G docker "${USER_TO_CREATE}"
 sudo echo "${USER_TO_CREATE} ALL=(ALL) NOPASSWD:ALL" | sudo tee "/etc/sudoers.d/99-${USER_TO_CREATE}" > /dev/null
 sudo chmod 0440 "/etc/sudoers.d/99-${USER_TO_CREATE}"
 
+# increase the total number of aio requests to run more tests in parallel, default is 65536
+echo "fs.aio-max-nr=1048576" >> /etc/sysctl.conf
+
 if [ -n "$GITHUB_TOKEN" ] && [ -n "$ORG" ] && [ -n "$TEAM" ]; then
     export LOGINS_FILE
     export KEYS_FILE

.github/scripts/nebius-manage-vm.py

@@ -374,7 +374,7 @@ def wrapper(sdk: SDK, args: argparse.Namespace) -> callable:
                 logger.info("Next run will be at %s", time.ctime(next_run_time))
                 while (
                     time.time() < next_run_time
-                    and time.time() - start_time < total_time_limit
+                    and time.time() - start_time < total_time_limit  # noqa: W503
                 ):
                     time.sleep(1)
             except Exception as e:

.github/scripts/shell-extractor.py

@@ -106,6 +106,8 @@ def parse_command_blocks(run_content):
 
 def write_runs_to_files(runs, output_dir, prefix):
     """
+    Write run commands to files.
+
     Write each run command to a .sh file with template {action name}-{index inside of action file}
     in the given output_dir.
 

.github/scripts/tests/generate-summary.py

@@ -2,7 +2,6 @@
 import argparse
 import dataclasses
 import os
-import re
 import json
 import sys
 from github import Github, Auth as GithubAuth
@@ -360,20 +359,21 @@ def update_pr_comment(
     test_history_url: str,
     is_dry_run: bool,
 ):
-    header = f"<!-- status pr={pr.number}, run={{}} -->"
-    header_re = re.compile(header.format(r"(\d+)"))
+    header = f"<!-- status pr={pr.number}, run={run_number}, build_preset={build_preset}, dry_run={is_dry_run} -->"
 
     body = None
+    comment = None
 
     for c in pr.get_issue_comments():
-        if matches := header_re.match(c.body):
-            if int(matches[1]) == run_number:
-                body = [c.body, "", "---", ""]
+        if c.body.startswith(header):
+            print(f"Found comment with id={c.id}")
+            comment = c
+            body = [c.body]
+            break
 
     if body is None:
-        body = [
-            header.format(run_number),
-        ]
+        body = [header]
+
         if is_dry_run:
             body.extend(
                 [
@@ -389,12 +389,19 @@ def update_pr_comment(
                 "",
             ]
         )
+    else:
+        body.extend(["", ""])
 
     body.extend(get_comment_text(pr, summary, build_preset, test_history_url))
 
     body = "\n".join(body)
 
-    pr.create_issue_comment(body)
+    if comment is None:
+        print("Creating new comment")
+        pr.create_issue_comment(body)
+    else:
+        print("Updating existing comment")
+        comment.edit(body)
 
 
 def main():

.github/workflows/approvals.yaml

@@ -36,7 +36,7 @@ jobs:
                 console.log("Left team members: " + leftUsers.join(', '));
                 console.log("Right team members: " + rightUsers.join(', '));
                 // users that can approve PRs without any restrictions by commenting /approve
-                const superUsers = ['qkrorlqr', 'EvgeniyKozev'];
+                const superUsers = ['SvartMetal', 'EvgeniyKozev'];
                 const prNumber = context.issue.number;
                 const owner = context.repo.owner;
                 const repo = context.repo.repo;

.github/workflows/build_and_test_on_demand.yaml

@@ -253,6 +253,7 @@ jobs:
       clean_ya_dir: ${{ github.event == 'workflow_dispatch' && 'no' || inputs.clean_ya_dir }}
       use_network_cache: ${{ github.event == 'workflow_dispatch' && 'yes'|| inputs.use_network_cache }}
       nebius: ${{ needs.provide-runner.outputs.nebius }}
+      truncate_enabled: ${{ contains(github.event.pull_request.labels.*.name, 'disable_truncate') && 'no' || 'yes' }}
     secrets: inherit
 
   sleep-if-needed:

.github/workflows/build_and_test_ya.yaml

@@ -86,6 +86,10 @@ on:
         type: string
         default: "no"
         description: "Run on nebius runners"
+      truncate_enabled:
+        type: string
+        default: "yes"
+        description: "Truncate enabled"
     outputs:
       sleep_after_tests:
         description: "sleep_after_tests"
@@ -166,7 +170,7 @@ jobs:
         clean_ya_dir: ${{ inputs.run_build && 'no' || inputs.clean_ya_dir }}
         use_network_cache: ${{ inputs.use_network_cache }}
         nebius: ${{ inputs.nebius }}
-
+        truncate_enabled: ${{ inputs.truncate_enabled }}
     - id: failure
       name: set sleep_after_tests in case of failure
       if: failure()

.github/workflows/create_and_delete_vm.yaml

@@ -179,6 +179,7 @@ jobs:
       clean_ya_dir: "no"
       use_network_cache: "yes"
       nebius: ${{ needs.provide-runner.outputs.nebius }}
+      truncate_enabled: ${{ contains(github.event.pull_request.labels.*.name, 'disable_truncate') && 'no' || 'yes' }}
     secrets: inherit
 
   release-runner:

.github/workflows/nightly-index-rebuild.yaml

@@ -3,14 +3,6 @@ on:
   schedule:
     - cron: "0 0 * * *"
   workflow_dispatch:
-    inputs:
-      nebius:
-        type: choice
-        default: "yes"
-        options:
-          - "yes"
-          - "no"
-        description: "Run on nebius infrastructure"
   workflow_call:
 
 jobs:
@@ -28,10 +20,10 @@ jobs:
       uses: ./.github/actions/s3cmd
       with:
         s3_bucket: ${{ vars.AWS_BUCKET }}
-        s3_endpoint: ${{ inputs.nebius == 'yes' && vars.NEBIUS_AWS_ENDPOINT || vars.AWS_ENDPOINT }}
-        s3_website_suffix: ${{ inputs.nebius == 'yes' && vars.NEBIUS_AWS_WEBSITE_SUFFIX || vars.AWS_WEBSITE_SUFFIX }}
-        s3_key_id: ${{ inputs.nebius == 'yes' && secrets.NEBIUS_AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}
-        s3_key_secret: ${{ inputs.nebius == 'yes' && secrets.NEBIUS_AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}
+        s3_endpoint: ${{ vars.GLOBAL_RUN_ON_NEBIUS == 'yes' && vars.NEBIUS_AWS_ENDPOINT || vars.AWS_ENDPOINT }}
+        s3_website_suffix: ${{ vars.GLOBAL_RUN_ON_NEBIUS == 'yes' && vars.NEBIUS_AWS_WEBSITE_SUFFIX || vars.AWS_WEBSITE_SUFFIX }}
+        s3_key_id: ${{ vars.GLOBAL_RUN_ON_NEBIUS == 'yes' && secrets.NEBIUS_AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}
+        s3_key_secret: ${{ vars.GLOBAL_RUN_ON_NEBIUS == 'yes' && secrets.NEBIUS_AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}
         folder_prefix: nebius-
         build_preset: "release"
         user: runner

.github/workflows/nightly.yaml

@@ -105,6 +105,6 @@ jobs:
       test_threads: 32
       disk_type: 'network-ssd-nonreplicated'
       use_network_cache: "yes"
-      upload_ya_dir: "yes"
+      upload_ya_dir: "no"
       clean_ya_dir: "yes"
       nebius: ${{ vars.GLOBAL_RUN_ON_NEBIUS }}

.github/workflows/packer.yaml

@@ -157,16 +157,16 @@ jobs:
     - name: Set up Nebius CLI
       if: inputs.nebius == 'yes'
       run: |
-        curl -sSL https://storage.ai.nebius.cloud/nebius/install.sh | bash
+        curl -sSL https://storage.eu-north1.nebius.cloud/cli/install.sh | bash
         cat <<EOF > sa.json
         ${sa_json}
         EOF
 
-        jq -r '."subject-credentials"."private-key"'  sa.json> private.pem
+        jq -r '."subject-credentials"."private-key"'  sa.json > private.pem
         echo "::add-mask::$(jq -r '."subject-credentials"."kid"' sa.json)"
         public_key_id=$(jq -r '."subject-credentials"."kid"' sa.json)
         echo "::add-mask::$(jq -r '."subject-credentials"."iss"' sa.json)"
-        service_account_id=$(jq -r '."subject-credentials"."iss" sa.json')
+        service_account_id=$(jq -r '."subject-credentials"."iss"' sa.json)
         echo "::add-mask::tenant-e00en3r863f7me6wtd"
         nebius profile create --endpoint api.eu-north1.nebius.cloud \
                               --profile nbs-github-user-sa \

CLANG-FORMAT.md

@@ -12,6 +12,6 @@ For other development environments, you should install clang-format-18 manually.
 sudo apt-get install clang-format-18
 ```
 
-Configure the plugin for your favorite code editor to use the [config](.clang-format) to format the code. For VS Code, for example, you can use https://marketplace.visualstudio.com/items?itemName=xaver.clang-format
+Configure the plugin for your favorite code editor to use the [config](.clang-format) to format the code.
 
 Note. clang-format searches for the .clang-format configuration file in the directory of the formatted file and in all parent directories. Therefore, no additional settings are required.

GITHUB.md

@@ -10,6 +10,7 @@ There is also a list of labels that slightly alters how and which tests are run:
 4. `asan` or `tsan` to add address sanitizer or thread sanitizer builds on top of the regular build.
 5. `recheck` trigger checks without commit. Removed automatically after launch.
 6. `allow-downgrade` allows to downgrade VM preset dynamically in case of problems with resources
+7. `disable_truncate`, by default, .err files are truncated to 1GiB, this disables that for this PR
 
 Also, you can launch [ya make](https://github.com/ydb-platform/nbs/actions/workflows/build_and_test_on_demand.yaml) or [cmake](https://github.com/ydb-platform/nbs/actions/workflows/build_and_test_on_demand_cmake.yaml) builds on your branch with any timeout you want (but please do not do more than 12 hours, VMs are expensive). You can find the IP of the VM inside of the jobs. The first occurrence happens in the `Prepare runner` job in the `Configure NCP` step, later IP is set in the header of jobs. You must use your GitHub key for it. User `github`. Feature not available for non-members.
 

VSCODE.md

@@ -33,3 +33,16 @@ Enable git hooks for pre-commit checks
 ```
 git config core.hooksPath .githooks
 ```
+
+### Debugging
+
+If you want to use debugging in VS Code you should to enable the static linkage.
+
+Add section below to '~/.ya/ya.conf'
+[[target_platform]]
+platform_name = "default-linux-x86_64"
+build_type = "relwithdebinfo"
+#build_type = "release"
+
+[target_platform.flags]
+FORCE_STATIC_LINKING="yes"

cloud/blockstore/apps/server/ya.make

@@ -24,7 +24,7 @@ PEERDIR(
     library/cpp/getopt
 )
 
-IF (BUILD_TYPE != "PROFILE" AND BUILD_TYPE != "DEBUG")
+IF (BUILD_TYPE != "PROFILE" AND BUILD_TYPE != "DEBUG" AND BUILD_TYPE != "RELWITHDEBINFO")
     SPLIT_DWARF()
 ENDIF()
 

cloud/blockstore/config/disk.proto

@@ -150,6 +150,22 @@ message TStorageDiscoveryConfig
 
 ////////////////////////////////////////////////////////////////////////////////
 
+message TDiskAgentThrottlingConfig
+{
+    // Host limits.
+    optional string InfraThrottlingConfigPath = 1;
+    optional uint32 DefaultNetworkMbitThroughput = 2;
+
+    // Fraction of network throughput utilized for migrations and shadow
+    // disk fill.
+    optional double DirectCopyBandwidthFraction = 3;
+
+    // Maximum bandwidth for one device in MiB/s.
+    optional uint64 MaxDeviceBandwidthMiB = 4;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 message TDiskAgentConfig
 {
     optional bool Enabled = 1;
@@ -275,6 +291,13 @@ message TDiskAgentConfig
     }
 
     repeated TPathToSerialNumber PathToSerialNumberMapping = 37;
+
+    // Settings for traffic shaping.
+    optional TDiskAgentThrottlingConfig ThrottlingConfig = 38;
+
+    // If enabled, IOParserActor allocates a storage buffer and copies the
+    // request data into it.
+    optional bool IOParserActorAllocateStorageEnabled = 39;
 }
 
 ////////////////////////////////////////////////////////////////////////////////