Merge branch 'main' into user/rattysed/issue-2008

.github/actions/nebius_cli/action.yaml

@@ -23,7 +23,7 @@ runs:
     - name: install nebius cli
       shell: bash
       run: |
-        curl -sSL https://storage.ai.nebius.cloud/nebius/install.sh | bash
+        curl -sSL https://storage.eu-north1.nebius.cloud/cli/install.sh | bash
     - name: configure nebius cli
       shell: bash
       run: |

.github/actions/nebius_threads_calculator/action.yaml

@@ -34,19 +34,19 @@ runs:
         if [[ $tests_size == *"large"* ]]; then
           case "$vm_preset" in
             "80vcpu-320gb")
-                test_threads=20
+                test_threads=16
                 build_threads=80
             ;;
             "64vcpu-256gb")
-                test_threads=18
+                test_threads=14
                 build_threads=64
             ;;
             "48vcpu-192gb")
-                test_threads=16
+                test_threads=12
                 build_threads=48
             ;;
             "32vcpu-128gb")
-                test_threads=12
+                test_threads=10
                 build_threads=32
             ;;
             "16vcpu-64gb")
@@ -73,11 +73,11 @@ runs:
         else
           case "$vm_preset" in
             "80vcpu-320gb")
-                test_threads=32
+                test_threads=24
                 build_threads=80
             ;;
             "64vcpu-256gb")
-                test_threads=24
+                test_threads=20
                 build_threads=64
             ;;
             "48vcpu-192gb")

.github/actions/test/action.yaml

@@ -56,6 +56,10 @@ inputs:
     required: false
     default: 'no'
     description: 'Use nebius'
+  truncate_enabled:
+    required: false
+    default: 'yes'
+    description: 'Truncate err files'
 
 runs:
   using: composite
@@ -90,6 +94,9 @@ runs:
           echo "Cleaning ya dir"
           rm -rf /home/github/.ya/
         fi
+        # checking /etc/hosts to see if vm was created correctly
+        cat /etc/hosts
+
     - name: ya test
       shell: bash --noprofile --norc -eo pipefail -x {0}
       run: |
@@ -280,8 +287,21 @@ runs:
           find "$TESTS_DATA_DIR" -type f -print0 | xargs -0 -n 10 file -i  | grep "application/x-executable" | awk -F: '{print $1}' | xargs rm
           echo "::endgroup::"
           echo "::group::remove-images-from-tests-data-dir"
-          find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image
-          find "$TESTS_DATA_DIR" \( -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image \) -delete
+          find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image -o -name NVMENBS01 -o -name generated_other_big_raw_image
+          find "$TESTS_DATA_DIR" \( -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image -o -name NVMENBS01 -o -name generated_other_big_raw_image \) -delete
+          echo "::endgroup::"
+          echo "::group::truncate-err-files"
+          find "$TESTS_DATA_DIR" -type f -name "*.err" -size +1G -print0 | while IFS= read -r -d '' file; do
+              orig_size=$(du -h "$file" | cut -f1)
+              echo "$file - $orig_size"
+              # shellcheck disable=SC2193
+              if [ "${{ inputs.truncate_enabled }}" == "yes" ]; then
+                truncate -s 1G "$file"
+                echo "... truncated (original size was $orig_size) ..." >> "$file"
+              else
+                echo "not truncated"
+              fi
+          done
           echo "::endgroup::"
           echo "::group::s3-sync"
           if [ "$SYNC_TO_S3" = "true" ];

.github/config/muted_ya_nebius.txt

@@ -0,0 +1,2 @@
+cloud/filestore/tests/fio_index/mount-kikimr-test *
+cloud/filestore/tests/fio_index/mount-local-test *

.github/scripts/github-runner.sh

@@ -92,6 +92,9 @@ sudo usermod -a -G docker "${USER_TO_CREATE}"
 sudo echo "${USER_TO_CREATE} ALL=(ALL) NOPASSWD:ALL" | sudo tee "/etc/sudoers.d/99-${USER_TO_CREATE}" > /dev/null
 sudo chmod 0440 "/etc/sudoers.d/99-${USER_TO_CREATE}"
 
+# increase the total number of aio requests to run more tests in parallel, default is 65536
+echo "fs.aio-max-nr=1048576" >> /etc/sysctl.conf
+
 if [ -n "$GITHUB_TOKEN" ] && [ -n "$ORG" ] && [ -n "$TEAM" ]; then
     export LOGINS_FILE
     export KEYS_FILE

.github/scripts/nebius-manage-vm.py

@@ -374,7 +374,7 @@ def wrapper(sdk: SDK, args: argparse.Namespace) -> callable:
                 logger.info("Next run will be at %s", time.ctime(next_run_time))
                 while (
                     time.time() < next_run_time
-                    and time.time() - start_time < total_time_limit
+                    and time.time() - start_time < total_time_limit  # noqa: W503
                 ):
                     time.sleep(1)
             except Exception as e:

.github/scripts/shell-extractor.py

@@ -106,6 +106,8 @@ def parse_command_blocks(run_content):
 
 def write_runs_to_files(runs, output_dir, prefix):
     """
+    Write run commands to files.
+
     Write each run command to a .sh file with template {action name}-{index inside of action file}
     in the given output_dir.
 

.github/scripts/tests/generate-summary.py

@@ -2,7 +2,6 @@
 import argparse
 import dataclasses
 import os
-import re
 import json
 import sys
 from github import Github, Auth as GithubAuth
@@ -360,20 +359,21 @@ def update_pr_comment(
     test_history_url: str,
     is_dry_run: bool,
 ):
-    header = f"<!-- status pr={pr.number}, run={{}} -->"
-    header_re = re.compile(header.format(r"(\d+)"))
+    header = f"<!-- status pr={pr.number}, run={run_number}, build_preset={build_preset}, dry_run={is_dry_run} -->"
 
     body = None
+    comment = None
 
     for c in pr.get_issue_comments():
-        if matches := header_re.match(c.body):
-            if int(matches[1]) == run_number:
-                body = [c.body, "", "---", ""]
+        if c.body.startswith(header):
+            print(f"Found comment with id={c.id}")
+            comment = c
+            body = [c.body]
+            break
 
     if body is None:
-        body = [
-            header.format(run_number),
-        ]
+        body = [header]
+
         if is_dry_run:
             body.extend(
                 [
@@ -389,12 +389,19 @@ def update_pr_comment(
                 "",
             ]
         )
+    else:
+        body.extend(["", ""])
 
     body.extend(get_comment_text(pr, summary, build_preset, test_history_url))
 
     body = "\n".join(body)
 
-    pr.create_issue_comment(body)
+    if comment is None:
+        print("Creating new comment")
+        pr.create_issue_comment(body)
+    else:
+        print("Updating existing comment")
+        comment.edit(body)
 
 
 def main():

.github/workflows/build_and_test_on_demand.yaml

@@ -253,6 +253,7 @@ jobs:
       clean_ya_dir: ${{ github.event == 'workflow_dispatch' && 'no' || inputs.clean_ya_dir }}
       use_network_cache: ${{ github.event == 'workflow_dispatch' && 'yes'|| inputs.use_network_cache }}
       nebius: ${{ needs.provide-runner.outputs.nebius }}
+      truncate_enabled: ${{ contains(github.event.pull_request.labels.*.name, 'disable_truncate') && 'no' || 'yes' }}
     secrets: inherit
 
   sleep-if-needed:

.github/workflows/build_and_test_ya.yaml

@@ -86,6 +86,10 @@ on:
         type: string
         default: "no"
         description: "Run on nebius runners"
+      truncate_enabled:
+        type: string
+        default: "yes"
+        description: "Truncate enabled"
     outputs:
       sleep_after_tests:
         description: "sleep_after_tests"
@@ -166,7 +170,7 @@ jobs:
         clean_ya_dir: ${{ inputs.run_build && 'no' || inputs.clean_ya_dir }}
         use_network_cache: ${{ inputs.use_network_cache }}
         nebius: ${{ inputs.nebius }}
-
+        truncate_enabled: ${{ inputs.truncate_enabled }}
     - id: failure
       name: set sleep_after_tests in case of failure
       if: failure()

.github/workflows/create_and_delete_vm.yaml

@@ -179,6 +179,7 @@ jobs:
       clean_ya_dir: "no"
       use_network_cache: "yes"
       nebius: ${{ needs.provide-runner.outputs.nebius }}
+      truncate_enabled: ${{ contains(github.event.pull_request.labels.*.name, 'disable_truncate') && 'no' || 'yes' }}
     secrets: inherit
 
   release-runner:

.github/workflows/nightly.yaml

@@ -105,6 +105,6 @@ jobs:
       test_threads: 32
       disk_type: 'network-ssd-nonreplicated'
       use_network_cache: "yes"
-      upload_ya_dir: "yes"
+      upload_ya_dir: "no"
       clean_ya_dir: "yes"
       nebius: ${{ vars.GLOBAL_RUN_ON_NEBIUS }}

.github/workflows/packer.yaml

@@ -157,16 +157,16 @@ jobs:
     - name: Set up Nebius CLI
       if: inputs.nebius == 'yes'
       run: |
-        curl -sSL https://storage.ai.nebius.cloud/nebius/install.sh | bash
+        curl -sSL https://storage.eu-north1.nebius.cloud/cli/install.sh | bash
         cat <<EOF > sa.json
         ${sa_json}
         EOF
 
-        jq -r '."subject-credentials"."private-key"'  sa.json> private.pem
+        jq -r '."subject-credentials"."private-key"'  sa.json > private.pem
         echo "::add-mask::$(jq -r '."subject-credentials"."kid"' sa.json)"
         public_key_id=$(jq -r '."subject-credentials"."kid"' sa.json)
         echo "::add-mask::$(jq -r '."subject-credentials"."iss"' sa.json)"
-        service_account_id=$(jq -r '."subject-credentials"."iss" sa.json')
+        service_account_id=$(jq -r '."subject-credentials"."iss"' sa.json)
         echo "::add-mask::tenant-e00en3r863f7me6wtd"
         nebius profile create --endpoint api.eu-north1.nebius.cloud \
                               --profile nbs-github-user-sa \

GITHUB.md

@@ -10,6 +10,7 @@ There is also a list of labels that slightly alters how and which tests are run:
 4. `asan` or `tsan` to add address sanitizer or thread sanitizer builds on top of the regular build.
 5. `recheck` trigger checks without commit. Removed automatically after launch.
 6. `allow-downgrade` allows to downgrade VM preset dynamically in case of problems with resources
+7. `disable_truncate`, by default, .err files are truncated to 1GiB, this disables that for this PR
 
 Also, you can launch [ya make](https://github.com/ydb-platform/nbs/actions/workflows/build_and_test_on_demand.yaml) or [cmake](https://github.com/ydb-platform/nbs/actions/workflows/build_and_test_on_demand_cmake.yaml) builds on your branch with any timeout you want (but please do not do more than 12 hours, VMs are expensive). You can find the IP of the VM inside of the jobs. The first occurrence happens in the `Prepare runner` job in the `Configure NCP` step, later IP is set in the header of jobs. You must use your GitHub key for it. User `github`. Feature not available for non-members.
 

VSCODE.md

@@ -33,3 +33,16 @@ Enable git hooks for pre-commit checks
 ```
 git config core.hooksPath .githooks
 ```
+
+### Debugging
+
+If you want to use debugging in VS Code you should to enable the static linkage.
+
+Add section below to '~/.ya/ya.conf'
+[[target_platform]]
+platform_name = "default-linux-x86_64"
+build_type = "relwithdebinfo"
+#build_type = "release"
+
+[target_platform.flags]
+FORCE_STATIC_LINKING="yes"

cloud/blockstore/apps/server/ya.make

@@ -24,7 +24,7 @@ PEERDIR(
     library/cpp/getopt
 )
 
-IF (BUILD_TYPE != "PROFILE" AND BUILD_TYPE != "DEBUG")
+IF (BUILD_TYPE != "PROFILE" AND BUILD_TYPE != "DEBUG" AND BUILD_TYPE != "RELWITHDEBINFO")
     SPLIT_DWARF()
 ENDIF()
 

cloud/blockstore/config/diagnostics.proto

@@ -4,6 +4,7 @@ package NCloud.NBlockStore.NProto;
 
 option go_package = "github.com/ydb-platform/nbs/cloud/blockstore/config";
 
+import "cloud/storage/core/protos/diagnostics.proto";
 import "cloud/storage/core/protos/trace.proto";
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -219,4 +220,7 @@ message TDiagnosticsConfig
 
     // Performance measurements coefficients for local HDD disks.
     optional TVolumePerfSettings LocalHDDPerfSettings = 51;
+
+    // Type of fetching CPU stats
+    optional NCloud.NProto.EStatsFetcherType StatsFetcherType = 52;
 }

cloud/blockstore/config/disk.proto

@@ -150,6 +150,22 @@ message TStorageDiscoveryConfig
 
 ////////////////////////////////////////////////////////////////////////////////
 
+message TDiskAgentThrottlingConfig
+{
+    // Host limits.
+    optional string InfraThrottlingConfigPath = 1;
+    optional uint32 DefaultNetworkMbitThroughput = 2;
+
+    // Fraction of network throughput utilized for migrations and shadow
+    // disk fill.
+    optional double DirectCopyBandwidthFraction = 3;
+
+    // Maximum bandwidth for one device in MiB/s.
+    optional uint64 MaxDeviceBandwidthMiB = 4;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 message TDiskAgentConfig
 {
     optional bool Enabled = 1;
@@ -275,6 +291,13 @@ message TDiskAgentConfig
     }
 
     repeated TPathToSerialNumber PathToSerialNumberMapping = 37;
+
+    // Settings for traffic shaping.
+    optional TDiskAgentThrottlingConfig ThrottlingConfig = 38;
+
+    // If enabled, IOParserActor allocates a storage buffer and copies the
+    // request data into it.
+    optional bool IOParserActorAllocateStorageEnabled = 39;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

cloud/blockstore/config/storage.proto

@@ -682,7 +682,8 @@ message TStorageServiceConfig
     // Number of ranges to process in a single Compaction run.
     optional uint32 CompactionRangeCountPerRun = 264;
 
-    // Specifies whether to use CompactionRangeCountPerRun.
+    // Specifies whether to use CompactionRangeCountPerRun,
+    // GarbageCompactionRangeCountPerRun and ForcedCompactionRangeCountPerRun.
     optional bool BatchCompactionEnabled = 265;
 
     // Timeout before allowing infra to withdraw our unavailable agents.
@@ -1089,7 +1090,27 @@ message TStorageServiceConfig
 
     // Enable buttons for device state changing, when they in error state.
     optional bool EnableToChangeErrorStatesFromDiskRegistryMonpage = 398;
-    
+
     // Enabling UsedQuota calculation as UsedIopsQuota + UsedBandwidthQuota
     optional bool CalculateSplittedUsedQuotaMetric = 399;
+
+    // Number of ranges to process in a single Compaction run
+    // in the garbage mode if batch compaction enabled.
+    optional uint32 GarbageCompactionRangeCountPerRun = 400;
+
+    // Number of ranges to process in a single Compaction run
+    // in the range mode if batch compaction enabled.
+    optional uint32 ForcedCompactionRangeCountPerRun = 401;
+
+    optional bool YdbViewerServiceEnabled = 402;
+
+    // When enabled, tag "use-intermediate-write-buffer" will be added
+    // after scrubbing finds a mismatch
+    optional bool AutomaticallyEnableBufferCopyingAfterChecksumMismatch = 403;
+
+    // Enabling direct sending AcquireDevices to disk agent.
+    optional bool NonReplicatedVolumeDirectAcquireEnabled = 404;
+
+    // Timeout for TDestroyVolumeActor (in milliseconds)
+    optional uint32 DestroyVolumeTimeout = 405;
 }