Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NBS-5217: Allow images from different folders in tests #2413

Merged
merged 3 commits into from
Nov 6, 2024
Merged

NBS-5217: Allow images from different folders in tests #2413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
af0e781
NBS-5217: Allow images from different folders in tests
ya-ksgamora Nov 1, 2024
1099156
NBS-5217: Allow images from different folders in tests
ya-ksgamora Nov 1, 2024
d98302d
NBS-5217: Add ability to create instances from different folders
ya-ksgamora Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion cloud/blockstore/pylibs/clusters/test_config/test_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class FolderDesc:
subnet_name: str
subnet_id: str
image_name: str
image_folder_id: str
filesystem_id: str
platform_id: str = None
service_account_id: str = None
Expand Down Expand Up @@ -60,7 +61,15 @@ def get_cluster_test_config(name: str, zone_id: str, config_path: str = None) ->
res.__dict__.update(cfg)
folders = dict()
for name in cfg['folder_desc']:
folder = FolderDesc(None, None, None, None, None, None)
folder = FolderDesc(
folder_id=None,
zone_id=None,
subnet_name=None,
subnet_id=None,
image_name=None,
image_folder_id=None,
filesystem_id=None,
)
folder.__dict__.update(cfg['folder_desc'][name])
folders[name] = folder
res.__dict__.update({
Expand Down
10 changes: 6 additions & 4 deletions cloud/blockstore/pylibs/ycp/ycp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ class CreateInstanceConfig:
memory: int
folder_id: str
image_name: str
image_folder_id: str
zone_id: str
subnet_name: str
subnet_id: str
Expand All @@ -167,6 +168,7 @@ class CreateDiskConfig:
placement_group_partition_count: int
zone_id: str
image_name: str
image_folder_id: str
snapshot_name: str
description: str
kek_id: str
Expand Down Expand Up @@ -315,9 +317,9 @@ def _resolve_entity_name(self, cmd, name, stderr, fail_if_not_found=True):

return None

def _resolve_image_name(self, folder_id, image_name, stderr):
def _resolve_image_name(self, image_folder_id, image_name, stderr):
return self._resolve_entity_name(
self._ycp.compute.image.list(folder_id=folder_id),
self._ycp.compute.image.list(folder_id=image_folder_id),
image_name,
stderr)

Expand Down Expand Up @@ -377,7 +379,7 @@ def create_instance(self, config: CreateInstanceConfig) -> Instance:
placement_group_id = entity['id']

image_id = self._resolve_image_name(
config.folder_id,
config.image_folder_id,
config.image_name,
stderr)

Expand Down Expand Up @@ -440,7 +442,7 @@ def create_disk(self, config: CreateDiskConfig) -> Disk:
snapshot_id = None
if config.image_name is not None:
image_id = self._resolve_image_name(
config.folder_id,
config.image_folder_id,
config.image_name,
stderr)
elif config.snapshot_name is not None:
Expand Down
4 changes: 4 additions & 0 deletions cloud/blockstore/pylibs/ycp/ycp_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ def create_instance(self,
placement_group_name: str = None,
host_group: str = None,
image_name: str = None,
image_folder_id: str = None,
name: str = None,
platform_id: str = None,
auto_delete: bool = True,
Expand All @@ -69,6 +70,7 @@ def create_instance(self,
cores=cores,
memory=memory * 1024 ** 3,
image_name=image_name or self._folder_desc.image_name,
image_folder_id=image_folder_id or self._folder_desc.image_folder_id or self._folder_desc.folder_id,
zone_id=self._folder_desc.zone_id,
subnet_name=self._folder_desc.subnet_name,
subnet_id=self._folder_desc.subnet_id,
Expand Down Expand Up @@ -113,6 +115,7 @@ def create_disk(self,
name: str = None,
kek_id: str = None,
image_name: str = None,
image_folder_id: str = None,
snapshot_name: str = None,
auto_delete: bool = True,
description: str = None) -> Ycp.Disk:
Expand All @@ -129,6 +132,7 @@ def create_disk(self,
folder_id=self._folder_desc.folder_id,
kek_id=kek_id,
image_name=image_name,
image_folder_id=image_folder_id or self._folder_desc.image_folder_id or self._folder_desc.folder_id,
snapshot_name=snapshot_name,
description=description,
)
Expand Down
8 changes: 4 additions & 4 deletions ...st/tests/canondata/test.test_check_emptiness_test_cluster1_/testing_out_stuff_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute instance list --folder-id fake-folder' status='NORMAL' ]
##teamcity[message text='|[ #0/zone1 |] Creating instance' status='NORMAL' ]
##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder' status='NORMAL' ]
##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder' status='NORMAL' ]
##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1' status='NORMAL' ]
##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute instance create --request -' status='NORMAL' ]
##teamcity[message text='|[ #0/zone1 |] Created instance <id=fake-instance-id, host=fake, ip=1:1:1:1:1:1:1:1>' status='NORMAL' ]
##teamcity[message text='|[ #0/zone1 |] Creating disk' status='NORMAL' ]
Expand All @@ -30,7 +30,7 @@
##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute instance list --folder-id fake-folder' status='NORMAL' ]
##teamcity[message text='|[ #1/zone1 |] Creating instance' status='NORMAL' ]
##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder' status='NORMAL' ]
##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder' status='NORMAL' ]
##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1' status='NORMAL' ]
##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute instance create --request -' status='NORMAL' ]
##teamcity[message text='|[ #1/zone1 |] Created instance <id=fake-instance-id, host=fake, ip=1:1:1:1:1:1:1:1>' status='NORMAL' ]
##teamcity[message text='|[ #1/zone1 |] Creating disk' status='NORMAL' ]
Expand Down Expand Up @@ -59,7 +59,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand All @@ -82,7 +82,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
3 changes: 3 additions & 0 deletions cloud/blockstore/tools/ci/check_emptiness_test/tests/test-configs/cluster1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"folder_id": "fake-folder",
"hot_attach_fs": true,
"image_name": "fake-name1",
"image_folder_id": "fake-images-folder1",
"platform_id": null,
"service_account_id": null,
"subnet_id": null,
Expand All @@ -20,6 +21,7 @@
"folder_id": "fake-folder",
"hot_attach_fs": true,
"image_name": "fake-name2",
"image_folder_id": "fake-images-folder2",
"platform_id": null,
"service_account_id": null,
"subnet_id": null,
Expand All @@ -32,6 +34,7 @@
"folder_id": "nbs.tests.folder",
"hot_attach_fs": true,
"image_name": "fake-name3",
"image_folder_id": "fake-images-folder3",
"platform_id": null,
"service_account_id": null,
"subnet_id": null,
Expand Down
1 change: 1 addition & 0 deletions cloud/blockstore/tools/ci/corruption_test_suite/lib/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ def run_corruption_test(module_factories: common.ModuleFactories, args, logger):
memory=_TEST_INSTANCE_MEMORY,
compute_node=args.compute_node,
image_name=image,
image_folder_id=folder.image_folder_id,
description="Corruption test") as instance:

server_version = module_factories.fetch_server_version(
Expand Down
2 changes: 1 addition & 1 deletion ...nondata/test.test_corruption_test_suite_nbs-cluster1-512bytes-bs_/512bytes-bs_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...tests/canondata/test.test_corruption_test_suite_nbs-cluster1-64MB-bs_/64MB-bs_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...t_corruption_test_suite_nbs-cluster1-ranges-intersection_/ranges-intersection_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...nondata/test.test_corruption_test_suite_nfs-cluster1-512bytes-bs_/512bytes-bs_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...tests/canondata/test.test_corruption_test_suite_nfs-cluster1-64MB-bs_/64MB-bs_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...t_corruption_test_suite_nfs-cluster1-ranges-intersection_/ranges-intersection_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
3 changes: 3 additions & 0 deletions cloud/blockstore/tools/ci/corruption_test_suite/tests/test-configs/cluster1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"folder_id": "fake-folder",
"hot_attach_fs": true,
"image_name": "fake-name1",
"image_folder_id": "fake-images-folder1",
"platform_id": null,
"service_account_id": null,
"subnet_id": null,
Expand All @@ -20,6 +21,7 @@
"folder_id": "fake-folder",
"hot_attach_fs": true,
"image_name": "fake-name2",
"image_folder_id": "fake-images-folder2",
"platform_id": null,
"service_account_id": null,
"subnet_id": null,
Expand All @@ -32,6 +34,7 @@
"folder_id": "nbs.tests.folder",
"hot_attach_fs": true,
"image_name": "fake-name3",
"image_folder_id": "fake-images-folder3",
"platform_id": null,
"service_account_id": null,
"subnet_id": null,
Expand Down
25 changes: 18 additions & 7 deletions cloud/blockstore/tools/ci/fio_performance_test_suite/lib/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,11 @@ def parse_args() -> argparse.Namespace:
type=str,
default=None,
help='use image with the specified name to create disks')
test_arguments_group.add_argument(
'--image-folder-id',
type=str,
default=None,
help='use image from the specified folder to create disks')
test_arguments_group.add_argument(
'--instance-cores',
type=int,
Expand Down Expand Up @@ -142,8 +147,8 @@ def __init__(
self._logger = logger
self._module_factories = module_factories

self._version_method_name = None
self._instance_image_name = None
def _version_method_name(self) -> str:
raise NotImplementedError()

def _run_tests(
self,
Expand Down Expand Up @@ -193,11 +198,12 @@ def run_test_suite(self) -> None:
platform_id=self._args.platform_id,
compute_node=self._args.compute_node,
placement_group_name=self._args.placement_group_name,
image_name=self._instance_image_name,
image_name=self._args.image_name,
image_folder_id=self._args.image_folder_id,
description='fio performance test') as instance:
version = self._module_factories.fetch_server_version(
self._args.dry_run,
self._version_method_name,
self._version_method_name(),
instance.compute_node,
cluster,
self._logger)
Expand Down Expand Up @@ -335,7 +341,9 @@ def __init__(
logger: logging.Logger,
) -> None:
super().__init__(module_factories, args, profiler, logger)
self._version_method_name = 'get_current_nbs_version'

def _version_method_name(self) -> str:
return 'get_current_nbs_version'

def _teardown_parallel_run(
self,
Expand Down Expand Up @@ -425,6 +433,7 @@ def _run_sequential_tests(
type_id=translate_disk_type(self._args.cluster, test_case.type),
bs=test_case.device_bs,
image_name=self._args.image_name,
image_folder_id=self._args.image_folder_id,
description=f'fio performance test: {test_case.name}') as disk:
try:
with ycp.attach_disk(
Expand Down Expand Up @@ -473,6 +482,7 @@ def _run_parallel_tests(
type_id=translate_disk_type(self._args.cluster, test_case.type),
bs=test_case.device_bs,
image_name=self._args.image_name,
image_folder_id=self._args.image_folder_id,
auto_delete=False,
description=f'fio performance test: {test_case.name}') as disk:
with ycp.attach_disk(
Expand Down Expand Up @@ -557,8 +567,9 @@ def __init__(
logger: logging.Logger,
) -> None:
super().__init__(module_factories, args, profiler, logger)
self._version_method_name = 'get_current_nfs_version'
self._instance_image_name = 'ubuntu-2004-eternal'

def _version_method_name(self) -> str:
return 'get_current_nfs_version'

def _mount_fs(
self,
Expand Down
2 changes: 1 addition & 1 deletion ...t_fio_performance_test_suite_False-nbs-cluster_and_platform0-default_/default_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...test_suite_False-nbs-cluster_and_platform0-large_block_size_/large_block_size_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...mance_test_suite_False-nbs-cluster_and_platform0-max_bandwidth_/max_bandwidth_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...fio_performance_test_suite_False-nbs-cluster_and_platform0-max_iops_/max_iops_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...mance_test_suite_False-nbs-cluster_and_platform0-mirrored_iops_/mirrored_iops_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...uite_False-nbs-cluster_and_platform0-mirrored_throughput_/mirrored_throughput_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...test.test_fio_performance_test_suite_False-nbs-cluster_and_platform0-nrd_/nrd_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
2 changes: 1 addition & 1 deletion ...fio_performance_test_suite_False-nbs-cluster_and_platform0-nrd_iops_/nrd_iops_results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
Input=
Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
Input=
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
Input=
Command=ycp --format json --profile fake-cluster1 compute instance create --request -
Input=
Expand Down
Loading
Loading