From 900f5f5f5b5d4f87512f1570c6f10452eeae9acb Mon Sep 17 00:00:00 2001 From: Andrei Kashin Date: Fri, 28 Jun 2024 14:31:06 +0100 Subject: [PATCH] Locust: Pre-generate users in FT benchmark (#11593) This PR allow to generate multiple large FT contracts that can be reused in the future benchmark runs. I ran this with 10M users per contract and was able to generate 4 contracts (one per worker), each with 10M users and 1.25GB of state in 24 hours (with gas_limit set to 10 PGas): ```sh locust -H 127.0.0.1:3030 \ -f locustfiles/ft.py \ --funding-key=$KEY \ --fixed-contract-names --num-ft-contracts=1 \ --num-passive-users=10000000 \ -u 4000 -r 500 --headless --processes 4 ``` --- pytest/tests/loadtest/locust/common/ft.py | 45 +++++++++++++++++------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/pytest/tests/loadtest/locust/common/ft.py b/pytest/tests/loadtest/locust/common/ft.py index 424ef4e9aa4..1c68accc474 100644 --- a/pytest/tests/loadtest/locust/common/ft.py +++ b/pytest/tests/loadtest/locust/common/ft.py @@ -1,3 +1,4 @@ +import logging from concurrent import futures import random import string @@ -53,7 +54,7 @@ def register_passive_user(self, node: NearNodeProxy, account: Account): """ Passive users are only used as receiver, not as signer. """ - node.send_tx_retry(InitFTAccount(self.account, account), + node.send_tx_async(InitFTAccount(self.account, account), locust_name="Init FT Account") self.registered_users.append(account.key.account_id) @@ -85,20 +86,30 @@ def create_passive_users(self, assert prefix_len > 4, f"user key {parent.key.account_id} is too long" chars = string.ascii_lowercase + string.digits - def create_account(): - prefix = ''.join(random.choices(chars, k=prefix_len)) + def create_account(i): + prefix = ''.join(random.Random(i).choices(chars, k=prefix_len)) account_id = f"{prefix}.{parent.key.account_id}" return Account(key.Key.from_seed_testonly(account_id)) - accounts = [create_account() for _ in range(num)] - node.prepare_accounts(accounts, - parent, - balance=1, - msg="create passive user") - with futures.ThreadPoolExecutor() as executor: - futures.wait( - executor.submit(self.register_passive_user, node, account) - for account in accounts) + with futures.ThreadPoolExecutor(max_workers=4) as executor: + batch_size = 500 + num_batches = (num + batch_size - 1) // batch_size + for i in range(num_batches): + accounts = [ + create_account(i) + for i in range(i * batch_size, min((i + 1) * + batch_size, num)) + ] + node.prepare_accounts(accounts, + parent, + balance=1, + msg="create passive user") + futures.wait( + executor.submit(self.register_passive_user, node, account) + for account in accounts) + logging.info( + f"{parent.key.account_id}: Processed batch {i + 1}/{num_batches}, created {(i + 1) * batch_size} users" + ) class TransferFT(FunctionCall): @@ -183,7 +194,12 @@ def on_locust_init(environment, **kwargs): ft_account = Account(contract_key) ft_contract = FTContract(ft_account, ft_account, ft_contract_code) ft_contract.install(node, funding_account) + if environment.parsed_options.num_passive_users > 0: + ft_contract.create_passive_users( + environment.parsed_options.num_passive_users, node, + funding_account) environment.ft_contracts.append(ft_contract) + logging.info(f"Finished setup for account {i} on worker {parent_id}") # FT specific CLI args @@ -205,3 +221,8 @@ def _(parser): help= "Whether the names of FT contracts will deterministically based on worker id and run id." ) + parser.add_argument( + "--num-passive-users", + type=int, + default=0, + help="Number of passive users to create in each FT contract.")