From add202705ead0866352ca01a75463fd933c12a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 30 Jul 2021 13:27:54 +0200 Subject: [PATCH 01/35] Fix typo --- smart-contracts/.env.ui.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart-contracts/.env.ui.example b/smart-contracts/.env.ui.example index df037710d8..89497b97dd 100644 --- a/smart-contracts/.env.ui.example +++ b/smart-contracts/.env.ui.example @@ -12,7 +12,7 @@ LOCAL_PROVIDER="http://localhost:7545" # ------------ # Network # ------------ -# Owner of the bridgebank, this must be set when we are deploying to tesnet or mainnet +# Owner of the bridgebank, this must be set when we are deploying to testnet or mainnet OWNER='0x627306090abaB3A6e1400e9345bC60c78a8BEf57' # Replace example MNEMONIC with your MetaMask mnemonic MNEMONIC="candy maple cake sugar pudding cream honey rich smooth crumble sweet treat" From 9375e0d4d1c389f3284cbed22a7f9cc2841ba4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 1 Aug 2021 19:10:10 +0200 Subject: [PATCH 02/35] Fix wrong path --- test/integration/sifchain_start_daemon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/sifchain_start_daemon.sh b/test/integration/sifchain_start_daemon.sh index a1f17b1fac..bc2f46b973 100755 --- a/test/integration/sifchain_start_daemon.sh +++ b/test/integration/sifchain_start_daemon.sh @@ -8,7 +8,7 @@ set -x SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -. /sifnode/test/integration/vagrantenv.sh +. $(SCRIPT_DIR)/vagrantenv.sh . ${TEST_INTEGRATION_DIR}/shell_utilities.sh whitelisted_validator=$(yes $VALIDATOR1_PASSWORD | sifnoded keys show --keyring-backend file -a --bech val $MONIKER --home $CHAINDIR/.sifnoded) From a8229642bbc8276006260b790d134f0b9624c10c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 2 Aug 2021 21:22:19 +0200 Subject: [PATCH 03/35] Cleanup --- ui/chains/post_migrate.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui/chains/post_migrate.sh b/ui/chains/post_migrate.sh index b651503a41..a42f8910e1 100755 --- a/ui/chains/post_migrate.sh +++ b/ui/chains/post_migrate.sh @@ -47,7 +47,8 @@ yarn peggy:whiteList "$USDC_ADDRESS" true yarn peggy:whiteList "$LINK_ADDRESS" true # Update local test addresses -cd $BASE_DIR/ui/core/ -./scripts/updateLocalTestAddresses.js +# (disabled: no such file or directory) +# cd $BASE_DIR/ui/core/ +# ./scripts/updateLocalTestAddresses.js echo "1" > $BASE_DIR/ui/node_modules/.migrate-complete From cdc78d242e1682ab7ec8579c5c2262d256f8da74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 3 Aug 2021 06:09:46 +0200 Subject: [PATCH 04/35] Fail if setup_sifchain.sh fails --- test/integration/start-integration-env.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/start-integration-env.sh b/test/integration/start-integration-env.sh index 6559261423..b1a38c08dd 100755 --- a/test/integration/start-integration-env.sh +++ b/test/integration/start-integration-env.sh @@ -63,6 +63,7 @@ set_persistant_env_var BRIDGE_TOKEN_ADDRESS $(cat $BASEDIR/smart-contracts/build set_persistant_env_var BRIDGE_BANK_ADDRESS $(cat $BASEDIR/smart-contracts/build/contracts/BridgeBank.json | jq -r '.networks["5777"].address') $envexportfile required rm -rf /tmp/sifchainrelayerdb -bash ${BASEDIR}/test/integration/setup_sifchain.sh && . $envexportfile +bash ${BASEDIR}/test/integration/setup_sifchain.sh +. $envexportfile logecho finished $0 From c40e3b9150a14f36e247a92fd4f4ddc3dfa2e7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 30 Jul 2021 13:30:10 +0200 Subject: [PATCH 05/35] Refactoring scripts for building UI Stack docker image - initial revision --- test/integration/make.py | 302 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 test/integration/make.py diff --git a/test/integration/make.py b/test/integration/make.py new file mode 100644 index 0000000000..6cf456fcef --- /dev/null +++ b/test/integration/make.py @@ -0,0 +1,302 @@ +import json +import logging +import os +import shutil +import subprocess +import yaml # pip install pyyaml +import urllib.request +import time + + +log = logging.getLogger(__name__) + + +def stdout_lines(res): + return res[0].splitlines() + +def project_dir(*paths): + return os.path.abspath(os.path.join(os.path.normpath(os.path.join(__file__, *([os.path.pardir]*3))), *paths)) + +def yaml_load(s): + return yaml.load(s, Loader=yaml.SafeLoader) + +def sif_format_amount(amount, denom): + return "{}{}".format(amount, denom) + +def http_get(url): + with urllib.request.urlopen(url) as r: + return r.read() + +def popen(args, env=None): + return subprocess.Popen(args, env=env) + +NULL_ADDRESS = "0x0000000000000000000000000000000000000000" + + +class Command: + def execst(self, args, cwd=None, env=None, stdin=None, binary=False): + if stdin is not None: + if type(stdin) == list: + stdin = "".join([line + "\n" for line in stdin]) + popen = subprocess.Popen(args, cwd=cwd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=not binary) + stdout_data, stderr_data = popen.communicate(input=stdin) + if popen.returncode != 0: + raise Exception("Command '{}' exited with returncode {}: {}".format(" ".join(args), popen.returncode, repr(stderr_data))) + return popen.returncode, stdout_data, stderr_data + + def read_text_file(self, path): + with open(path, "rt") as f: + return f.read() # TODO Convert to exec + + def rmdir(self, path): + if os.path.exists(path): + shutil.rmtree(path) # TODO Convert to exec + + def copy_file(self, src, dst): + shutil.copy(src, dst) + + def get_user_home(self, *paths): + return os.path.join(os.environ["HOME"], *paths) + + +class Ganache(Command): + def start_ganache_cli(self, mnemonic=None, db=None, port=None, host=None, network_id=None, gas_price=None, gas_limit=None): + args = ["ganache-cli"] + \ + (["--mnemonic", " ".join(mnemonic)] if mnemonic else []) + \ + (["--db", db] if db else []) + \ + (["--port", str(port)] if port is not None else []) + \ + (["--host", host] if host else []) + \ + (["--networkId", str(network_id)] if network_id is not None else []) + \ + (["--gasPrice", str(gas_price)] if gas_price is not None else []) + \ + (["--gasLimit", str(gas_limit)] if gas_limit is not None else []) + return popen(args) + + +class Sifnoded(Command): + def sifnoded_init_test(self, moniker, chain_id): + args = ["sifnoded", "init", moniker, "--chain-id={}".format(chain_id)] + self.execst(args) + + def sifnoded_generate_deterministic_account(self, name, mnemonic): + args = ["sifnoded", "keys", "add", name, "--keyring-backend={}".format("test"), "--recover"] + stdin = [" ".join(mnemonic)] + res = self.execst(args, stdin=stdin) + return yaml_load(res[1])[0] + + def sifnoded_keys_show(self, name, bech=None, keyring_backend=None): + keyring_backend = "test" + args = ["sifnoded", "keys", "show", name] + \ + (["--bech", bech] if bech else []) + \ + (["--keyring-backend={}".format(keyring_backend)] if keyring_backend else []) + res = self.execst(args) + return yaml_load(res[1]) + + def sifnoded_add_genesis_account(self, address, tokens): + tokens_str = ",".join([sif_format_amount(amount, denom) for amount, denom in tokens]) + args = ["sifnoded", "add-genesis-account", address, tokens_str] + self.execst(args) + + def sifnoded_add_genesis_validators(self, address): + args = ["sifnoded", "add-genesis-validators", address] + res = self.execst(args) + return res + + def sifnoded_tx_clp_create_pool(self, chain_id, keyring_backend, from_name, symbol, fees, native_amount, external_amount): + args = ["sifnoded", "tx", "clp", "create-pool", "--chain-id={}".format(chain_id), + "--keyring-backend={}".format(keyring_backend), "--from", from_name, "--symbol", symbol, "--fees", + sif_format_amount(*fees), "--nativeAmount", str(native_amount), "--externalAmount", str(external_amount), + "--yes"] + self.execst(args) + + def sifnoded_launch(self, minimum_gas_prices=None): + args = ["sifnoded", "start"] + \ + (["--minimum-gas-prices", sif_format_amount(*minimum_gas_prices)] if minimum_gas_prices is not None else []) + return popen(args) + + def sifnoded_get_status(self, host, port): + url = "http://{}:{}/node_info".format(host, port) + return json.loads(http_get(url).decode("UTF-8")) + + +class Integrator(Ganache, Sifnoded, Command): + def ebrelayer_init(self, ethereum_private_key, tendermind_node, web3_provider, bridge_registry_contract_address, + validator_moniker, validator_mnemonic, chain_id, gas, gas_prices): + env = {"ETHEREUM_PRIVATE_KEY": ethereum_private_key} + args = ["ebrelayer", "init", tendermind_node, web3_provider, bridge_registry_contract_address, + validator_moniker, validator_mnemonic, "--chain-id={}".format(chain_id), "--gas", str(gas), "--gas-prices", + sif_format_amount(*gas_prices)] + return popen(args, env=env) + + def sif_wait_up(self, host, port): + while True: + from urllib.error import URLError + try: + return self.sifnoded_get_status(host, port) + except URLError: + time.sleep(1) + + def yarn(self, args, cwd=None, env=None): + return self.execst(["yarn"] + args, cwd=cwd, env=env) + + +class UIPlaybook: + # From ui/chains/credentials.sh + SHADOWFIEND_NAME = "shadowfiend" + SHADOWFIEND_MNEMONIC = ["race", "draft", "rival", "universe", "maid", "cheese", "steel", "logic", "crowd", "fork", + "comic", "easy", "truth", "drift", "tomorrow", "eye", "buddy", "head", "time", "cash", "swing", "swift", + "midnight", "borrow"] + AKASHA_NAME = "akasha" + AKASHA_MNEMONIC = ["hand", "inmate", "canvas", "head", "lunar", "naive", "increase", "recycle", "dog", "ecology", + "inhale", "december", "wide", "bubble", "hockey", "dice", "worth", "gravity", "ketchup", "feed", "balance", + "parent", "secret", "orchard"] + JUNIPER_NAME = "juniper" + JUNIPER_MNEMONIC = ["clump", "genre", "baby", "drum", "canvas", "uncover", "firm", "liberty", "verb", "moment", + "access", "draft", "erupt", "fog", "alter", "gadget", "elder", "elephant", "divide", "biology", "choice", + "sentence", "oppose", "avoid"] + ETHEREUM_ROOT_MNEMONIC = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", + "crumble", "sweet", "treat"] + CHAIN_ID = "sifchain-local" + + def __init__(self, cmd): + self.cmd = cmd + + def ui_sif_launch(self): + self.cmd.rmdir(self.cmd.get_user_home(".sifnoded")) + self.cmd.sifnoded_init_test("test", UIPlaybook.CHAIN_ID) + self.cmd.copy_file(project_dir("ui", "chains", "sif", "app.toml"), self.cmd.get_user_home(".sifnoded", "config", "app.toml")) + log.info(f"Generating deterministic account - {UIPlaybook.SHADOWFIEND_NAME}...") + shadowfiend_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.SHADOWFIEND_NAME, UIPlaybook.SHADOWFIEND_MNEMONIC) + log.info(f"Generating deterministic account - {UIPlaybook.AKASHA_NAME}...") + akasha_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.AKASHA_NAME, UIPlaybook.AKASHA_MNEMONIC) + log.info(f"Generating deterministic account - {UIPlaybook.JUNIPER_NAME}...") + juniper_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.JUNIPER_NAME, UIPlaybook.JUNIPER_MNEMONIC) + # shadowfiend_address = self.sifnoded_keys_show(SHADOWFIEND_NAME)[0]["address"] + # akasha_address = self.sifnoded_keys_show(AKASHA_NAME)[0]["address"] + # juniper_address = self.sifnoded_keys_show(JUNIPER_NAME)[0]["address"] + shadowfiend_address = shadowfiend_account["address"] + akasha_address = akasha_account["address"] + juniper_address = juniper_account["address"] + + tokens_shadowfiend = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] + tokens_akasha = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] + tokens_juniper = [[10**22, "rowan"], [10**22, "cusdc"], [10**20, "clink"], [10**20, "ceth"]] + self.cmd.sifnoded_add_genesis_account(shadowfiend_address, tokens_shadowfiend) + self.cmd.sifnoded_add_genesis_account(akasha_address, tokens_akasha) + self.cmd.sifnoded_add_genesis_account(juniper_address, tokens_juniper) + + shadowfiend_address_bech_val = self.cmd.sifnoded_keys_show(UIPlaybook.SHADOWFIEND_NAME, bech="val")[0]["address"] + self.cmd.sifnoded_add_genesis_validators(shadowfiend_address_bech_val) + + amount = "1000000000000000000000000stake" + self.cmd.execst(["sifnoded", "gentx", UIPlaybook.SHADOWFIEND_NAME, amount, "--chain-id={}".format(UIPlaybook.CHAIN_ID), "--keyring-backend=test"]) + + log.info("Validating genesis file...") + self.cmd.execst(["sifnoded", "collect-gentxs"]) + log.info("Starting test chain...") + self.cmd.execst(["sifnoded", "validate-genesis"]) + + def ui_sif_start(self): + return self.cmd.sifnoded_launch(minimum_gas_prices=[0.5, "rowan"]) + + def migrate(self): + # peggy + smart_contracts_dir = project_dir("smart-contracts") + self.cmd.copy_file(os.path.join(smart_contracts_dir, ".env.ui.example"), os.path.join(smart_contracts_dir, ".env")) + self.cmd.yarn(cwd=smart_contracts_dir) + self.cmd.yarn(["migrate"], cwd=smart_contracts_dir) + + # eth + # send through atk and btk tokens to eth chain + self.cmd.yarn(["migrate"], cwd=project_dir("ui", "chains", "eth")) + + # sif + # Original scripts say "if we don't sleep there are issues" + keyring_backend = "test" + time.sleep(10) + log.info("Creating liquidity pool from catk:rowan...") + self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "catk", [10**5, "rowan"], 10**25, 10**25) + time.sleep(5) + log.info("Creating liquidity pool from cbtk:rowan...") + self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "cbtk", [10**5, "rowan"], 10**25, 10**25) + # should now be able to swap from catk:cbtk + time.sleep(5) + log.info("Creating liquidity pool from ceth:rowan...") + self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "ceth", [10**5, "rowan"], 10**25, 83*10**20) + # should now be able to swap from x:ceth + time.sleep(5) + log.info("Creating liquidity pool from cusdc:rowan...") + self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "cusdc", [10**5, "rowan"], 10**25, 10**25) + time.sleep(5) + log.info("Creating liquidity pool from clink:rowan...") + self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "clink", [10**5, "rowan"], 10**25, 588235*10**18) + time.sleep(5) + log.info("Creating liquidity pool from ctest:rowan...") + self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "ctest", [10**5, "rowan"], 10**25, 10**13) + + def get_smart_contract_address(path): + return json.loads(self.cmd.read_text_file(project_dir(path)))["networks"]["5777"]["address"] + + # post_migrate.sh + atk_address, btk_address, usdc_address, link_address = [ + get_smart_contract_address(project_dir(f"ui/chains/eth/build/contracts/{x}.json")) + for x in ["AliceToken", "BobToken", "UsdCoin", "LinkCoin"] + ] + bridge_token_address = get_smart_contract_address(project_dir("smart-contracts/build/contracts/BridgeToken.json")) + bridge_registry_address = get_smart_contract_address(project_dir("smart-contracts/build/contracts/BridgeRegistry.json")) + + # From smart-contracts/.env.ui.example + smart_contracts_env_ui_example_vars = { + "ETHEREUM_PRIVATE_KEY": "c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3", + "INFURA_PROJECT_ID": "JFSH7439sjsdtqTM23Dz", + "LOCAL_PROVIDER": "http://localhost:7545", + } + + def set_token_lock_burn_limit(update_address, amount): + env = smart_contracts_env_ui_example_vars.copy() + env["UPDATE_ADDRESS"] = update_address + # Needs: ETHEREUM_PRIVATE_KEY, INFURA_PROJECT_ID, LOCAL_PROVIDER, UPDATE_ADDRESS + # Hint: call web3 directly, avoid npx + truffle + script + # Maybe: self.cmd.yarn(["integrationtest:setTokenLockBurnLimit", str(amount)]) + self.cmd.execst(["npx", "truffle", "exec", "scripts/setTokenLockBurnLimit.js", str(amount)], env=env, cwd=smart_contracts_dir) + + set_token_lock_burn_limit(NULL_ADDRESS, 31*10**18) + set_token_lock_burn_limit(bridge_token_address, 10**25) + set_token_lock_burn_limit(atk_address, 10**25) + set_token_lock_burn_limit(btk_address, 10**25) + set_token_lock_burn_limit(usdc_address, 10**25) + set_token_lock_burn_limit(link_address, 10**25) + + # Whitelist test tokens + for addr in [atk_address, btk_address, usdc_address, link_address]: + self.cmd.yarn(["peggy:whitelist", addr, "true"], cwd=smart_contracts_dir) + + # Peggy + ethereum_private_key = smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"] + self.cmd.ebrelayer_init(ethereum_private_key, "tcp://localhost:26657", "ws://localhost:7545/", + bridge_registry_address, UIPlaybook.SHADOWFIEND_NAME, UIPlaybook.SHADOWFIEND_MNEMONIC, + "--chain-id={}".format(UIPlaybook.CHAIN_ID), 5*10**12, [0.5, "rowan"]) + + def eth_start(self): + return self.cmd.start_ganache_cli(mnemonic=UIPlaybook.ETHEREUM_ROOT_MNEMONIC, + db=self.cmd.get_user_home(".ganachedb"), port=7545, network_id=5777, gas_price=20000000000, + gas_limit=6721975, host="0.0.0.0") + + def run(self): + self.cmd.yarn([], cwd=project_dir()) # TODO Where? + + ganache_cli_proc = self.eth_start() + print(repr(ganache_cli_proc)) + self.ui_sif_launch() + sifnoded_proc = self.ui_sif_start() + print(repr(sifnoded_proc)) + self.cmd.sif_wait_up("localhost", 1317) + self.migrate() + + +def main(): + cmd = Integrator() + ui_playbook = UIPlaybook(cmd) + ui_playbook.run() + +if __name__ == "__main__": + main() From 9d75aacb9780079738ec31423eeadd2343e68cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 31 Jul 2021 01:19:11 +0200 Subject: [PATCH 06/35] Refactoring --- test/integration/make.py | 127 +++++++++++++++++++++++++++++---------- 1 file changed, 95 insertions(+), 32 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 6cf456fcef..cfffb84e04 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -58,6 +58,12 @@ def copy_file(self, src, dst): def get_user_home(self, *paths): return os.path.join(os.environ["HOME"], *paths) + def tar_create(self, path, tar_path, compression=None): + comp_opts = {"gz": "z"} + comp = comp_opts[compression] if compression in comp_opts else "" + args = ["tar", "cf" + comp, tar_path, "."] + self.execst(args, cwd=path) + class Ganache(Command): def start_ganache_cli(self, mnemonic=None, db=None, port=None, host=None, network_id=None, gas_price=None, gas_limit=None): @@ -73,7 +79,7 @@ def start_ganache_cli(self, mnemonic=None, db=None, port=None, host=None, networ class Sifnoded(Command): - def sifnoded_init_test(self, moniker, chain_id): + def sifnoded_init(self, moniker, chain_id): args = ["sifnoded", "init", moniker, "--chain-id={}".format(chain_id)] self.execst(args) @@ -160,9 +166,48 @@ class UIPlaybook: def __init__(self, cmd): self.cmd = cmd - def ui_sif_launch(self): + def eth_start(self): + return + + def run(self): + # self.cmd.yarn([], cwd=project_dir()) # TODO Where? + # ganache_cli_proc = self.eth_start() + # print(repr(ganache_cli_proc)) + # self.ui_sif_launch() + # sifnoded_proc = self.ui_sif_start() + # print(repr(sifnoded_proc)) + # self.cmd.sif_wait_up("localhost", 1317) + self.stack_save_snapshot() + + def stack_save_snapshot(self): + # ui-stack.yml + # cd .; go get -v -t -d ./... + # cd ui; yarn install --frozen-lockfile --silent + # Compile smart contracts: + # cd ui; yarn build + + keyring_backend = "test" + + # yarn stack --save-snapshot -> ui/scripts/stack.sh -> ui/scripts/stack-save-snapshot.sh + # rm ui/node_modules/.migrate-complete + + # yarn stack --save-snapshot -> ui/scripts/stack.sh -> ui/scripts/stack-save-snapshot.sh => ui/scripts/stack-launch.sh + # ui/scripts/stack-launch.sh -> ui/scripts/_sif-build.sh -> ui/chains/sif/build.sh + # killall sifnoded + # rm $(which sifnoded) self.cmd.rmdir(self.cmd.get_user_home(".sifnoded")) - self.cmd.sifnoded_init_test("test", UIPlaybook.CHAIN_ID) + self.cmd(["make", "install"], project_dir()) + + # ui/scripts/stack-launch.sh -> ui/scripts/_eth.sh -> ui/chains/etc/launch.sh + self.cmd.rmdir(self.cmd.get_user_home(".ganachedb")) + self.cmd.yarn([], cwd=project_dir("ui/chains/eth")) # Installs ui/chains/eth/node_modules + # Note that this runs ganache-cli from $PATH whereas scripts start it with yarn in ui/chains/eth + ganache_proc = self.cmd.start_ganache_cli(mnemonic=UIPlaybook.ETHEREUM_ROOT_MNEMONIC, + db=self.cmd.get_user_home(".ganachedb"), port=7545, network_id=5777, gas_price=20000000000, + gas_limit=6721975, host="0.0.0.0") + + # ui/scripts/stack-launch.sh -> ui/scripts/_sif.sh -> ui/chains/sif/launch.sh + self.cmd.sifnoded_init("test", UIPlaybook.CHAIN_ID) self.cmd.copy_file(project_dir("ui", "chains", "sif", "app.toml"), self.cmd.get_user_home(".sifnoded", "config", "app.toml")) log.info(f"Generating deterministic account - {UIPlaybook.SHADOWFIEND_NAME}...") shadowfiend_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.SHADOWFIEND_NAME, UIPlaybook.SHADOWFIEND_MNEMONIC) @@ -187,31 +232,32 @@ def ui_sif_launch(self): shadowfiend_address_bech_val = self.cmd.sifnoded_keys_show(UIPlaybook.SHADOWFIEND_NAME, bech="val")[0]["address"] self.cmd.sifnoded_add_genesis_validators(shadowfiend_address_bech_val) - amount = "1000000000000000000000000stake" - self.cmd.execst(["sifnoded", "gentx", UIPlaybook.SHADOWFIEND_NAME, amount, "--chain-id={}".format(UIPlaybook.CHAIN_ID), "--keyring-backend=test"]) + amount = sif_format_amount(10**24, "stake") + self.cmd.execst(["sifnoded", "gentx", UIPlaybook.SHADOWFIEND_NAME, amount, "--chain-id={}".format(UIPlaybook.CHAIN_ID), "--keyring-backend={}".format(keyring_backend)]) - log.info("Validating genesis file...") + log.info("Collecting genesis txs...") self.cmd.execst(["sifnoded", "collect-gentxs"]) - log.info("Starting test chain...") + log.info("Validating genesis file...") self.cmd.execst(["sifnoded", "validate-genesis"]) - def ui_sif_start(self): - return self.cmd.sifnoded_launch(minimum_gas_prices=[0.5, "rowan"]) + log.info("Starting test chain...") + sifnoded_proc = self.cmd.sifnoded_launch(minimum_gas_prices=[0.5, "rowan"]) - def migrate(self): - # peggy + # sifnoded must be up before continuing + self.cmd.sif_wait_up("localhost", 1317) + + # ui/scripts/_migrate.sh -> ui/chains/peggy/migrate.sh smart_contracts_dir = project_dir("smart-contracts") self.cmd.copy_file(os.path.join(smart_contracts_dir, ".env.ui.example"), os.path.join(smart_contracts_dir, ".env")) - self.cmd.yarn(cwd=smart_contracts_dir) + self.cmd.yarn([], cwd=smart_contracts_dir) self.cmd.yarn(["migrate"], cwd=smart_contracts_dir) - # eth + # ui/scripts/_migrate.sh -> ui/chains/eth/migrate.sh # send through atk and btk tokens to eth chain - self.cmd.yarn(["migrate"], cwd=project_dir("ui", "chains", "eth")) + self.cmd.yarn(["migrate"], cwd=project_dir("ui/chains/eth")) - # sif + # ui/scripts/_migrate.sh -> ui/chains/sif/migrate.sh # Original scripts say "if we don't sleep there are issues" - keyring_backend = "test" time.sleep(10) log.info("Creating liquidity pool from catk:rowan...") self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "catk", [10**5, "rowan"], 10**25, 10**25) @@ -233,10 +279,10 @@ def migrate(self): log.info("Creating liquidity pool from ctest:rowan...") self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "ctest", [10**5, "rowan"], 10**25, 10**13) + # ui/scripts/_migrate.sh -> ui/chains/post_migrate.sh def get_smart_contract_address(path): return json.loads(self.cmd.read_text_file(project_dir(path)))["networks"]["5777"]["address"] - # post_migrate.sh atk_address, btk_address, usdc_address, link_address = [ get_smart_contract_address(project_dir(f"ui/chains/eth/build/contracts/{x}.json")) for x in ["AliceToken", "BobToken", "UsdCoin", "LinkCoin"] @@ -265,32 +311,49 @@ def set_token_lock_burn_limit(update_address, amount): set_token_lock_burn_limit(btk_address, 10**25) set_token_lock_burn_limit(usdc_address, 10**25) set_token_lock_burn_limit(link_address, 10**25) + # signal migrate-complete # Whitelist test tokens for addr in [atk_address, btk_address, usdc_address, link_address]: self.cmd.yarn(["peggy:whitelist", addr, "true"], cwd=smart_contracts_dir) - # Peggy + # ui/scripts/stack-launch.sh -> ui/scripts/_peggy.sh -> ui/chains/peggy/launch.sh + # rm -rf ui/chains/peggy/relayerdb ethereum_private_key = smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"] - self.cmd.ebrelayer_init(ethereum_private_key, "tcp://localhost:26657", "ws://localhost:7545/", + ebrelayer_proc = self.cmd.ebrelayer_init(ethereum_private_key, "tcp://localhost:26657", "ws://localhost:7545/", bridge_registry_address, UIPlaybook.SHADOWFIEND_NAME, UIPlaybook.SHADOWFIEND_MNEMONIC, "--chain-id={}".format(UIPlaybook.CHAIN_ID), 5*10**12, [0.5, "rowan"]) - def eth_start(self): - return self.cmd.start_ganache_cli(mnemonic=UIPlaybook.ETHEREUM_ROOT_MNEMONIC, - db=self.cmd.get_user_home(".ganachedb"), port=7545, network_id=5777, gas_price=20000000000, - gas_limit=6721975, host="0.0.0.0") + # At this point we have 3 running processes - ganache_proc, sifnoded_proc and ebrelayer_proc + # await sif-node-up and migrate-complete - def run(self): - self.cmd.yarn([], cwd=project_dir()) # TODO Where? + time.sleep(30) + # ui/scripts/_snapshot.sh - ganache_cli_proc = self.eth_start() - print(repr(ganache_cli_proc)) - self.ui_sif_launch() - sifnoded_proc = self.ui_sif_start() - print(repr(sifnoded_proc)) - self.cmd.sif_wait_up("localhost", 1317) - self.migrate() + # ui/scripts/stack-pause.sh + # killall sifnoded sifnoded ebrelayer ganache-cli + sifnoded_proc.kill() + ebrelayer_proc.kill() + ganache_proc.kill() + time.sleep(10) + + # ui/chains/peggy/snapshot.sh: + # mkdir -p ui/chains/snapshots + # mkdir -p ui/chains/peggy/relayerdb + # cd ui/chains/peggy/relayerdb && tar -zcvf ui/chains/snapshots/peggy.tar.gz + self.cmd.tar_create(project_dir("ui/chains/peggy/relayerdb"), project_dir("ui/chains/snapshots/peggy.tar.gz"), compression="gz") + # mkdir -p smart-contracts/build + # cd smart-contracts/build && tar -zcvf ui/chains/snapshots/peggy_build.tar.gz + self.cmd.tar_create(project_dir("smart-contracts/build"), project_dir("ui/chains/snapshots/peggy_build.tar.gz"), compression="gz") + + # ui/chains/sif/snapshot.sh: + # mkdir -p ui/chains/snapshots + # cd ~/.sifnoded && tar -zcvf ui/chains/snapshots/sif.tar.gz + self.cmd.tar_create(self.cmd.get_user_home(".sifnoded"), project_dir("ui/chains/snapshots/sif.tar.gz"), compression="gz") + + # ui/chains/etc/snapshot.sh: + # cd ~/.ganachedb && tar -zcvf ui/chains/snapshots/eth.tar.gz + self.cmd.tar_create(self.cmd.get_user_home(".ganachedb"), project_dir("ui/chains/snapshots/eth.tar.gz"), compression="gz") def main(): From d7f8bb294daa13f5347f5aee8faffb0ea3ecd3ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 31 Jul 2021 06:06:58 +0200 Subject: [PATCH 07/35] Refactoring --- test/integration/make.py | 77 ++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index cfffb84e04..e7dca10ab9 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -161,23 +161,17 @@ class UIPlaybook: "sentence", "oppose", "avoid"] ETHEREUM_ROOT_MNEMONIC = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", "crumble", "sweet", "treat"] - CHAIN_ID = "sifchain-local" def __init__(self, cmd): self.cmd = cmd - - def eth_start(self): - return + self.chain_id = "sifchain-local" + self.keyring_backend = "test" + self.ganache_db_path = self.cmd.get_user_home(".ganachedb") + self.sifnoded_path = self.cmd.get_user_home(".sifnoded") def run(self): - # self.cmd.yarn([], cwd=project_dir()) # TODO Where? - # ganache_cli_proc = self.eth_start() - # print(repr(ganache_cli_proc)) - # self.ui_sif_launch() - # sifnoded_proc = self.ui_sif_start() - # print(repr(sifnoded_proc)) - # self.cmd.sif_wait_up("localhost", 1317) self.stack_save_snapshot() + self.stack_push() def stack_save_snapshot(self): # ui-stack.yml @@ -186,8 +180,6 @@ def stack_save_snapshot(self): # Compile smart contracts: # cd ui; yarn build - keyring_backend = "test" - # yarn stack --save-snapshot -> ui/scripts/stack.sh -> ui/scripts/stack-save-snapshot.sh # rm ui/node_modules/.migrate-complete @@ -195,32 +187,31 @@ def stack_save_snapshot(self): # ui/scripts/stack-launch.sh -> ui/scripts/_sif-build.sh -> ui/chains/sif/build.sh # killall sifnoded # rm $(which sifnoded) - self.cmd.rmdir(self.cmd.get_user_home(".sifnoded")) + self.cmd.rmdir(self.sifnoded_path) self.cmd(["make", "install"], project_dir()) # ui/scripts/stack-launch.sh -> ui/scripts/_eth.sh -> ui/chains/etc/launch.sh - self.cmd.rmdir(self.cmd.get_user_home(".ganachedb")) + self.cmd.rmdir(self.ganache_db_path) self.cmd.yarn([], cwd=project_dir("ui/chains/eth")) # Installs ui/chains/eth/node_modules # Note that this runs ganache-cli from $PATH whereas scripts start it with yarn in ui/chains/eth - ganache_proc = self.cmd.start_ganache_cli(mnemonic=UIPlaybook.ETHEREUM_ROOT_MNEMONIC, - db=self.cmd.get_user_home(".ganachedb"), port=7545, network_id=5777, gas_price=20000000000, - gas_limit=6721975, host="0.0.0.0") + ganache_proc = self.cmd.start_ganache_cli(mnemonic=UIPlaybook.ETHEREUM_ROOT_MNEMONIC, db=self.ganache_db_path, + port=7545, network_id=5777, gas_price=20000000000, gas_limit=6721975, host="0.0.0.0") # ui/scripts/stack-launch.sh -> ui/scripts/_sif.sh -> ui/chains/sif/launch.sh - self.cmd.sifnoded_init("test", UIPlaybook.CHAIN_ID) - self.cmd.copy_file(project_dir("ui", "chains", "sif", "app.toml"), self.cmd.get_user_home(".sifnoded", "config", "app.toml")) + self.cmd.sifnoded_init("test", self.chain_id) + self.cmd.copy_file(project_dir("ui/chains/sif/app.toml"), os.path.join(self.sifnoded_path, "config/app.toml")) log.info(f"Generating deterministic account - {UIPlaybook.SHADOWFIEND_NAME}...") shadowfiend_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.SHADOWFIEND_NAME, UIPlaybook.SHADOWFIEND_MNEMONIC) log.info(f"Generating deterministic account - {UIPlaybook.AKASHA_NAME}...") akasha_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.AKASHA_NAME, UIPlaybook.AKASHA_MNEMONIC) log.info(f"Generating deterministic account - {UIPlaybook.JUNIPER_NAME}...") juniper_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.JUNIPER_NAME, UIPlaybook.JUNIPER_MNEMONIC) - # shadowfiend_address = self.sifnoded_keys_show(SHADOWFIEND_NAME)[0]["address"] - # akasha_address = self.sifnoded_keys_show(AKASHA_NAME)[0]["address"] - # juniper_address = self.sifnoded_keys_show(JUNIPER_NAME)[0]["address"] shadowfiend_address = shadowfiend_account["address"] akasha_address = akasha_account["address"] juniper_address = juniper_account["address"] + assert shadowfiend_address == self.cmd.sifnoded_keys_show(UIPlaybook.SHADOWFIEND_NAME)[0]["address"] + assert akasha_address == self.cmd.sifnoded_keys_show(UIPlaybook.AKASHA_NAME)[0]["address"] + assert juniper_address == self.cmd.sifnoded_keys_show(UIPlaybook.JUNIPER_NAME)[0]["address"] tokens_shadowfiend = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] tokens_akasha = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] @@ -233,7 +224,8 @@ def stack_save_snapshot(self): self.cmd.sifnoded_add_genesis_validators(shadowfiend_address_bech_val) amount = sif_format_amount(10**24, "stake") - self.cmd.execst(["sifnoded", "gentx", UIPlaybook.SHADOWFIEND_NAME, amount, "--chain-id={}".format(UIPlaybook.CHAIN_ID), "--keyring-backend={}".format(keyring_backend)]) + self.cmd.execst(["sifnoded", "gentx", UIPlaybook.SHADOWFIEND_NAME, amount, f"--chain-id={self.chain_id}", + f"--keyring-backend={self.keyring_backend}"]) log.info("Collecting genesis txs...") self.cmd.execst(["sifnoded", "collect-gentxs"]) @@ -260,24 +252,24 @@ def stack_save_snapshot(self): # Original scripts say "if we don't sleep there are issues" time.sleep(10) log.info("Creating liquidity pool from catk:rowan...") - self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "catk", [10**5, "rowan"], 10**25, 10**25) + self.cmd.sifnoded_tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "catk", [10**5, "rowan"], 10**25, 10**25) time.sleep(5) log.info("Creating liquidity pool from cbtk:rowan...") - self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "cbtk", [10**5, "rowan"], 10**25, 10**25) + self.cmd.sifnoded_tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "cbtk", [10**5, "rowan"], 10**25, 10**25) # should now be able to swap from catk:cbtk time.sleep(5) log.info("Creating liquidity pool from ceth:rowan...") - self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "ceth", [10**5, "rowan"], 10**25, 83*10**20) + self.cmd.sifnoded_tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "ceth", [10**5, "rowan"], 10**25, 83*10**20) # should now be able to swap from x:ceth time.sleep(5) log.info("Creating liquidity pool from cusdc:rowan...") - self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "cusdc", [10**5, "rowan"], 10**25, 10**25) + self.cmd.sifnoded_tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "cusdc", [10**5, "rowan"], 10**25, 10**25) time.sleep(5) log.info("Creating liquidity pool from clink:rowan...") - self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "clink", [10**5, "rowan"], 10**25, 588235*10**18) + self.cmd.sifnoded_tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "clink", [10**5, "rowan"], 10**25, 588235*10**18) time.sleep(5) log.info("Creating liquidity pool from ctest:rowan...") - self.cmd.sifnoded_tx_clp_create_pool(UIPlaybook.CHAIN_ID, keyring_backend, "akasha", "ctest", [10**5, "rowan"], 10**25, 10**13) + self.cmd.sifnoded_tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "ctest", [10**5, "rowan"], 10**25, 10**13) # ui/scripts/_migrate.sh -> ui/chains/post_migrate.sh def get_smart_contract_address(path): @@ -322,7 +314,7 @@ def set_token_lock_burn_limit(update_address, amount): ethereum_private_key = smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"] ebrelayer_proc = self.cmd.ebrelayer_init(ethereum_private_key, "tcp://localhost:26657", "ws://localhost:7545/", bridge_registry_address, UIPlaybook.SHADOWFIEND_NAME, UIPlaybook.SHADOWFIEND_MNEMONIC, - "--chain-id={}".format(UIPlaybook.CHAIN_ID), 5*10**12, [0.5, "rowan"]) + "--chain-id={}".format(self.chain_id), 5*10**12, [0.5, "rowan"]) # At this point we have 3 running processes - ganache_proc, sifnoded_proc and ebrelayer_proc # await sif-node-up and migrate-complete @@ -330,30 +322,31 @@ def set_token_lock_burn_limit(update_address, amount): time.sleep(30) # ui/scripts/_snapshot.sh - # ui/scripts/stack-pause.sh + # ui/scripts/stack-pause.sh: # killall sifnoded sifnoded ebrelayer ganache-cli sifnoded_proc.kill() ebrelayer_proc.kill() ganache_proc.kill() time.sleep(10) + snapshots_dir = project_dir("ui/chains/snapshots") + self.cmd.mkdir(snapshots_dir) # ui/chains/peggy/snapshot.sh: - # mkdir -p ui/chains/snapshots # mkdir -p ui/chains/peggy/relayerdb - # cd ui/chains/peggy/relayerdb && tar -zcvf ui/chains/snapshots/peggy.tar.gz - self.cmd.tar_create(project_dir("ui/chains/peggy/relayerdb"), project_dir("ui/chains/snapshots/peggy.tar.gz"), compression="gz") + self.cmd.tar_create(project_dir("ui/chains/peggy/relayerdb"), os.path.join(snapshots_dir, "peggy.tar.gz"), compression="gz") # mkdir -p smart-contracts/build - # cd smart-contracts/build && tar -zcvf ui/chains/snapshots/peggy_build.tar.gz - self.cmd.tar_create(project_dir("smart-contracts/build"), project_dir("ui/chains/snapshots/peggy_build.tar.gz"), compression="gz") + self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(snapshots_dir, "peggy_build.tar.gz"), compression="gz") # ui/chains/sif/snapshot.sh: - # mkdir -p ui/chains/snapshots - # cd ~/.sifnoded && tar -zcvf ui/chains/snapshots/sif.tar.gz - self.cmd.tar_create(self.cmd.get_user_home(".sifnoded"), project_dir("ui/chains/snapshots/sif.tar.gz"), compression="gz") + self.cmd.tar_create(self.sifnoded_path, os.path.join(snapshots_dir, "sif.tar.gz"), compression="gz") # ui/chains/etc/snapshot.sh: - # cd ~/.ganachedb && tar -zcvf ui/chains/snapshots/eth.tar.gz - self.cmd.tar_create(self.cmd.get_user_home(".ganachedb"), project_dir("ui/chains/snapshots/eth.tar.gz"), compression="gz") + self.cmd.tar_create(self.ganache_db_path, os.path.join(snapshots_dir, "eth.tar.gz"), compression="gz") + + def stack_push(self): + # ui/scripts/stack-push.sh + # $PWD=ui + pass def main(): From 23fb4fdafa34d79e4aa62e9bee9833ee93fc78ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 31 Jul 2021 06:39:27 +0200 Subject: [PATCH 08/35] Refactoring --- test/integration/make.py | 108 ++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 36 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index e7dca10ab9..b12dc49269 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -12,7 +12,15 @@ def stdout_lines(res): - return res[0].splitlines() + return res[1].splitlines() + +def exactly_one(items): + if len(items) == 0: + raise ValueError("Zero items") + elif len(items) > 1: + raise ValueError("Multiple items") + else: + return items[0] def project_dir(*paths): return os.path.abspath(os.path.join(os.path.normpath(os.path.join(__file__, *([os.path.pardir]*3))), *paths)) @@ -146,22 +154,6 @@ def yarn(self, args, cwd=None, env=None): class UIPlaybook: - # From ui/chains/credentials.sh - SHADOWFIEND_NAME = "shadowfiend" - SHADOWFIEND_MNEMONIC = ["race", "draft", "rival", "universe", "maid", "cheese", "steel", "logic", "crowd", "fork", - "comic", "easy", "truth", "drift", "tomorrow", "eye", "buddy", "head", "time", "cash", "swing", "swift", - "midnight", "borrow"] - AKASHA_NAME = "akasha" - AKASHA_MNEMONIC = ["hand", "inmate", "canvas", "head", "lunar", "naive", "increase", "recycle", "dog", "ecology", - "inhale", "december", "wide", "bubble", "hockey", "dice", "worth", "gravity", "ketchup", "feed", "balance", - "parent", "secret", "orchard"] - JUNIPER_NAME = "juniper" - JUNIPER_MNEMONIC = ["clump", "genre", "baby", "drum", "canvas", "uncover", "firm", "liberty", "verb", "moment", - "access", "draft", "erupt", "fog", "alter", "gadget", "elder", "elephant", "divide", "biology", "choice", - "sentence", "oppose", "avoid"] - ETHEREUM_ROOT_MNEMONIC = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", - "crumble", "sweet", "treat"] - def __init__(self, cmd): self.cmd = cmd self.chain_id = "sifchain-local" @@ -169,9 +161,21 @@ def __init__(self, cmd): self.ganache_db_path = self.cmd.get_user_home(".ganachedb") self.sifnoded_path = self.cmd.get_user_home(".sifnoded") - def run(self): - self.stack_save_snapshot() - self.stack_push() + # From ui/chains/credentials.sh + self.shadowfiend_name = "shadowfiend" + self.shadowfiend_mnemonic = ["race", "draft", "rival", "universe", "maid", "cheese", "steel", "logic", "crowd", + "fork", "comic", "easy", "truth", "drift", "tomorrow", "eye", "buddy", "head", "time", "cash", "swing", + "swift", "midnight", "borrow"] + self.akasha_name = "akasha" + self.akasha_mnemonic = ["hand", "inmate", "canvas", "head", "lunar", "naive", "increase", "recycle", "dog", + "ecology", "inhale", "december", "wide", "bubble", "hockey", "dice", "worth", "gravity", "ketchup", "feed", + "balance", "parent", "secret", "orchard"] + self.juniper_name = "juniper" + self.juniper_mnemonic = ["clump", "genre", "baby", "drum", "canvas", "uncover", "firm", "liberty", "verb", + "moment", "access", "draft", "erupt", "fog", "alter", "gadget", "elder", "elephant", "divide", "biology", + "choice", "sentence", "oppose", "avoid"] + self.ethereum_root_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", + "crumble", "sweet", "treat"] def stack_save_snapshot(self): # ui-stack.yml @@ -194,24 +198,24 @@ def stack_save_snapshot(self): self.cmd.rmdir(self.ganache_db_path) self.cmd.yarn([], cwd=project_dir("ui/chains/eth")) # Installs ui/chains/eth/node_modules # Note that this runs ganache-cli from $PATH whereas scripts start it with yarn in ui/chains/eth - ganache_proc = self.cmd.start_ganache_cli(mnemonic=UIPlaybook.ETHEREUM_ROOT_MNEMONIC, db=self.ganache_db_path, + ganache_proc = self.cmd.start_ganache_cli(mnemonic=self.ethereum_root_mnemonic, db=self.ganache_db_path, port=7545, network_id=5777, gas_price=20000000000, gas_limit=6721975, host="0.0.0.0") # ui/scripts/stack-launch.sh -> ui/scripts/_sif.sh -> ui/chains/sif/launch.sh self.cmd.sifnoded_init("test", self.chain_id) self.cmd.copy_file(project_dir("ui/chains/sif/app.toml"), os.path.join(self.sifnoded_path, "config/app.toml")) - log.info(f"Generating deterministic account - {UIPlaybook.SHADOWFIEND_NAME}...") - shadowfiend_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.SHADOWFIEND_NAME, UIPlaybook.SHADOWFIEND_MNEMONIC) - log.info(f"Generating deterministic account - {UIPlaybook.AKASHA_NAME}...") - akasha_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.AKASHA_NAME, UIPlaybook.AKASHA_MNEMONIC) - log.info(f"Generating deterministic account - {UIPlaybook.JUNIPER_NAME}...") - juniper_account = self.cmd.sifnoded_generate_deterministic_account(UIPlaybook.JUNIPER_NAME, UIPlaybook.JUNIPER_MNEMONIC) + log.info(f"Generating deterministic account - {self.shadowfiend_name}...") + shadowfiend_account = self.cmd.sifnoded_generate_deterministic_account(self.shadowfiend_name, self.shadowfiend_mnemonic) + log.info(f"Generating deterministic account - {self.akasha_name}...") + akasha_account = self.cmd.sifnoded_generate_deterministic_account(self.akasha_name, self.akasha_mnemonic) + log.info(f"Generating deterministic account - {self.juniper_name}...") + juniper_account = self.cmd.sifnoded_generate_deterministic_account(self.juniper_name, self.juniper_mnemonic) shadowfiend_address = shadowfiend_account["address"] akasha_address = akasha_account["address"] juniper_address = juniper_account["address"] - assert shadowfiend_address == self.cmd.sifnoded_keys_show(UIPlaybook.SHADOWFIEND_NAME)[0]["address"] - assert akasha_address == self.cmd.sifnoded_keys_show(UIPlaybook.AKASHA_NAME)[0]["address"] - assert juniper_address == self.cmd.sifnoded_keys_show(UIPlaybook.JUNIPER_NAME)[0]["address"] + assert shadowfiend_address == self.cmd.sifnoded_keys_show(self.shadowfiend_name)[0]["address"] + assert akasha_address == self.cmd.sifnoded_keys_show(self.akasha_name)[0]["address"] + assert juniper_address == self.cmd.sifnoded_keys_show(self.juniper_name)[0]["address"] tokens_shadowfiend = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] tokens_akasha = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] @@ -220,11 +224,11 @@ def stack_save_snapshot(self): self.cmd.sifnoded_add_genesis_account(akasha_address, tokens_akasha) self.cmd.sifnoded_add_genesis_account(juniper_address, tokens_juniper) - shadowfiend_address_bech_val = self.cmd.sifnoded_keys_show(UIPlaybook.SHADOWFIEND_NAME, bech="val")[0]["address"] + shadowfiend_address_bech_val = self.cmd.sifnoded_keys_show(self.shadowfiend_name, bech="val")[0]["address"] self.cmd.sifnoded_add_genesis_validators(shadowfiend_address_bech_val) amount = sif_format_amount(10**24, "stake") - self.cmd.execst(["sifnoded", "gentx", UIPlaybook.SHADOWFIEND_NAME, amount, f"--chain-id={self.chain_id}", + self.cmd.execst(["sifnoded", "gentx", self.shadowfiend_name, amount, f"--chain-id={self.chain_id}", f"--keyring-backend={self.keyring_backend}"]) log.info("Collecting genesis txs...") @@ -313,8 +317,8 @@ def set_token_lock_burn_limit(update_address, amount): # rm -rf ui/chains/peggy/relayerdb ethereum_private_key = smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"] ebrelayer_proc = self.cmd.ebrelayer_init(ethereum_private_key, "tcp://localhost:26657", "ws://localhost:7545/", - bridge_registry_address, UIPlaybook.SHADOWFIEND_NAME, UIPlaybook.SHADOWFIEND_MNEMONIC, - "--chain-id={}".format(self.chain_id), 5*10**12, [0.5, "rowan"]) + bridge_registry_address, self.shadowfiend_name, self.shadowfiend_mnemonic, f"--chain-id={self.chain_id}", + 5*10**12, [0.5, "rowan"]) # At this point we have 3 running processes - ganache_proc, sifnoded_proc and ebrelayer_proc # await sif-node-up and migrate-complete @@ -346,13 +350,45 @@ def set_token_lock_burn_limit(update_address, amount): def stack_push(self): # ui/scripts/stack-push.sh # $PWD=ui - pass + + # User must be logged in to Docker hub: + # ~/.docker/config.json must exist and .auths['ghcr.io'].auth != null + commit = exactly_one(stdout_lines(self.cmd.execst(["git", "rev-parse", "HEAD"], cwd=project_dir()))) + branch = exactly_one(stdout_lines(self.cmd.execst(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=project_dir()))) + + image_root = "ghcr.io/sifchain/sifnode/ui-stack" + image_name = "{}:{}".format(image_root, commit) + stable_tag = "{}:{}".format(image_root, branch.replace("/", "__")) + + running_in_ci = bool(os.environ.get("CI")) + + if running_in_ci: + # # reverse grep for go.mod because on CI this can be altered by installing go dependencies + # if [[ -z "$CI" && ! -z "$(git status --porcelain --untracked-files=no)" ]]; then + # echo "Git workspace must be clean to save git commit hash" + # exit 1 + # fi + pass + + log.info("Github Registry Login found.") + log.info("Building new container...") + log.info(f"New image name: {image_name}") + + self.cmd.execst(["docker", "build", "-f", project_dir("ui/scripts/stack.Dockerfile"), "-t", image_name, "."], + cwd=project_dir(), env={"DOCKER_BUILDKIT", "1"}) + + if running_in_ci: + log.info(f"Tagging image as {stable_tag}...") + self.cmd.execst(["docker", "tag", image_name, stable_tag]) + self.cmd.execst(["docker", "push", image_name]) + self.cmd.execst(["docker", "push", stable_tag]) def main(): cmd = Integrator() ui_playbook = UIPlaybook(cmd) - ui_playbook.run() + ui_playbook.stack_save_snapshot() + ui_playbook.stack_push() if __name__ == "__main__": main() From 8b68f40ec91d6a9f5891b244adcc638047692761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 31 Jul 2021 10:15:41 +0200 Subject: [PATCH 09/35] Proof of concept --- test/integration/make.py | 66 +++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index b12dc49269..1a4621e196 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -3,14 +3,14 @@ import os import shutil import subprocess -import yaml # pip install pyyaml -import urllib.request +import sys import time +import urllib.request +import yaml # pip install pyyaml log = logging.getLogger(__name__) - def stdout_lines(res): return res[1].splitlines() @@ -36,17 +36,29 @@ def http_get(url): return r.read() def popen(args, env=None): + if env: + env = dict_merge(os.environ, env) return subprocess.Popen(args, env=env) +def dict_merge(*dicts): + result = {} + for d in dicts: + for k, v in d.items(): + result[k] = v + return result + NULL_ADDRESS = "0x0000000000000000000000000000000000000000" class Command: - def execst(self, args, cwd=None, env=None, stdin=None, binary=False): + def execst(self, args, cwd=None, env=None, stdin=None, binary=False, pipe=True): if stdin is not None: if type(stdin) == list: stdin = "".join([line + "\n" for line in stdin]) - popen = subprocess.Popen(args, cwd=cwd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=not binary) + p = subprocess.PIPE if pipe else None + if env: + env = dict_merge(os.environ, env) + popen = subprocess.Popen(args, cwd=cwd, env=env, stdin=subprocess.PIPE, stdout=p, stderr=p, text=not binary) stdout_data, stderr_data = popen.communicate(input=stdin) if popen.returncode != 0: raise Exception("Command '{}' exited with returncode {}: {}".format(" ".join(args), popen.returncode, repr(stderr_data))) @@ -56,6 +68,9 @@ def read_text_file(self, path): with open(path, "rt") as f: return f.read() # TODO Convert to exec + def mkdir(self, path): + os.makedirs(path, exist_ok=True) + def rmdir(self, path): if os.path.exists(path): shutil.rmtree(path) # TODO Convert to exec @@ -66,11 +81,23 @@ def copy_file(self, src, dst): def get_user_home(self, *paths): return os.path.join(os.environ["HOME"], *paths) + def mktempdir(self): + return exactly_one(stdout_lines(self.execst(["mktemp", "-d"]))) + def tar_create(self, path, tar_path, compression=None): comp_opts = {"gz": "z"} comp = comp_opts[compression] if compression in comp_opts else "" - args = ["tar", "cf" + comp, tar_path, "."] - self.execst(args, cwd=path) + # tar on 9p filesystem reports "file shrank by ... bytes" and exits with errorcode 1 + tar_quirks = True + if tar_quirks: + tmpdir = self.mktempdir() + try: + shutil.copytree(path, tmpdir, dirs_exist_ok=True) + self.execst(["tar", "cf" + comp, tar_path, "."], cwd=tmpdir) + finally: + self.rmdir(tmpdir) + else: + self.execst(["tar", "cf" + comp, tar_path, "."], cwd=path) class Ganache(Command): @@ -89,7 +116,8 @@ def start_ganache_cli(self, mnemonic=None, db=None, port=None, host=None, networ class Sifnoded(Command): def sifnoded_init(self, moniker, chain_id): args = ["sifnoded", "init", moniker, "--chain-id={}".format(chain_id)] - self.execst(args) + res = self.execst(args) + return json.loads(res[2]) # output is on stderr def sifnoded_generate_deterministic_account(self, name, mnemonic): args = ["sifnoded", "keys", "add", name, "--keyring-backend={}".format("test"), "--recover"] @@ -108,7 +136,7 @@ def sifnoded_keys_show(self, name, bech=None, keyring_backend=None): def sifnoded_add_genesis_account(self, address, tokens): tokens_str = ",".join([sif_format_amount(amount, denom) for amount, denom in tokens]) args = ["sifnoded", "add-genesis-account", address, tokens_str] - self.execst(args) + self.execst(args, pipe=False) def sifnoded_add_genesis_validators(self, address): args = ["sifnoded", "add-genesis-validators", address] @@ -120,7 +148,8 @@ def sifnoded_tx_clp_create_pool(self, chain_id, keyring_backend, from_name, symb "--keyring-backend={}".format(keyring_backend), "--from", from_name, "--symbol", symbol, "--fees", sif_format_amount(*fees), "--nativeAmount", str(native_amount), "--externalAmount", str(external_amount), "--yes"] - self.execst(args) + res = self.execst(args) + return yaml_load(res[1]) def sifnoded_launch(self, minimum_gas_prices=None): args = ["sifnoded", "start"] + \ @@ -137,8 +166,8 @@ def ebrelayer_init(self, ethereum_private_key, tendermind_node, web3_provider, b validator_moniker, validator_mnemonic, chain_id, gas, gas_prices): env = {"ETHEREUM_PRIVATE_KEY": ethereum_private_key} args = ["ebrelayer", "init", tendermind_node, web3_provider, bridge_registry_contract_address, - validator_moniker, validator_mnemonic, "--chain-id={}".format(chain_id), "--gas", str(gas), "--gas-prices", - sif_format_amount(*gas_prices)] + validator_moniker, " ".join(validator_mnemonic), "--chain-id={}".format(chain_id), "--gas", str(gas), + "--gas-prices", sif_format_amount(*gas_prices)] return popen(args, env=env) def sif_wait_up(self, host, port): @@ -150,7 +179,7 @@ def sif_wait_up(self, host, port): time.sleep(1) def yarn(self, args, cwd=None, env=None): - return self.execst(["yarn"] + args, cwd=cwd, env=env) + return self.execst(["yarn"] + args, cwd=cwd, env=env, pipe=False) class UIPlaybook: @@ -192,7 +221,7 @@ def stack_save_snapshot(self): # killall sifnoded # rm $(which sifnoded) self.cmd.rmdir(self.sifnoded_path) - self.cmd(["make", "install"], project_dir()) + self.cmd.execst(["make", "install"], cwd=project_dir(), pipe=False) # ui/scripts/stack-launch.sh -> ui/scripts/_eth.sh -> ui/chains/etc/launch.sh self.cmd.rmdir(self.ganache_db_path) @@ -311,10 +340,11 @@ def set_token_lock_burn_limit(update_address, amount): # Whitelist test tokens for addr in [atk_address, btk_address, usdc_address, link_address]: - self.cmd.yarn(["peggy:whitelist", addr, "true"], cwd=smart_contracts_dir) + self.cmd.yarn(["peggy:whiteList", addr, "true"], cwd=smart_contracts_dir) # ui/scripts/stack-launch.sh -> ui/scripts/_peggy.sh -> ui/chains/peggy/launch.sh # rm -rf ui/chains/peggy/relayerdb + # ebrelayer is in $GOBIN, gets installed by "make install" ethereum_private_key = smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"] ebrelayer_proc = self.cmd.ebrelayer_init(ethereum_private_key, "tcp://localhost:26657", "ws://localhost:7545/", bridge_registry_address, self.shadowfiend_name, self.shadowfiend_mnemonic, f"--chain-id={self.chain_id}", @@ -334,7 +364,7 @@ def set_token_lock_burn_limit(update_address, amount): time.sleep(10) snapshots_dir = project_dir("ui/chains/snapshots") - self.cmd.mkdir(snapshots_dir) + self.cmd.mkdir(snapshots_dir) # TODO self.cmd.rmdir(snapshots_dir) # ui/chains/peggy/snapshot.sh: # mkdir -p ui/chains/peggy/relayerdb self.cmd.tar_create(project_dir("ui/chains/peggy/relayerdb"), os.path.join(snapshots_dir, "peggy.tar.gz"), compression="gz") @@ -363,6 +393,7 @@ def stack_push(self): running_in_ci = bool(os.environ.get("CI")) if running_in_ci: + res = self.cmd.execst(["git", "status", "--porcelain", "--untracked-files=no"], cwd=project_dir()) # # reverse grep for go.mod because on CI this can be altered by installing go dependencies # if [[ -z "$CI" && ! -z "$(git status --porcelain --untracked-files=no)" ]]; then # echo "Git workspace must be clean to save git commit hash" @@ -375,7 +406,7 @@ def stack_push(self): log.info(f"New image name: {image_name}") self.cmd.execst(["docker", "build", "-f", project_dir("ui/scripts/stack.Dockerfile"), "-t", image_name, "."], - cwd=project_dir(), env={"DOCKER_BUILDKIT", "1"}) + cwd=project_dir(), env={"DOCKER_BUILDKIT": "1"}, pipe=False) if running_in_ci: log.info(f"Tagging image as {stable_tag}...") @@ -385,6 +416,7 @@ def stack_push(self): def main(): + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(message)s") cmd = Integrator() ui_playbook = UIPlaybook(cmd) ui_playbook.stack_save_snapshot() From 76f5e03d43a87597f350768c46ebcd3483bb028c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 31 Jul 2021 20:42:14 +0200 Subject: [PATCH 10/35] Assimilate scripts from test/integration --- test/integration/make.py | 202 +++++++++++++++++++++++++++++++++++---- 1 file changed, 183 insertions(+), 19 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 1a4621e196..67b8bdfff9 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -1,6 +1,7 @@ import json import logging import os +import re import shutil import subprocess import sys @@ -64,6 +65,9 @@ def execst(self, args, cwd=None, env=None, stdin=None, binary=False, pipe=True): raise Exception("Command '{}' exited with returncode {}: {}".format(" ".join(args), popen.returncode, repr(stderr_data))) return popen.returncode, stdout_data, stderr_data + def rm(self, path): + os.remove(path) + def read_text_file(self, path): with open(path, "rt") as f: return f.read() # TODO Convert to exec @@ -78,12 +82,18 @@ def rmdir(self, path): def copy_file(self, src, dst): shutil.copy(src, dst) + def exists(self, path): + return os.path.exists(path) + def get_user_home(self, *paths): return os.path.join(os.environ["HOME"], *paths) def mktempdir(self): return exactly_one(stdout_lines(self.execst(["mktemp", "-d"]))) + def mktempfile(self): + return exactly_one(stdout_lines(self.execst(["mktemp"]))) + def tar_create(self, path, tar_path, compression=None): comp_opts = {"gz": "z"} comp = comp_opts[compression] if compression in comp_opts else "" @@ -101,7 +111,8 @@ def tar_create(self, path, tar_path, compression=None): class Ganache(Command): - def start_ganache_cli(self, mnemonic=None, db=None, port=None, host=None, network_id=None, gas_price=None, gas_limit=None): + def start_ganache_cli(self, mnemonic=None, db=None, port=None, host=None, network_id=None, gas_price=None, + gas_limit=None, block_time=None, account_keys_path=None, popen_args=None): args = ["ganache-cli"] + \ (["--mnemonic", " ".join(mnemonic)] if mnemonic else []) + \ (["--db", db] if db else []) + \ @@ -109,8 +120,10 @@ def start_ganache_cli(self, mnemonic=None, db=None, port=None, host=None, networ (["--host", host] if host else []) + \ (["--networkId", str(network_id)] if network_id is not None else []) + \ (["--gasPrice", str(gas_price)] if gas_price is not None else []) + \ - (["--gasLimit", str(gas_limit)] if gas_limit is not None else []) - return popen(args) + (["--gasLimit", str(gas_limit)] if gas_limit is not None else []) + \ + (["--blockTime", str(block_time)] if block_time is not None else []) + \ + (["--account_keys_path", account_keys_path] if account_keys_path is not None else []) + return popen(args, **(popen_args if popen_args is not None else dict())) class Sifnoded(Command): @@ -162,6 +175,9 @@ def sifnoded_get_status(self, host, port): class Integrator(Ganache, Sifnoded, Command): + def __init__(self): + self.smart_contracts_dir = project_dir("smart-contracts") + def ebrelayer_init(self, ethereum_private_key, tendermind_node, web3_provider, bridge_registry_contract_address, validator_moniker, validator_mnemonic, chain_id, gas, gas_prices): env = {"ETHEREUM_PRIVATE_KEY": ethereum_private_key} @@ -181,11 +197,97 @@ def sif_wait_up(self, host, port): def yarn(self, args, cwd=None, env=None): return self.execst(["yarn"] + args, cwd=cwd, env=env, pipe=False) - -class UIPlaybook: + def primitive_parse_env_file(self, path): + def split(lines): + result = dict() + for line in lines: + m = patt.match(line) + result[m[1]] = m[2] + return result + + tmp1 = self.mktempfile() + tmp2 = self.mktempfile() + try: + self.execst(["bash", "-c", "set -o posix; IFS=' '; set > {}; source {}; set > {}".format(tmp1, path, tmp2)]) + t1 = self.read_text_file(tmp1).splitlines() + t2 = self.read_text_file(tmp2).splitlines() + finally: + self.rm(tmp1) + self.rm(tmp2) + patt = re.compile("^(.*?)=(.*)$") + d1 = split(t1) + d2 = split(t2) + result = dict() + for k, v in d2.items(): + if (k in d1) and (d1[k] == d2[k]): + continue + if k in ["_", "BASH_ARGC"]: + continue + result[k] = v + return result + + def build_smart_contracts_for_integration_tests(self): + self.execst(["make", "clean-smartcontracts"], cwd=self.smart_contracts_dir) + self.yarn(["install"], cwd=self.smart_contracts_dir) + + def deploy_smart_contracts_for_integration_tests(self, network_name, consensus_threshold=None, operator=None, + owner=None, initial_validator_addresses=None, initial_validator_powers=None, pauser=None, + mainnet_gas_price=None, env_file=None + ): + env = {} + if consensus_threshold is not None: # required + env["CONSENSUS_THRESHOLD"] = str(consensus_threshold) + if operator is not None: # required + env["OPERATOR"] = operator + if owner is not None: # required + env["OWNER"] = owner + if initial_validator_addresses is not None: + env["INITIAL_VALIDATOR_ADDRESSES"] = ",".join(initial_validator_addresses) + if initial_validator_powers is not None: + env["INITIAL_VALIDATOR_POWERS"] = ",".join(initial_validator_powers) + if pauser is not None: + env["PAUSER"] = pauser + if mainnet_gas_price is not None: + env["MAINNET_GAS_PRICE"] = mainnet_gas_price + + env_path = project_dir("smart-contracts/.env") + if env_file is not None: + self.copy_file(env_file, env_path) + + if self.exists(env_path): + fenv = self.primitive_parse_env_file(env_file) + for k, v in env.items(): + if k in fenv: + if env[k] == fenv[k]: + log.warning(f"Variable '{k}' specified both as a parameter and in '{env_path}'") + else: + log.warning(f"Variable '{k}' has different values, parameter: {env[k]}, in '{env_path}': " + f"{fenv[k]}. According to observation, value from parameter will be used.") + + # TODO ui scripts use just "yarn; yarn migrate" alias "npx truffle migrate --reset", + self.execst(["npx", "truffle", "deploy", "--network", network_name, "--reset"], env=env, + cwd=self.smart_contracts_dir, pipe=False) + + def deploy_smart_contracts_for_ui_stack(self): + self.copy_file(os.path.join(self.smart_contracts_dir, ".env.ui.example"), os.path.join(self.smart_contracts_dir, ".env")) + # TODO Might not be neccessary + self.yarn([], cwd=self.smart_contracts_dir) + self.yarn(["migrate"], cwd=self.smart_contracts_dir) + + def get_smart_contract_address(self, compiled_json_path, network_id): + return json.loads(self.read_text_file(compiled_json_path))["networks"][str(network_id)]["address"] + + def get_bridge_smart_contract_addresses(self, network_id): + return [self.get_smart_contract_address(os.path.join( + self.smart_contracts_dir, f"build/contracts/{x}.json"), network_id) + for x in ["BridgeToken", "BridgeRegistry", "BridgeBank"]] + +class UIStackPlaybook: def __init__(self, cmd): self.cmd = cmd self.chain_id = "sifchain-local" + self.network_name = "develop" + self.network_id = 5777 self.keyring_backend = "test" self.ganache_db_path = self.cmd.get_user_home(".ganachedb") self.sifnoded_path = self.cmd.get_user_home(".sifnoded") @@ -228,7 +330,7 @@ def stack_save_snapshot(self): self.cmd.yarn([], cwd=project_dir("ui/chains/eth")) # Installs ui/chains/eth/node_modules # Note that this runs ganache-cli from $PATH whereas scripts start it with yarn in ui/chains/eth ganache_proc = self.cmd.start_ganache_cli(mnemonic=self.ethereum_root_mnemonic, db=self.ganache_db_path, - port=7545, network_id=5777, gas_price=20000000000, gas_limit=6721975, host="0.0.0.0") + port=7545, network_id=self.network_id, gas_price=20000000000, gas_limit=6721975, host="0.0.0.0") # ui/scripts/stack-launch.sh -> ui/scripts/_sif.sh -> ui/chains/sif/launch.sh self.cmd.sifnoded_init("test", self.chain_id) @@ -272,10 +374,7 @@ def stack_save_snapshot(self): self.cmd.sif_wait_up("localhost", 1317) # ui/scripts/_migrate.sh -> ui/chains/peggy/migrate.sh - smart_contracts_dir = project_dir("smart-contracts") - self.cmd.copy_file(os.path.join(smart_contracts_dir, ".env.ui.example"), os.path.join(smart_contracts_dir, ".env")) - self.cmd.yarn([], cwd=smart_contracts_dir) - self.cmd.yarn(["migrate"], cwd=smart_contracts_dir) + self.cmd.deploy_smart_contracts_for_ui_stack() # ui/scripts/_migrate.sh -> ui/chains/eth/migrate.sh # send through atk and btk tokens to eth chain @@ -305,15 +404,13 @@ def stack_save_snapshot(self): self.cmd.sifnoded_tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "ctest", [10**5, "rowan"], 10**25, 10**13) # ui/scripts/_migrate.sh -> ui/chains/post_migrate.sh - def get_smart_contract_address(path): - return json.loads(self.cmd.read_text_file(project_dir(path)))["networks"]["5777"]["address"] atk_address, btk_address, usdc_address, link_address = [ - get_smart_contract_address(project_dir(f"ui/chains/eth/build/contracts/{x}.json")) + self.cmd.get_smart_contract_address(project_dir(f"ui/chains/eth/build/contracts/{x}.json"), self.network_id) for x in ["AliceToken", "BobToken", "UsdCoin", "LinkCoin"] ] - bridge_token_address = get_smart_contract_address(project_dir("smart-contracts/build/contracts/BridgeToken.json")) - bridge_registry_address = get_smart_contract_address(project_dir("smart-contracts/build/contracts/BridgeRegistry.json")) + + bridge_token_address, bridge_registry_address, bridge_bank = self.cmd.get_bridge_smart_contract_addresses(self.network_id) # From smart-contracts/.env.ui.example smart_contracts_env_ui_example_vars = { @@ -328,7 +425,7 @@ def set_token_lock_burn_limit(update_address, amount): # Needs: ETHEREUM_PRIVATE_KEY, INFURA_PROJECT_ID, LOCAL_PROVIDER, UPDATE_ADDRESS # Hint: call web3 directly, avoid npx + truffle + script # Maybe: self.cmd.yarn(["integrationtest:setTokenLockBurnLimit", str(amount)]) - self.cmd.execst(["npx", "truffle", "exec", "scripts/setTokenLockBurnLimit.js", str(amount)], env=env, cwd=smart_contracts_dir) + self.cmd.execst(["npx", "truffle", "exec", "scripts/setTokenLockBurnLimit.js", str(amount)], env=env, cwd=self.cmd.smart_contracts_dir) set_token_lock_burn_limit(NULL_ADDRESS, 31*10**18) set_token_lock_burn_limit(bridge_token_address, 10**25) @@ -340,7 +437,7 @@ def set_token_lock_burn_limit(update_address, amount): # Whitelist test tokens for addr in [atk_address, btk_address, usdc_address, link_address]: - self.cmd.yarn(["peggy:whiteList", addr, "true"], cwd=smart_contracts_dir) + self.cmd.yarn(["peggy:whiteList", addr, "true"], cwd=self.cmd.smart_contracts_dir) # ui/scripts/stack-launch.sh -> ui/scripts/_peggy.sh -> ui/chains/peggy/launch.sh # rm -rf ui/chains/peggy/relayerdb @@ -383,6 +480,8 @@ def stack_push(self): # User must be logged in to Docker hub: # ~/.docker/config.json must exist and .auths['ghcr.io'].auth != null + log.info("Github Registry Login found.") + commit = exactly_one(stdout_lines(self.cmd.execst(["git", "rev-parse", "HEAD"], cwd=project_dir()))) branch = exactly_one(stdout_lines(self.cmd.execst(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=project_dir()))) @@ -401,7 +500,6 @@ def stack_push(self): # fi pass - log.info("Github Registry Login found.") log.info("Building new container...") log.info(f"New image name: {image_name}") @@ -415,12 +513,78 @@ def stack_push(self): self.cmd.execst(["docker", "push", stable_tag]) +class IntegrationTestsPlaybook: + def __init__(self, cmd): + self.cmd = cmd + self.ethereum_private_key = "c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3" + self.owner = "0x627306090abaB3A6e1400e9345bC60c78a8BEf57" + # we may eventually switch things so PAUSER and OWNER aren't the same account, but for now they're the same + self.pauser = self.owner + # set_persistant_env_var BASEDIR $(fullpath $BASEDIR) $envexportfile + # set_persistant_env_var SIFCHAIN_BIN $BASEDIR/cmd $envexportfile + # set_persistant_env_var envexportfile $(fullpath $envexportfile) $envexportfile + # set_persistant_env_var TEST_INTEGRATION_DIR ${BASEDIR}/test/integration $envexportfile + # set_persistant_env_var TEST_INTEGRATION_PY_DIR ${BASEDIR}/test/integration/src/py $envexportfile + # set_persistant_env_var SMART_CONTRACTS_DIR ${BASEDIR}/smart-contracts $envexportfile + # set_persistant_env_var datadir ${TEST_INTEGRATION_DIR}/vagrant/data $envexportfile + # set_persistant_env_var CONTAINER_NAME integration_sifnode1_1 $envexportfile + # set_persistant_env_var NETWORKDIR $BASEDIR/deploy/networks $envexportfile + # set_persistant_env_var GANACHE_DB_DIR $(mktemp -d /tmp/ganachedb.XXXX) $envexportfile + # set_persistant_env_var ETHEREUM_WEBSOCKET_ADDRESS ws://localhost:7545/ $envexportfile + # set_persistant_env_var CHAINNET localnet $envexportfile + self.network_name = "develop" + self.network_id = 5777 + + def run(self): + integration_tests_dir = project_dir("test/integration") + data_dir = project_dir("test/integration/vagrant/data") + + # make go binaries (a lot of nonsense!) + self.cmd.execst(["make"], cwd=integration_tests_dir, env={"BASEDIR": project_dir()}) + + self.cmd.build_smart_contracts_for_integration_tests() + + # test/integration/ganache-start.sh: + # 1. pkill -9 -f ganache-cli || true + # 2. while nc -z localhost 7545; do sleep 1; done + # 3. nohup tmux new-session -d -s my_session "ganache-cli ${block_delay} -h 0.0.0.0 --mnemonic \ + # 'candy maple cake sugar pudding cream honey rich smooth crumble sweet treat' \ + # --networkId '5777' --port '7545' --db ${GANACHE_DB_DIR} --account_keys_path $GANACHE_KEYS_JSON \ + # > $GANACHE_LOG 2>&1" + # 4. sleep 5 + # 5. while ! nc -z localhost 4545; do sleep 5; done + # GANACHE_LOG=ui/test/integration/vagrant/data/logs/ganache.$(filenamedate).txt + mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", "crumble", "sweet", "treat"] + block_time = None # TODO + account_keys_path = os.path.join(data_dir, "ganachekeys.json") + ganache_db_path = self.cmd.mktempdir() + self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=mnemonic, network_id=self.network_id, + port=7545, db=ganache_db_path, account_keys_path=account_keys_path) + + ganache_keys = json.loads(self.cmd.read_text_file(account_keys_path)) + ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] + ebrelayer_ethereum_private_key = ganache_keys["private_keys"][ebrelayer_ethereum_addr] + # TODO Check for possible non-determinism of dict().keys() ordering (c.f. test/integration/vagrantenv.sh) + # TODO ebrelayer_ethereum_private_key is NOT the same as in test/integration/.env.ciExample + assert ebrelayer_ethereum_addr == "0x5aeda56215b167893e80b4fe645ba6d5bab767de" + assert ebrelayer_ethereum_private_key == "8d5366123cb560bb606379f90a0bfd4769eecc0557f1b362dcae9012b548b1e5" + + self.cmd.deploy_smart_contracts_for_integration_tests(self.network_name, owner=self.owner, pauser=self.pauser, + initial_validator_addresses=[ebrelayer_ethereum_addr], + env_file=project_dir("test/integration/.env.ciExample")) + + bridge_token_address, bridge_registry_address, bridge_bank = self.cmd.get_bridge_smart_contract_addresses(self.network_id) + return + + def main(): logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(message)s") cmd = Integrator() - ui_playbook = UIPlaybook(cmd) + ui_playbook = UIStackPlaybook(cmd) ui_playbook.stack_save_snapshot() ui_playbook.stack_push() + it_playbook = IntegrationTestsPlaybook(cmd) + # it_playbook.run() if __name__ == "__main__": main() From 000ecb77e37d0dcb639ec1861983d97c447c128a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 31 Jul 2021 23:14:04 +0200 Subject: [PATCH 11/35] Refactoring --- test/integration/make.py | 101 ++++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 67b8bdfff9..2e05ddda87 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -230,6 +230,18 @@ def build_smart_contracts_for_integration_tests(self): self.execst(["make", "clean-smartcontracts"], cwd=self.smart_contracts_dir) self.yarn(["install"], cwd=self.smart_contracts_dir) + def _check_env_vs_file(self, env, env_path): + if (not self.exists(env_path)) or (env is None): + return + fenv = self.primitive_parse_env_file(env_path) + for k, v in env.items(): + if k in fenv: + if env[k] == fenv[k]: + log.warning(f"Variable '{k}' specified both as a parameter and in '{env_path}'") + else: + log.warning(f"Variable '{k}' has different values, parameter: {env[k]}, in '{env_path}': " + f"{fenv[k]}. According to observation, value from parameter will be used.") + def deploy_smart_contracts_for_integration_tests(self, network_name, consensus_threshold=None, operator=None, owner=None, initial_validator_addresses=None, initial_validator_powers=None, pauser=None, mainnet_gas_price=None, env_file=None @@ -250,19 +262,11 @@ def deploy_smart_contracts_for_integration_tests(self, network_name, consensus_t if mainnet_gas_price is not None: env["MAINNET_GAS_PRICE"] = mainnet_gas_price - env_path = project_dir("smart-contracts/.env") + env_path = os.path.join(self.smart_contracts_dir, ".env") if env_file is not None: self.copy_file(env_file, env_path) - if self.exists(env_path): - fenv = self.primitive_parse_env_file(env_file) - for k, v in env.items(): - if k in fenv: - if env[k] == fenv[k]: - log.warning(f"Variable '{k}' specified both as a parameter and in '{env_path}'") - else: - log.warning(f"Variable '{k}' has different values, parameter: {env[k]}, in '{env_path}': " - f"{fenv[k]}. According to observation, value from parameter will be used.") + self._check_env_vs_file(env, env_path) # TODO ui scripts use just "yarn; yarn migrate" alias "npx truffle migrate --reset", self.execst(["npx", "truffle", "deploy", "--network", network_name, "--reset"], env=env, @@ -282,6 +286,24 @@ def get_bridge_smart_contract_addresses(self, network_id): self.smart_contracts_dir, f"build/contracts/{x}.json"), network_id) for x in ["BridgeToken", "BridgeRegistry", "BridgeBank"]] + def truffle_exec(self, script_name, *script_args, env=None): + self._check_env_vs_file(env, os.path.join(self.smart_contracts_dir, ".env")) + script_path = os.path.join(self.smart_contracts_dir, f"scripts/{script_name}.js") + # Hint: call web3 directly, avoid npx + truffle + script + # Maybe: self.cmd.yarn(["integrationtest:setTokenLockBurnLimit", str(amount)]) + self.execst(["npx", "truffle", "exec", script_path] + list(script_args), env=env, cwd=self.smart_contracts_dir) + + def set_token_lock_burn_limit(self, update_address, amount, ethereum_private_key, infura_project_id, local_provider): + env = { + "ETHEREUM_PRIVATE_KEY": ethereum_private_key, + "UPDATE_ADDRESS": update_address, + "INFURA_PROJECT_ID": infura_project_id, + "LOCAL_PROVIDER": local_provider, + } + # Needs: ETHEREUM_PRIVATE_KEY, INFURA_PROJECT_ID, LOCAL_PROVIDER, UPDATE_ADDRESS + self.truffle_exec("setTokenLockBurnLimit", str(amount), env=env) + + class UIStackPlaybook: def __init__(self, cmd): self.cmd = cmd @@ -419,20 +441,23 @@ def stack_save_snapshot(self): "LOCAL_PROVIDER": "http://localhost:7545", } - def set_token_lock_burn_limit(update_address, amount): - env = smart_contracts_env_ui_example_vars.copy() - env["UPDATE_ADDRESS"] = update_address - # Needs: ETHEREUM_PRIVATE_KEY, INFURA_PROJECT_ID, LOCAL_PROVIDER, UPDATE_ADDRESS - # Hint: call web3 directly, avoid npx + truffle + script - # Maybe: self.cmd.yarn(["integrationtest:setTokenLockBurnLimit", str(amount)]) - self.cmd.execst(["npx", "truffle", "exec", "scripts/setTokenLockBurnLimit.js", str(amount)], env=env, cwd=self.cmd.smart_contracts_dir) - - set_token_lock_burn_limit(NULL_ADDRESS, 31*10**18) - set_token_lock_burn_limit(bridge_token_address, 10**25) - set_token_lock_burn_limit(atk_address, 10**25) - set_token_lock_burn_limit(btk_address, 10**25) - set_token_lock_burn_limit(usdc_address, 10**25) - set_token_lock_burn_limit(link_address, 10**25) + burn_limits = [ + [NULL_ADDRESS, 31 * 10 ** 18], + [bridge_token_address, 10 ** 25], + [atk_address, 10 ** 25], + [btk_address, 10 ** 25], + [usdc_address, 10 ** 25], + [link_address, 10 ** 25], + ] + for address, amount in burn_limits: + self.cmd.set_token_lock_burn_limit( + address, + amount, + smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"], + smart_contracts_env_ui_example_vars["INFURA_PROJECT_ID"], + smart_contracts_env_ui_example_vars["LOCAL_PROVIDER"] + ) + # signal migrate-complete # Whitelist test tokens @@ -569,11 +594,29 @@ def run(self): assert ebrelayer_ethereum_addr == "0x5aeda56215b167893e80b4fe645ba6d5bab767de" assert ebrelayer_ethereum_private_key == "8d5366123cb560bb606379f90a0bfd4769eecc0557f1b362dcae9012b548b1e5" + env_file = project_dir("test/integration/.env.ciExample") self.cmd.deploy_smart_contracts_for_integration_tests(self.network_name, owner=self.owner, pauser=self.pauser, - initial_validator_addresses=[ebrelayer_ethereum_addr], - env_file=project_dir("test/integration/.env.ciExample")) + initial_validator_addresses=[ebrelayer_ethereum_addr], env_file=env_file) bridge_token_address, bridge_registry_address, bridge_bank = self.cmd.get_bridge_smart_contract_addresses(self.network_id) + + burn_limits = [ + [NULL_ADDRESS, 31*10**18], + [bridge_token_address, 10**25], + ] + env_file_vars = self.cmd.primitive_parse_env_file(env_file) + for address, amount in burn_limits: + self.cmd.set_token_lock_burn_limit( + address, + amount, + env_file_vars["ETHEREUM_PRIVATE_KEY"], + env_file_vars["INFURA_PROJECT_ID"], + env_file_vars["LOCAL_PROVIDER"] + ) + + # test/integration/setup_sifchain.sh: + # TODO + return @@ -581,10 +624,10 @@ def main(): logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(message)s") cmd = Integrator() ui_playbook = UIStackPlaybook(cmd) - ui_playbook.stack_save_snapshot() - ui_playbook.stack_push() + # ui_playbook.stack_save_snapshot() + # ui_playbook.stack_push() it_playbook = IntegrationTestsPlaybook(cmd) - # it_playbook.run() + it_playbook.run() if __name__ == "__main__": main() From 358ee301222f64c2ea37ee067ffbd08d6fe27dcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 1 Aug 2021 08:09:00 +0200 Subject: [PATCH 12/35] Refactoring --- test/integration/make.py | 49 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 2e05ddda87..5ed09e5698 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -52,7 +52,7 @@ def dict_merge(*dicts): class Command: - def execst(self, args, cwd=None, env=None, stdin=None, binary=False, pipe=True): + def execst(self, args, cwd=None, env=None, stdin=None, binary=False, pipe=True, check_exit=True): if stdin is not None: if type(stdin) == list: stdin = "".join([line + "\n" for line in stdin]) @@ -61,7 +61,7 @@ def execst(self, args, cwd=None, env=None, stdin=None, binary=False, pipe=True): env = dict_merge(os.environ, env) popen = subprocess.Popen(args, cwd=cwd, env=env, stdin=subprocess.PIPE, stdout=p, stderr=p, text=not binary) stdout_data, stderr_data = popen.communicate(input=stdin) - if popen.returncode != 0: + if check_exit and (popen.returncode != 0): raise Exception("Command '{}' exited with returncode {}: {}".format(" ".join(args), popen.returncode, repr(stderr_data))) return popen.returncode, stdout_data, stderr_data @@ -564,6 +564,10 @@ def run(self): integration_tests_dir = project_dir("test/integration") data_dir = project_dir("test/integration/vagrant/data") + chainnet = "localnet" + tcp_url = "tcp://0.0.0.0:26657" + ethereum_websocket_address = "ws://localhost:7545/" + # make go binaries (a lot of nonsense!) self.cmd.execst(["make"], cwd=integration_tests_dir, env={"BASEDIR": project_dir()}) @@ -600,6 +604,7 @@ def run(self): bridge_token_address, bridge_registry_address, bridge_bank = self.cmd.get_bridge_smart_contract_addresses(self.network_id) + # TODO This should be last (after return from setup_sifchain.sh) burn_limits = [ [NULL_ADDRESS, 31*10**18], [bridge_token_address, 10**25], @@ -615,14 +620,52 @@ def run(self): ) # test/integration/setup_sifchain.sh: - # TODO + networks_dir = project_dir("deploy/networks") + self.cmd.rmdir(networks_dir) # networks_dir has many directories without write permission, so change those before deleting it + self.cmd.mkdir(networks_dir) + self.cmd.execst(["rake", f"genesis:network:scaffold[{chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) + netdef = exactly_one(yaml_load(self.cmd.read_text_file(project_dir(networks_dir, "network-definition.yml")))) + + moniker = netdef["moniker"] + validator1_address = netdef["address"] + validator1_password = netdef["password"] + mnemonic = netdef["mnemonic"] + chaindir = os.path.join(networks_dir, f"validators/{chainnet}/{moniker}") + # SIFNODED_LOG=$datadir/logs/sifnoded.log + + # test/integration/sifchain_start_daemon.sh: + sifchaind_home = os.path.join(chaindir, ".sifnoded") + whitelisted_validator = exactly_one(stdout_lines(self.cmd.execst(["sifnoded", "keys", "show", + "--keyring-backend", "file", "-a", "--bech", "val", moniker, "--home", sifchaind_home], + stdin=[validator1_password]))) + self.cmd.execst(["sifnoded", "add-genesis-validators", whitelisted_validator, "--home", sifchaind_home]) + adminuser_addr = json.loads(self.cmd.execst(["sifnoded", "keys", "add", "sifnodeadmin", "--keyring-backend", + "test", "--output", "json"], stdin=["y"])[1])["address"] + self.cmd.execst(["sifnoded", "add-genesis-account", adminuser_addr, sif_format_amount(10**20, "rowan"), + "--home", sifchaind_home], pipe=False) + self.cmd.execst(["sifnoded", "set-genesis-oracle-admin", adminuser_addr, "--home", sifchaind_home], pipe=False) + sifnoded_proc = popen(["sifnoded", "start", "--minimum-gas-prices", sif_format_amount(0.5, "rowan"), + "--rpc.laddr", tcp_url, "--home", sifchaind_home]) + + rest_server_proc = popen(["sifnoded", "rest-server", "--laddr", "tcp://0.0.0.0:1317"]) # TODO cwd + + # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh + # TODO wait_for_rpc + # TODO src/py/test_utilities.py: wait_for_sif_account {netdef_json} {validator1_addr} + res = self.cmd.execst(["sifnoded", "keys", "delete", moniker, "--keyring-backend", "test"], stdin=["y"]) # TODO The specified key could not be found in the keyring + res = self.cmd.execst(["sifnoded", "keys", "add", moniker, "--keyring-backend", "test", "--recover"], stdin=[mnemonic]) return + def main(): logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(message)s") cmd = Integrator() + cmd.execst(["pkill", "node"], check_exit=False) + cmd.execst(["pkill", "ebrelayer"], check_exit=False) + cmd.execst(["pkill", "sifnoded"], check_exit=False) + time.sleep(3) ui_playbook = UIStackPlaybook(cmd) # ui_playbook.stack_save_snapshot() # ui_playbook.stack_push() From 80c4bd41d4f01beea258ffd7871849d7a894d0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 1 Aug 2021 21:14:40 +0200 Subject: [PATCH 13/35] Refactoring --- test/integration/make.py | 85 +++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 22 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 5ed09e5698..6fac3eb71d 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -72,6 +72,10 @@ def read_text_file(self, path): with open(path, "rt") as f: return f.read() # TODO Convert to exec + def write_text_file(self, path, s): + with open(path, "wt") as f: + f.write(s) + def mkdir(self, path): os.makedirs(path, exist_ok=True) @@ -146,6 +150,9 @@ def sifnoded_keys_show(self, name, bech=None, keyring_backend=None): res = self.execst(args) return yaml_load(res[1]) + def sifnoded_keys_add(self, args, stdin=None): + return yaml_load(self.execst(["sifnoded", "keys", "add"] + args, stdin=stdin)[1]) + def sifnoded_add_genesis_account(self, address, tokens): tokens_str = ",".join([sif_format_amount(amount, denom) for amount, denom in tokens]) args = ["sifnoded", "add-genesis-account", address, tokens_str] @@ -173,17 +180,29 @@ def sifnoded_get_status(self, host, port): url = "http://{}:{}/node_info".format(host, port) return json.loads(http_get(url).decode("UTF-8")) + def tcp_probe_connect(self, host, port): + res = self.execst(["nc", "-z", host, str(port)], check_exit=False) + return res[0] == 0 + + def wait_for_file(self, path): + while not self.exists(path): + time.sleep(1) class Integrator(Ganache, Sifnoded, Command): def __init__(self): self.smart_contracts_dir = project_dir("smart-contracts") def ebrelayer_init(self, ethereum_private_key, tendermind_node, web3_provider, bridge_registry_contract_address, - validator_moniker, validator_mnemonic, chain_id, gas, gas_prices): + validator_moniker, validator_mnemonic, chain_id, gas=None, gas_prices=None, node=None, keyring_backend=None, + sign_with=None): env = {"ETHEREUM_PRIVATE_KEY": ethereum_private_key} args = ["ebrelayer", "init", tendermind_node, web3_provider, bridge_registry_contract_address, - validator_moniker, " ".join(validator_mnemonic), "--chain-id={}".format(chain_id), "--gas", str(gas), - "--gas-prices", sif_format_amount(*gas_prices)] + validator_moniker, " ".join(validator_mnemonic), "--chain-id={}".format(chain_id)] + \ + (["--gas", str(gas)] if gas is not None else []) + \ + (["--gas-prices", sif_format_amount(*gas_prices)] if gas_prices is not None else []) + \ + (["--node", node] if node is not None else []) + \ + (["--keyring-backend", keyring_backend] if keyring_backend is not None else []) + \ + (["--from", sign_with] if sign_with is not None else []) return popen(args, env=env) def sif_wait_up(self, host, port): @@ -469,8 +488,8 @@ def stack_save_snapshot(self): # ebrelayer is in $GOBIN, gets installed by "make install" ethereum_private_key = smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"] ebrelayer_proc = self.cmd.ebrelayer_init(ethereum_private_key, "tcp://localhost:26657", "ws://localhost:7545/", - bridge_registry_address, self.shadowfiend_name, self.shadowfiend_mnemonic, f"--chain-id={self.chain_id}", - 5*10**12, [0.5, "rowan"]) + bridge_registry_address, self.shadowfiend_name, self.shadowfiend_mnemonic, self.chain_id, gas=5*10**12, + gas_prices=[0.5, "rowan"]) # At this point we have 3 running processes - ganache_proc, sifnoded_proc and ebrelayer_proc # await sif-node-up and migrate-complete @@ -561,7 +580,7 @@ def __init__(self, cmd): self.network_id = 5777 def run(self): - integration_tests_dir = project_dir("test/integration") + test_integration_dir = project_dir("test/integration") data_dir = project_dir("test/integration/vagrant/data") chainnet = "localnet" @@ -569,7 +588,7 @@ def run(self): ethereum_websocket_address = "ws://localhost:7545/" # make go binaries (a lot of nonsense!) - self.cmd.execst(["make"], cwd=integration_tests_dir, env={"BASEDIR": project_dir()}) + self.cmd.execst(["make"], cwd=test_integration_dir, env={"BASEDIR": project_dir()}) self.cmd.build_smart_contracts_for_integration_tests() @@ -583,12 +602,15 @@ def run(self): # 4. sleep 5 # 5. while ! nc -z localhost 4545; do sleep 5; done # GANACHE_LOG=ui/test/integration/vagrant/data/logs/ganache.$(filenamedate).txt - mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", "crumble", "sweet", "treat"] + validator_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", "crumble", "sweet", "treat"] block_time = None # TODO account_keys_path = os.path.join(data_dir, "ganachekeys.json") ganache_db_path = self.cmd.mktempdir() - self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=mnemonic, network_id=self.network_id, - port=7545, db=ganache_db_path, account_keys_path=account_keys_path) + ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=validator_mnemonic, + network_id=self.network_id, port=7545, db=ganache_db_path, account_keys_path=account_keys_path) + + self.cmd.wait_for_file(account_keys_path) + time.sleep(2) ganache_keys = json.loads(self.cmd.read_text_file(account_keys_path)) ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] @@ -602,19 +624,20 @@ def run(self): self.cmd.deploy_smart_contracts_for_integration_tests(self.network_name, owner=self.owner, pauser=self.pauser, initial_validator_addresses=[ebrelayer_ethereum_addr], env_file=env_file) - bridge_token_address, bridge_registry_address, bridge_bank = self.cmd.get_bridge_smart_contract_addresses(self.network_id) + bridge_token_sc_addr, bridge_registry_sc_addr, bridge_bank_sc_addr = \ + self.cmd.get_bridge_smart_contract_addresses(self.network_id) # TODO This should be last (after return from setup_sifchain.sh) burn_limits = [ [NULL_ADDRESS, 31*10**18], - [bridge_token_address, 10**25], + [bridge_token_sc_addr, 10**25], ] env_file_vars = self.cmd.primitive_parse_env_file(env_file) for address, amount in burn_limits: self.cmd.set_token_lock_burn_limit( address, amount, - env_file_vars["ETHEREUM_PRIVATE_KEY"], + env_file_vars["ETHEREUM_PRIVATE_KEY"], # != ebrelayer_ethereum_private_key env_file_vars["INFURA_PROJECT_ID"], env_file_vars["LOCAL_PROVIDER"] ) @@ -625,19 +648,22 @@ def run(self): self.cmd.mkdir(networks_dir) self.cmd.execst(["rake", f"genesis:network:scaffold[{chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) netdef = exactly_one(yaml_load(self.cmd.read_text_file(project_dir(networks_dir, "network-definition.yml")))) + netdef_json = os.path.join(data_dir, "netdef.json") + self.cmd.write_text_file(netdef_json, json.dumps(netdef)) - moniker = netdef["moniker"] + validator_moniker = netdef["moniker"] validator1_address = netdef["address"] validator1_password = netdef["password"] - mnemonic = netdef["mnemonic"] - chaindir = os.path.join(networks_dir, f"validators/{chainnet}/{moniker}") + validator_mnemonic = netdef["mnemonic"].split(" ") + chaindir = os.path.join(networks_dir, f"validators/{chainnet}/{validator_moniker}") # SIFNODED_LOG=$datadir/logs/sifnoded.log # test/integration/sifchain_start_daemon.sh: sifchaind_home = os.path.join(chaindir, ".sifnoded") whitelisted_validator = exactly_one(stdout_lines(self.cmd.execst(["sifnoded", "keys", "show", - "--keyring-backend", "file", "-a", "--bech", "val", moniker, "--home", sifchaind_home], + "--keyring-backend", "file", "-a", "--bech", "val", validator_moniker, "--home", sifchaind_home], stdin=[validator1_password]))) + log.info(f"Whitelisted validator: {whitelisted_validator}") self.cmd.execst(["sifnoded", "add-genesis-validators", whitelisted_validator, "--home", sifchaind_home]) adminuser_addr = json.loads(self.cmd.execst(["sifnoded", "keys", "add", "sifnodeadmin", "--keyring-backend", "test", "--output", "json"], stdin=["y"])[1])["address"] @@ -650,21 +676,35 @@ def run(self): rest_server_proc = popen(["sifnoded", "rest-server", "--laddr", "tcp://0.0.0.0:1317"]) # TODO cwd # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh - # TODO wait_for_rpc - # TODO src/py/test_utilities.py: wait_for_sif_account {netdef_json} {validator1_addr} - res = self.cmd.execst(["sifnoded", "keys", "delete", moniker, "--keyring-backend", "test"], stdin=["y"]) # TODO The specified key could not be found in the keyring - res = self.cmd.execst(["sifnoded", "keys", "add", moniker, "--keyring-backend", "test", "--recover"], stdin=[mnemonic]) + while not self.cmd.tcp_probe_connect("localhost", 26657): + time.sleep(1) + self.cmd.execst(["python3", os.path.join(test_integration_dir, "src/py/wait_for_sif_account.py"), + netdef_json, validator1_address], env={"USER1ADDR": "nothing"}) + time.sleep(10) + # Error: The specified item could not be found in the keyring + # res = self.cmd.execst(["sifnoded", "keys", "delete", moniker, "--keyring-backend", "test"], stdin=["y"]) + self.cmd.sifnoded_keys_add([validator_moniker, "--keyring-backend", "test", "--recover"], + stdin=[" ".join(validator_mnemonic)]) - return + ebrelayer_proc = self.cmd.ebrelayer_init(ebrelayer_ethereum_private_key, tcp_url, ethereum_websocket_address, + bridge_registry_sc_addr, validator_moniker, validator_mnemonic, chainnet, node=tcp_url, + keyring_backend="test", sign_with=validator_moniker) + + return ganache_proc, sifnoded_proc, ebrelayer_proc def main(): logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(message)s") cmd = Integrator() + # Reset state: + # pkill node; pkill ebrelayer; pkill sifnoded; rm -rvf $HOME/.sifnoded; rm -rvf ./vagrant/data; mkdir vagrant/data cmd.execst(["pkill", "node"], check_exit=False) cmd.execst(["pkill", "ebrelayer"], check_exit=False) cmd.execst(["pkill", "sifnoded"], check_exit=False) + cmd.rmdir(cmd.get_user_home(".sifnoded")) + cmd.rmdir(project_dir("test/integration/vagrant/data")) + cmd.mkdir(project_dir("test/integration/vagrant/data")) time.sleep(3) ui_playbook = UIStackPlaybook(cmd) # ui_playbook.stack_save_snapshot() @@ -672,5 +712,6 @@ def main(): it_playbook = IntegrationTestsPlaybook(cmd) it_playbook.run() + if __name__ == "__main__": main() From 8911f2bd0e6201105325822202c0010dcbe9bf93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 2 Aug 2021 21:10:09 +0200 Subject: [PATCH 14/35] Refactoring --- test/integration/make.py | 102 ++++++++++++++++++++++++++------------- 1 file changed, 68 insertions(+), 34 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 6fac3eb71d..fa0f76c08f 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -275,7 +275,7 @@ def deploy_smart_contracts_for_integration_tests(self, network_name, consensus_t if initial_validator_addresses is not None: env["INITIAL_VALIDATOR_ADDRESSES"] = ",".join(initial_validator_addresses) if initial_validator_powers is not None: - env["INITIAL_VALIDATOR_POWERS"] = ",".join(initial_validator_powers) + env["INITIAL_VALIDATOR_POWERS"] = ",".join([str(x) for x in initial_validator_powers]) if pauser is not None: env["PAUSER"] = pauser if mainnet_gas_price is not None: @@ -578,6 +578,9 @@ def __init__(self, cmd): # set_persistant_env_var CHAINNET localnet $envexportfile self.network_name = "develop" self.network_id = 5777 + self.using_ganache_gui = False + self.snapshots_dir = self.cmd.get_user_home(".sifnode-snapshots") + self.state_vars = {} def run(self): test_integration_dir = project_dir("test/integration") @@ -592,41 +595,56 @@ def run(self): self.cmd.build_smart_contracts_for_integration_tests() - # test/integration/ganache-start.sh: - # 1. pkill -9 -f ganache-cli || true - # 2. while nc -z localhost 7545; do sleep 1; done - # 3. nohup tmux new-session -d -s my_session "ganache-cli ${block_delay} -h 0.0.0.0 --mnemonic \ - # 'candy maple cake sugar pudding cream honey rich smooth crumble sweet treat' \ - # --networkId '5777' --port '7545' --db ${GANACHE_DB_DIR} --account_keys_path $GANACHE_KEYS_JSON \ - # > $GANACHE_LOG 2>&1" - # 4. sleep 5 - # 5. while ! nc -z localhost 4545; do sleep 5; done - # GANACHE_LOG=ui/test/integration/vagrant/data/logs/ganache.$(filenamedate).txt - validator_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", "crumble", "sweet", "treat"] - block_time = None # TODO - account_keys_path = os.path.join(data_dir, "ganachekeys.json") - ganache_db_path = self.cmd.mktempdir() - ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=validator_mnemonic, - network_id=self.network_id, port=7545, db=ganache_db_path, account_keys_path=account_keys_path) - - self.cmd.wait_for_file(account_keys_path) - time.sleep(2) - - ganache_keys = json.loads(self.cmd.read_text_file(account_keys_path)) - ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] - ebrelayer_ethereum_private_key = ganache_keys["private_keys"][ebrelayer_ethereum_addr] - # TODO Check for possible non-determinism of dict().keys() ordering (c.f. test/integration/vagrantenv.sh) - # TODO ebrelayer_ethereum_private_key is NOT the same as in test/integration/.env.ciExample - assert ebrelayer_ethereum_addr == "0x5aeda56215b167893e80b4fe645ba6d5bab767de" - assert ebrelayer_ethereum_private_key == "8d5366123cb560bb606379f90a0bfd4769eecc0557f1b362dcae9012b548b1e5" + if self.using_ganache_gui: + ebrelayer_ethereum_addr = "0x8e2bE12daDbCcbf7c98DBb59f98f22DFF0eF3F2c" + ebrelayer_ethereum_private_key = "2eaddbc0bca859ff5b09c5a48a2feaeaf464f7cbf8ddbfa4a32a625a8322fe99" + ganache_proc = None + else: + # test/integration/ganache-start.sh: + # 1. pkill -9 -f ganache-cli || true + # 2. while nc -z localhost 7545; do sleep 1; done + # 3. nohup tmux new-session -d -s my_session "ganache-cli ${block_delay} -h 0.0.0.0 --mnemonic \ + # 'candy maple cake sugar pudding cream honey rich smooth crumble sweet treat' \ + # --networkId '5777' --port '7545' --db ${GANACHE_DB_DIR} --account_keys_path $GANACHE_KEYS_JSON \ + # > $GANACHE_LOG 2>&1" + # 4. sleep 5 + # 5. while ! nc -z localhost 4545; do sleep 5; done + # GANACHE_LOG=ui/test/integration/vagrant/data/logs/ganache.$(filenamedate).txt + validator_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", "crumble", "sweet", "treat"] + block_time = None # TODO + account_keys_path = os.path.join(data_dir, "ganachekeys.json") + ganache_db_path = self.cmd.mktempdir() + self.state_vars["GANACHE_DB_PATH"] = ganache_db_path + ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=validator_mnemonic, + network_id=self.network_id, port=7545, db=ganache_db_path, account_keys_path=account_keys_path) + + self.cmd.wait_for_file(account_keys_path) # Created by ganache-cli + time.sleep(2) + + ganache_keys = json.loads(self.cmd.read_text_file(account_keys_path)) + ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] + ebrelayer_ethereum_private_key = ganache_keys["private_keys"][ebrelayer_ethereum_addr] + # TODO Check for possible non-determinism of dict().keys() ordering (c.f. test/integration/vagrantenv.sh) + # TODO ebrelayer_ethereum_private_key is NOT the same as in test/integration/.env.ciExample + assert ebrelayer_ethereum_addr == "0x5aeda56215b167893e80b4fe645ba6d5bab767de" + assert ebrelayer_ethereum_private_key == "8d5366123cb560bb606379f90a0bfd4769eecc0557f1b362dcae9012b548b1e5" env_file = project_dir("test/integration/.env.ciExample") + env_vars = self.cmd.primitive_parse_env_file(env_file) self.cmd.deploy_smart_contracts_for_integration_tests(self.network_name, owner=self.owner, pauser=self.pauser, + operator=env_vars["OPERATOR"], consensus_threshold=int(env_vars["CONSENSUS_THRESHOLD"]), + initial_validator_powers=[int(x) for x in env_vars["INITIAL_VALIDATOR_POWERS"].split(",")], initial_validator_addresses=[ebrelayer_ethereum_addr], env_file=env_file) bridge_token_sc_addr, bridge_registry_sc_addr, bridge_bank_sc_addr = \ self.cmd.get_bridge_smart_contract_addresses(self.network_id) + # # Proof of concept: restart ganache-cli + # time.sleep(10) + # ganache_proc.kill() + # ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=validator_mnemonic, + # network_id=self.network_id, port=7545, db=ganache_db_path, account_keys_path=account_keys_path) + # TODO This should be last (after return from setup_sifchain.sh) burn_limits = [ [NULL_ADDRESS, 31*10**18], @@ -639,7 +657,7 @@ def run(self): amount, env_file_vars["ETHEREUM_PRIVATE_KEY"], # != ebrelayer_ethereum_private_key env_file_vars["INFURA_PROJECT_ID"], - env_file_vars["LOCAL_PROVIDER"] + env_file_vars["LOCAL_PROVIDER"], # for web3.js to connect to ganache ) # test/integration/setup_sifchain.sh: @@ -692,13 +710,19 @@ def run(self): return ganache_proc, sifnoded_proc, ebrelayer_proc + def create_snapshot(self, snapshot_name): + self.cmd.mkdir(self.snapshots_dir) + named_snapshot_dir = os.path.join(self.snapshots_dir, snapshot_name) + self.cmd.mkdir(named_snapshot_dir) + ganache_db_path = self.state_vars["GANACHE_DB_PATH"] + self.cmd.tar_create(ganache_db_path, os.path.join(named_snapshot_dir, "ganache.tar.gz"), compression="gz") + self.cmd.tar_create(project_dir("deploy/networks"), os.path.join(named_snapshot_dir, "networks.tar.gz"), compression="gz") + self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(named_snapshot_dir, "smart-contracts.tar.gz"), compression="gz") -def main(): - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(message)s") - cmd = Integrator() - # Reset state: +def cleanup_and_reset_state(): # pkill node; pkill ebrelayer; pkill sifnoded; rm -rvf $HOME/.sifnoded; rm -rvf ./vagrant/data; mkdir vagrant/data + cmd = Command() cmd.execst(["pkill", "node"], check_exit=False) cmd.execst(["pkill", "ebrelayer"], check_exit=False) cmd.execst(["pkill", "sifnoded"], check_exit=False) @@ -706,11 +730,21 @@ def main(): cmd.rmdir(project_dir("test/integration/vagrant/data")) cmd.mkdir(project_dir("test/integration/vagrant/data")) time.sleep(3) + + +def main(): + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(message)s") + cleanup_and_reset_state() + cmd = Integrator() ui_playbook = UIStackPlaybook(cmd) # ui_playbook.stack_save_snapshot() # ui_playbook.stack_push() it_playbook = IntegrationTestsPlaybook(cmd) - it_playbook.run() + processes = it_playbook.run() + for p in processes: + if p is not None: + p.kill() + it_playbook.create_snapshot("test_snapshot_1") if __name__ == "__main__": From 89364773b5ab4d7af32a615411e96d9deee56ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 3 Aug 2021 11:02:07 +0200 Subject: [PATCH 15/35] Support for setting default balance in ganache-cli --- test/integration/make.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/make.py b/test/integration/make.py index fa0f76c08f..982dd4a14c 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -116,7 +116,7 @@ def tar_create(self, path, tar_path, compression=None): class Ganache(Command): def start_ganache_cli(self, mnemonic=None, db=None, port=None, host=None, network_id=None, gas_price=None, - gas_limit=None, block_time=None, account_keys_path=None, popen_args=None): + gas_limit=None, default_balance_ether=None, block_time=None, account_keys_path=None, popen_args=None): args = ["ganache-cli"] + \ (["--mnemonic", " ".join(mnemonic)] if mnemonic else []) + \ (["--db", db] if db else []) + \ @@ -125,6 +125,7 @@ def start_ganache_cli(self, mnemonic=None, db=None, port=None, host=None, networ (["--networkId", str(network_id)] if network_id is not None else []) + \ (["--gasPrice", str(gas_price)] if gas_price is not None else []) + \ (["--gasLimit", str(gas_limit)] if gas_limit is not None else []) + \ + (["--defaultBalanceEther", str(default_balance_ether)] if default_balance_ether is not None else []) + \ (["--blockTime", str(block_time)] if block_time is not None else []) + \ (["--account_keys_path", account_keys_path] if account_keys_path is not None else []) return popen(args, **(popen_args if popen_args is not None else dict())) From f7e7ef29c20e69896dd48ca579e973772c5a44f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 5 Aug 2021 03:50:04 +0200 Subject: [PATCH 16/35] State cleanup --- test/integration/make.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/make.py b/test/integration/make.py index 982dd4a14c..7bccd82852 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -722,6 +722,7 @@ def create_snapshot(self, snapshot_name): def cleanup_and_reset_state(): + # git checkout 4cb7322b6b282babd93a0d0aedda837c9134e84e deploy # pkill node; pkill ebrelayer; pkill sifnoded; rm -rvf $HOME/.sifnoded; rm -rvf ./vagrant/data; mkdir vagrant/data cmd = Command() cmd.execst(["pkill", "node"], check_exit=False) @@ -729,7 +730,9 @@ def cleanup_and_reset_state(): cmd.execst(["pkill", "sifnoded"], check_exit=False) cmd.rmdir(cmd.get_user_home(".sifnoded")) cmd.rmdir(project_dir("test/integration/vagrant/data")) - cmd.mkdir(project_dir("test/integration/vagrant/data")) + cmd.rmdir(project_dir("test/integration/relayerdb")) + cmd.rmdir(project_dir("test/integration/.pytest_cache")) + cmd.rmdir(project_dir("relayerdb")) time.sleep(3) From 1605f092f546e8f06a34f5d2cf041b694f6582ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 8 Aug 2021 13:27:05 +0200 Subject: [PATCH 17/35] Refactoring --- test/integration/make.py | 100 +++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 19 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 7bccd82852..a633cb6d60 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -15,6 +15,9 @@ def stdout_lines(res): return res[1].splitlines() +def joinlines(lines): + return "".join([x + os.linesep for x in lines]) + def exactly_one(items): if len(items) == 0: raise ValueError("Zero items") @@ -66,7 +69,8 @@ def execst(self, args, cwd=None, env=None, stdin=None, binary=False, pipe=True, return popen.returncode, stdout_data, stderr_data def rm(self, path): - os.remove(path) + if os.path.exists(path): + os.remove(path) def read_text_file(self, path): with open(path, "rt") as f: @@ -561,6 +565,7 @@ def stack_push(self): class IntegrationTestsPlaybook: def __init__(self, cmd): self.cmd = cmd + # Fixed, set in start-integration-env.sh self.ethereum_private_key = "c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3" self.owner = "0x627306090abaB3A6e1400e9345bC60c78a8BEf57" # we may eventually switch things so PAUSER and OWNER aren't the same account, but for now they're the same @@ -582,23 +587,25 @@ def __init__(self, cmd): self.using_ganache_gui = False self.snapshots_dir = self.cmd.get_user_home(".sifnode-snapshots") self.state_vars = {} + self.test_integration_dir = project_dir("test/integration") + self.data_dir = project_dir("test/integration/vagrant/data") def run(self): - test_integration_dir = project_dir("test/integration") - data_dir = project_dir("test/integration/vagrant/data") + self.cmd.mkdir(self.data_dir) chainnet = "localnet" tcp_url = "tcp://0.0.0.0:26657" ethereum_websocket_address = "ws://localhost:7545/" # make go binaries (a lot of nonsense!) - self.cmd.execst(["make"], cwd=test_integration_dir, env={"BASEDIR": project_dir()}) + self.cmd.execst(["make"], cwd=self.test_integration_dir, env={"BASEDIR": project_dir()}) self.cmd.build_smart_contracts_for_integration_tests() if self.using_ganache_gui: ebrelayer_ethereum_addr = "0x8e2bE12daDbCcbf7c98DBb59f98f22DFF0eF3F2c" ebrelayer_ethereum_private_key = "2eaddbc0bca859ff5b09c5a48a2feaeaf464f7cbf8ddbfa4a32a625a8322fe99" + ganache_db_path = None ganache_proc = None else: # test/integration/ganache-start.sh: @@ -613,7 +620,7 @@ def run(self): # GANACHE_LOG=ui/test/integration/vagrant/data/logs/ganache.$(filenamedate).txt validator_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", "crumble", "sweet", "treat"] block_time = None # TODO - account_keys_path = os.path.join(data_dir, "ganachekeys.json") + account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") ganache_db_path = self.cmd.mktempdir() self.state_vars["GANACHE_DB_PATH"] = ganache_db_path ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=validator_mnemonic, @@ -667,7 +674,7 @@ def run(self): self.cmd.mkdir(networks_dir) self.cmd.execst(["rake", f"genesis:network:scaffold[{chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) netdef = exactly_one(yaml_load(self.cmd.read_text_file(project_dir(networks_dir, "network-definition.yml")))) - netdef_json = os.path.join(data_dir, "netdef.json") + netdef_json = os.path.join(self.data_dir, "netdef.json") self.cmd.write_text_file(netdef_json, json.dumps(netdef)) validator_moniker = netdef["moniker"] @@ -695,21 +702,66 @@ def run(self): rest_server_proc = popen(["sifnoded", "rest-server", "--laddr", "tcp://0.0.0.0:1317"]) # TODO cwd # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh + # This script is also called from tests + while not self.cmd.tcp_probe_connect("localhost", 26657): time.sleep(1) - self.cmd.execst(["python3", os.path.join(test_integration_dir, "src/py/wait_for_sif_account.py"), - netdef_json, validator1_address], env={"USER1ADDR": "nothing"}) + self.wait_for_sif_account(netdef_json, validator1_address) time.sleep(10) - # Error: The specified item could not be found in the keyring - # res = self.cmd.execst(["sifnoded", "keys", "delete", moniker, "--keyring-backend", "test"], stdin=["y"]) - self.cmd.sifnoded_keys_add([validator_moniker, "--keyring-backend", "test", "--recover"], - stdin=[" ".join(validator_mnemonic)]) - + self.remove_and_add_sifnoded_keys(validator_moniker, validator_mnemonic) ebrelayer_proc = self.cmd.ebrelayer_init(ebrelayer_ethereum_private_key, tcp_url, ethereum_websocket_address, bridge_registry_sc_addr, validator_moniker, validator_mnemonic, chainnet, node=tcp_url, keyring_backend="test", sign_with=validator_moniker) - return ganache_proc, sifnoded_proc, ebrelayer_proc + vagrantenv_path = project_dir("test/integration/vagrantenv.sh") + vagrantenv = { + "ETHEREUM_PRIVATE_KEY": self.ethereum_private_key, + "OWNER": self.owner, + "PAUSER": self.pauser, + "BASEDIR": project_dir(), + # export SIFCHAIN_BIN="/home/jurez/work/projects/sif/sifnode/local/cmd" + "envexportfile": vagrantenv_path, + # export TEST_INTEGRATION_DIR="/home/jurez/work/projects/sif/sifnode/local/test/integration" + # export TEST_INTEGRATION_PY_DIR="/home/jurez/work/projects/sif/sifnode/local/test/integration/src/py" + "SMART_CONTRACTS_DIR": project_dir("smart-contracts"), + # export datadir="/home/jurez/work/projects/sif/sifnode/local/test/integration/vagrant/data" + # export CONTAINER_NAME="integration_sifnode1_1" + "NETWORKDIR": networks_dir, + # export ETHEREUM_WEBSOCKET_ADDRESS="ws://localhost:7545/" + # export CHAINNET="localnet" + "GANACHE_DB_DIR": ganache_db_path, + # export GANACHE_KEYS_JSON="/home/jurez/work/projects/sif/sifnode/local/test/integration/vagrant/data/ganachekeys.json" + # export EBRELAYER_ETHEREUM_ADDR="0x5aeda56215b167893e80b4fe645ba6d5bab767de" + # export EBRELAYER_ETHEREUM_PRIVATE_KEY="8d5366123cb560bb606379f90a0bfd4769eecc0557f1b362dcae9012b548b1e5" + # # BRIDGE_REGISTRY_ADDRESS and ETHEREUM_CONTRACT_ADDRESS are synonyms + "BRIDGE_REGISTRY_ADDRESS": bridge_registry_sc_addr, + "BRIDGE_TOKEN_ADDRESS": bridge_token_sc_addr, + "BRIDGE_BANK_ADDRESS": bridge_bank_sc_addr, + "NETDEF": os.path.join(networks_dir, "network-definition.yml"), + "NETDEF_JSON": project_dir("test/integration/vagrant/data/netdef.json"), + "MONIKER": validator_moniker, + "VALIDATOR1_PASSWORD": validator1_password, + "VALIDATOR1_ADDR": validator1_address, + "MNEMONIC": " ".join(validator_mnemonic), + "CHAINDIR": os.path.join(networks_dir, "validators", chainnet, validator_moniker), + "SIFCHAIN_ADMIN_ACCOUNT": adminuser_addr, # Needed by test_peggy_fees.py (via conftest.py) + } + self.cmd.write_text_file(vagrantenv_path, joinlines([f"{k}=\"{v}\"" for k, v in vagrantenv.items()])) + + return vagrantenv, (ganache_proc, sifnoded_proc, ebrelayer_proc, rest_server_proc) + + def wait_for_sif_account(self, netdef_json, validator1_address): + return self.cmd.execst(["python3", os.path.join(self.test_integration_dir, "src/py/wait_for_sif_account.py"), + netdef_json, validator1_address], env={"USER1ADDR": "nothing"}) + + def remove_and_add_sifnoded_keys(self, validator_moniker, validator_mnemonic): + # Error: The specified item could not be found in the keyring + # This is not neccessary during start-integration-env.sh (as the key does not exist yet), but is neccessary + # during tests that restart ebrelayer + # res = self.cmd.execst(["sifnoded", "keys", "delete", moniker, "--keyring-backend", "test"], stdin=["y"]) + self.cmd.execst(["sifnoded", "keys", "delete", validator_moniker, "--keyring-backend", "test"], stdin=["y"], check_exit=False) + self.cmd.sifnoded_keys_add([validator_moniker, "--keyring-backend", "test", "--recover"], + stdin=[" ".join(validator_mnemonic)]) def create_snapshot(self, snapshot_name): self.cmd.mkdir(self.snapshots_dir) @@ -728,11 +780,21 @@ def cleanup_and_reset_state(): cmd.execst(["pkill", "node"], check_exit=False) cmd.execst(["pkill", "ebrelayer"], check_exit=False) cmd.execst(["pkill", "sifnoded"], check_exit=False) - cmd.rmdir(cmd.get_user_home(".sifnoded")) - cmd.rmdir(project_dir("test/integration/vagrant/data")) + + # rm -rvf /tmp/tmp.xxxx (ganache DB, unique for every run) cmd.rmdir(project_dir("test/integration/relayerdb")) - cmd.rmdir(project_dir("test/integration/.pytest_cache")) - cmd.rmdir(project_dir("relayerdb")) + cmd.rmdir(project_dir("smart-contracts/build")) + cmd.rmdir(project_dir("test/integration/vagrant/data")) + + # Not sure if this is needed too + cmd.rmdir(cmd.get_user_home(".sifnoded")) + + # Additional cleanup (not neccessary to make it work) + # cmd.rm(project_dir("smart-contracts/combined.log")) + # cmd.rmdir(project_dir("test/integration/.pytest_cache")) + # cmd,rm(project_dir("smart-contracts/.env")) + # cmd.rmdir(project_dir("deploy/networks")) + # cmd.rmdir(project_dir("smart-contracts/.openzeppelin")) time.sleep(3) @@ -744,7 +806,7 @@ def main(): # ui_playbook.stack_save_snapshot() # ui_playbook.stack_push() it_playbook = IntegrationTestsPlaybook(cmd) - processes = it_playbook.run() + _, processes = it_playbook.run() for p in processes: if p is not None: p.kill() From bafebe1d45b0768c346a2156b2bf7c2dad9620f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 8 Aug 2021 21:35:55 +0200 Subject: [PATCH 18/35] Refactoring --- test/integration/make.py | 49 +++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index a633cb6d60..ffb3f649d0 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -622,7 +622,6 @@ def run(self): block_time = None # TODO account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") ganache_db_path = self.cmd.mktempdir() - self.state_vars["GANACHE_DB_PATH"] = ganache_db_path ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=validator_mnemonic, network_id=self.network_id, port=7545, db=ganache_db_path, account_keys_path=account_keys_path) @@ -699,6 +698,7 @@ def run(self): sifnoded_proc = popen(["sifnoded", "start", "--minimum-gas-prices", sif_format_amount(0.5, "rowan"), "--rpc.laddr", tcp_url, "--home", sifchaind_home]) + # TODO Process exits immediately with returncode 1 rest_server_proc = popen(["sifnoded", "rest-server", "--laddr", "tcp://0.0.0.0:1317"]) # TODO cwd # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh @@ -714,7 +714,7 @@ def run(self): keyring_backend="test", sign_with=validator_moniker) vagrantenv_path = project_dir("test/integration/vagrantenv.sh") - vagrantenv = { + self.state_vars = { "ETHEREUM_PRIVATE_KEY": self.ethereum_private_key, "OWNER": self.owner, "PAUSER": self.pauser, @@ -746,9 +746,9 @@ def run(self): "CHAINDIR": os.path.join(networks_dir, "validators", chainnet, validator_moniker), "SIFCHAIN_ADMIN_ACCOUNT": adminuser_addr, # Needed by test_peggy_fees.py (via conftest.py) } - self.cmd.write_text_file(vagrantenv_path, joinlines([f"{k}=\"{v}\"" for k, v in vagrantenv.items()])) + self.cmd.write_text_file(vagrantenv_path, joinlines([f"{k}=\"{v}\"" for k, v in self.state_vars.items()])) - return vagrantenv, (ganache_proc, sifnoded_proc, ebrelayer_proc, rest_server_proc) + return ganache_proc, sifnoded_proc, ebrelayer_proc, rest_server_proc def wait_for_sif_account(self, netdef_json, validator1_address): return self.cmd.execst(["python3", os.path.join(self.test_integration_dir, "src/py/wait_for_sif_account.py"), @@ -766,11 +766,14 @@ def remove_and_add_sifnoded_keys(self, validator_moniker, validator_mnemonic): def create_snapshot(self, snapshot_name): self.cmd.mkdir(self.snapshots_dir) named_snapshot_dir = os.path.join(self.snapshots_dir, snapshot_name) + if self.cmd.exists(named_snapshot_dir): + raise Exception(f"Directory '{named_snapshot_dir}' already exists") self.cmd.mkdir(named_snapshot_dir) - ganache_db_path = self.state_vars["GANACHE_DB_PATH"] + ganache_db_path = self.state_vars["GANACHE_DB_DIR"] self.cmd.tar_create(ganache_db_path, os.path.join(named_snapshot_dir, "ganache.tar.gz"), compression="gz") self.cmd.tar_create(project_dir("deploy/networks"), os.path.join(named_snapshot_dir, "networks.tar.gz"), compression="gz") self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(named_snapshot_dir, "smart-contracts.tar.gz"), compression="gz") + self.cmd.write_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"), json.dumps(self.state_vars, indent=4)) def cleanup_and_reset_state(): @@ -806,7 +809,7 @@ def main(): # ui_playbook.stack_save_snapshot() # ui_playbook.stack_push() it_playbook = IntegrationTestsPlaybook(cmd) - _, processes = it_playbook.run() + processes = it_playbook.run() for p in processes: if p is not None: p.kill() @@ -815,3 +818,37 @@ def main(): if __name__ == "__main__": main() + + +# Trace of test_utilities.py get_required_env_var/get_optional_env_var: +# +# BASEDIR (required), value=/home/jurez/work/projects/sif/sifnode/local +# BRIDGE_BANK_ADDRESS (optional), value=0x30753E4A8aad7F8597332E813735Def5dD395028 +# BRIDGE_BANK_ADDRESS (required), value=0x30753E4A8aad7F8597332E813735Def5dD395028 +# BRIDGE_REGISTRY_ADDRESS (required), value=0xf204a4Ef082f5c04bB89F7D5E6568B796096735a +# BRIDGE_TOKEN_ADDRESS (optional), value=0x82D50AD3C1091866E258Fd0f1a7cC9674609D254 +# BRIDGE_TOKEN_ADDRESS (required), value=0x82D50AD3C1091866E258Fd0f1a7cC9674609D254 +# CHAINDIR (required), 3x value +# CHAINNET (required), value=localnet +# DEPLOYMENT_NAME (optional), value=None +# ETHEREUM_ADDRESS (optional), value=None +# ETHEREUM_NETWORK (optional), value=None +# ETHEREUM_NETWORK_ID (optional), value=None +# ETHEREUM_WEBSOCKET_ADDRESS (required), value=ws://localhost:7545/ +# GANACHE_KEYS_FILE (optional), value=None +# HOME (required), value=/home/jurez +# MNEMONIC (required), value=future tattoo gesture artist tomato accuse chuckle polar ivory strategy rail flower apart virus burger rhythm either describe habit attend absurd aspect predict parent +# MONIKER (required), value=wandering-flower +# OPERATOR_ADDRESS (optional), value=None +# OPERATOR_PRIVATE_KEY (optional), value=None +# OPERATOR_PRIVATE_KEY (optional), value=c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3 +# ROWAN_SOURCE (optional), value=None +# ROWAN_SOURCE_KEY (optional), value=None +# SIFCHAIN_ADMIN_ACCOUNT (required), value=sif1896ner48vrg8m05k48ykc6yydlxc4yvm23hp5m +# SIFNODE (optional), value=None +# SMART_CONTRACTS_DIR (required), 2x value +# SMART_CONTRACT_ARTIFACT_DIR (optional), value=None +# SOLIDITY_JSON_PATH (optional), value=None +# TEST_INTEGRATION_DIR (required), value=/home/jurez/work/projects/sif/sifnode/local/test/integration +# VALIDATOR1_ADDR (optional), 3x value +# VALIDATOR1_PASSWORD (optional), 3x value From 75e4bfa9b523d56b027c251f40837234a0dc636d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 9 Aug 2021 12:03:41 +0200 Subject: [PATCH 19/35] Refactoring --- test/integration/make.py | 111 +++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 27 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index ffb3f649d0..bc2e0277bc 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -589,14 +589,15 @@ def __init__(self, cmd): self.state_vars = {} self.test_integration_dir = project_dir("test/integration") self.data_dir = project_dir("test/integration/vagrant/data") + self.chainnet = "localnet" + self.tcp_url = "tcp://0.0.0.0:26657" + self.ethereum_websocket_address = "ws://localhost:7545/" + self.ganache_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", + "crumble", "sweet", "treat"] def run(self): self.cmd.mkdir(self.data_dir) - chainnet = "localnet" - tcp_url = "tcp://0.0.0.0:26657" - ethereum_websocket_address = "ws://localhost:7545/" - # make go binaries (a lot of nonsense!) self.cmd.execst(["make"], cwd=self.test_integration_dir, env={"BASEDIR": project_dir()}) @@ -618,12 +619,12 @@ def run(self): # 4. sleep 5 # 5. while ! nc -z localhost 4545; do sleep 5; done # GANACHE_LOG=ui/test/integration/vagrant/data/logs/ganache.$(filenamedate).txt - validator_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", "crumble", "sweet", "treat"] block_time = None # TODO account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") ganache_db_path = self.cmd.mktempdir() - ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=validator_mnemonic, - network_id=self.network_id, port=7545, db=ganache_db_path, account_keys_path=account_keys_path) + ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", + mnemonic=self.ganache_mnemonic, network_id=self.network_id, port=7545, db=ganache_db_path, + account_keys_path=account_keys_path) self.cmd.wait_for_file(account_keys_path) # Created by ganache-cli time.sleep(2) @@ -671,7 +672,7 @@ def run(self): networks_dir = project_dir("deploy/networks") self.cmd.rmdir(networks_dir) # networks_dir has many directories without write permission, so change those before deleting it self.cmd.mkdir(networks_dir) - self.cmd.execst(["rake", f"genesis:network:scaffold[{chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) + self.cmd.execst(["rake", f"genesis:network:scaffold[{self.chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) netdef = exactly_one(yaml_load(self.cmd.read_text_file(project_dir(networks_dir, "network-definition.yml")))) netdef_json = os.path.join(self.data_dir, "netdef.json") self.cmd.write_text_file(netdef_json, json.dumps(netdef)) @@ -680,7 +681,7 @@ def run(self): validator1_address = netdef["address"] validator1_password = netdef["password"] validator_mnemonic = netdef["mnemonic"].split(" ") - chaindir = os.path.join(networks_dir, f"validators/{chainnet}/{validator_moniker}") + chaindir = os.path.join(networks_dir, f"validators/{self.chainnet}/{validator_moniker}") # SIFNODED_LOG=$datadir/logs/sifnoded.log # test/integration/sifchain_start_daemon.sh: @@ -696,9 +697,10 @@ def run(self): "--home", sifchaind_home], pipe=False) self.cmd.execst(["sifnoded", "set-genesis-oracle-admin", adminuser_addr, "--home", sifchaind_home], pipe=False) sifnoded_proc = popen(["sifnoded", "start", "--minimum-gas-prices", sif_format_amount(0.5, "rowan"), - "--rpc.laddr", tcp_url, "--home", sifchaind_home]) + "--rpc.laddr", self.tcp_url, "--home", sifchaind_home]) # TODO Process exits immediately with returncode 1 + # TODO Why does it not stop start-integration-env.sh? rest_server_proc = popen(["sifnoded", "rest-server", "--laddr", "tcp://0.0.0.0:1317"]) # TODO cwd # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh @@ -709,9 +711,9 @@ def run(self): self.wait_for_sif_account(netdef_json, validator1_address) time.sleep(10) self.remove_and_add_sifnoded_keys(validator_moniker, validator_mnemonic) - ebrelayer_proc = self.cmd.ebrelayer_init(ebrelayer_ethereum_private_key, tcp_url, ethereum_websocket_address, - bridge_registry_sc_addr, validator_moniker, validator_mnemonic, chainnet, node=tcp_url, - keyring_backend="test", sign_with=validator_moniker) + ebrelayer_proc = self.cmd.ebrelayer_init(ebrelayer_ethereum_private_key, self.tcp_url, + self.ethereum_websocket_address, bridge_registry_sc_addr, validator_moniker, validator_mnemonic, + self.chainnet, node=self.tcp_url, keyring_backend="test", sign_with=validator_moniker) vagrantenv_path = project_dir("test/integration/vagrantenv.sh") self.state_vars = { @@ -743,13 +745,23 @@ def run(self): "VALIDATOR1_PASSWORD": validator1_password, "VALIDATOR1_ADDR": validator1_address, "MNEMONIC": " ".join(validator_mnemonic), - "CHAINDIR": os.path.join(networks_dir, "validators", chainnet, validator_moniker), + "CHAINDIR": os.path.join(networks_dir, "validators", self.chainnet, validator_moniker), "SIFCHAIN_ADMIN_ACCOUNT": adminuser_addr, # Needed by test_peggy_fees.py (via conftest.py) } - self.cmd.write_text_file(vagrantenv_path, joinlines([f"{k}=\"{v}\"" for k, v in self.state_vars.items()])) + self.write_vagrantenv_sh() return ganache_proc, sifnoded_proc, ebrelayer_proc, rest_server_proc + def write_vagrantenv_sh(self): + env = dict_merge(self.state_vars, { + # For running test/integration/execute_integration_tests_against_*.sh + "TEST_INTEGRATION": project_dir("test/integration"), + "TEST_INTEGRATION_PY_DIR": project_dir("test/integration/src/py"), + "SMART_CONTRACTS_DIR": self.cmd.smart_contracts_dir, + }) + vagrantenv_path = project_dir("test/integration/vagrantenv.sh") + self.cmd.write_text_file(vagrantenv_path, joinlines([f"export {k}=\"{v}\"" for k, v in env.items()])) + def wait_for_sif_account(self, netdef_json, validator1_address): return self.cmd.execst(["python3", os.path.join(self.test_integration_dir, "src/py/wait_for_sif_account.py"), netdef_json, validator1_address], env={"USER1ADDR": "nothing"}) @@ -775,6 +787,39 @@ def create_snapshot(self, snapshot_name): self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(named_snapshot_dir, "smart-contracts.tar.gz"), compression="gz") self.cmd.write_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"), json.dumps(self.state_vars, indent=4)) + def restart_processes(self): + block_time = None + ganache_db_path = self.state_vars["GANACHE_DB_DIR"] + account_keys_path = None + ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=self.ganache_mnemonic, + network_id=self.network_id, port=7545, db=ganache_db_path, account_keys_path=account_keys_path) + + validator_moniker = self.state_vars["MONIKER"] + networks_dir = project_dir("deploy/networks") + chaindir = os.path.join(networks_dir, f"validators/{self.chainnet}/{validator_moniker}") + sifchaind_home = os.path.join(chaindir, ".sifnoded") + sifnoded_proc = popen(["sifnoded", "start", "--minimum-gas-prices", sif_format_amount(0.5, "rowan"), + "--rpc.laddr", self.tcp_url, "--home", sifchaind_home]) + + bridge_token_sc_addr, bridge_registry_sc_addr, bridge_bank_sc_addr = \ + self.cmd.get_bridge_smart_contract_addresses(self.network_id) + + validator_mnemonic = self.state_vars["MNEMONIC"].split(" ") + account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") + ganache_keys = json.loads(self.cmd.read_text_file(account_keys_path)) + ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] + ebrelayer_ethereum_private_key = ganache_keys["private_keys"][ebrelayer_ethereum_addr] + + ebrelayer_proc = self.cmd.ebrelayer_init(ebrelayer_ethereum_private_key, self.tcp_url, + self.ethereum_websocket_address, bridge_registry_sc_addr, validator_moniker, validator_mnemonic, + self.chainnet, node=self.tcp_url, keyring_backend="test", sign_with=validator_moniker) + + return ganache_proc, sifnoded_proc, ebrelayer_proc, None + + def restore_snapshot(self, snapshot_name): + named_snapshot_dir = os.path.join(self.snapshots_dir, snapshot_name) + self.state_vars = json.loads(self.cmd.read_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"))) + def cleanup_and_reset_state(): # git checkout 4cb7322b6b282babd93a0d0aedda837c9134e84e deploy @@ -801,23 +846,35 @@ def cleanup_and_reset_state(): time.sleep(3) -def main(): +def main(argv): logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(message)s") - cleanup_and_reset_state() + what = argv[0] if argv else None cmd = Integrator() - ui_playbook = UIStackPlaybook(cmd) - # ui_playbook.stack_save_snapshot() - # ui_playbook.stack_push() - it_playbook = IntegrationTestsPlaybook(cmd) - processes = it_playbook.run() - for p in processes: - if p is not None: - p.kill() - it_playbook.create_snapshot("test_snapshot_1") + if what == "run_ui_playbook": + ui_playbook = UIStackPlaybook(cmd) + ui_playbook.stack_save_snapshot() + ui_playbook.stack_push() + elif what == "create_snapshot": + snapshot_name = argv[1] + cleanup_and_reset_state() + cmd = Integrator() + it_playbook = IntegrationTestsPlaybook(cmd) + processes = it_playbook.run() + for p in processes: + if p is not None: + p.kill() + processes1 = it_playbook.restart_processes() + it_playbook.create_snapshot(snapshot_name) + elif what == "restore_snapshot": + snapshot_name = argv[1] + it_playbook = IntegrationTestsPlaybook(cmd) + it_playbook.restore_snapshot(snapshot_name) + else: + raise Exception("Missing/unknown command") if __name__ == "__main__": - main() + main(sys.argv[1:]) # Trace of test_utilities.py get_required_env_var/get_optional_env_var: From 4a25f5179a358d2febd81cc7d28c93ef15deebb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 9 Aug 2021 14:42:59 +0200 Subject: [PATCH 20/35] Refactoring --- test/integration/make.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/make.py b/test/integration/make.py index bc2e0277bc..db561b4163 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -755,9 +755,11 @@ def run(self): def write_vagrantenv_sh(self): env = dict_merge(self.state_vars, { # For running test/integration/execute_integration_tests_against_*.sh - "TEST_INTEGRATION": project_dir("test/integration"), + "TEST_INTEGRATION_DIR": project_dir("test/integration"), "TEST_INTEGRATION_PY_DIR": project_dir("test/integration/src/py"), "SMART_CONTRACTS_DIR": self.cmd.smart_contracts_dir, + "datadir": self.data_dir, # Needed by test_rollback_chain.py that calls ganache_start.sh + "GANACHE_KEYS_JSON": os.path.join(self.data_dir, "ganachekeys.json"), # Needed by test_rollback_chain.py that calls ganache_start.sh }) vagrantenv_path = project_dir("test/integration/vagrantenv.sh") self.cmd.write_text_file(vagrantenv_path, joinlines([f"export {k}=\"{v}\"" for k, v in env.items()])) @@ -863,6 +865,7 @@ def main(argv): for p in processes: if p is not None: p.kill() + p.wait() processes1 = it_playbook.restart_processes() it_playbook.create_snapshot(snapshot_name) elif what == "restore_snapshot": From 297c05412ca545cce8581bebc4fc73593193a6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 10 Aug 2021 14:38:11 +0200 Subject: [PATCH 21/35] Compatibility fixes for integration tests --- test/integration/make.py | 78 ++++++++++--------- test/integration/src/py/conftest.py | 2 +- .../src/py/test_ebrelayer_replay.py | 2 +- .../src/py/test_ebrelayer_restart.py | 13 ++-- test/integration/src/py/test_utilities.py | 24 +++++- 5 files changed, 70 insertions(+), 49 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index db561b4163..fc1efc5cd3 100644 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -51,6 +51,9 @@ def dict_merge(*dicts): result[k] = v return result +def format_as_shell_env_vars(env, export=True): + return ["{}{}=\"{}\"".format("export " if export else "", k, v) for k, v in env.items()] + NULL_ADDRESS = "0x0000000000000000000000000000000000000000" @@ -733,8 +736,8 @@ def run(self): # export CHAINNET="localnet" "GANACHE_DB_DIR": ganache_db_path, # export GANACHE_KEYS_JSON="/home/jurez/work/projects/sif/sifnode/local/test/integration/vagrant/data/ganachekeys.json" - # export EBRELAYER_ETHEREUM_ADDR="0x5aeda56215b167893e80b4fe645ba6d5bab767de" - # export EBRELAYER_ETHEREUM_PRIVATE_KEY="8d5366123cb560bb606379f90a0bfd4769eecc0557f1b362dcae9012b548b1e5" + "EBRELAYER_ETHEREUM_ADDR": ebrelayer_ethereum_addr, + "EBRELAYER_ETHEREUM_PRIVATE_KEY": ebrelayer_ethereum_private_key, # Needed by sifchain_run_ebrelayer.sh # # BRIDGE_REGISTRY_ADDRESS and ETHEREUM_CONTRACT_ADDRESS are synonyms "BRIDGE_REGISTRY_ADDRESS": bridge_registry_sc_addr, "BRIDGE_TOKEN_ADDRESS": bridge_token_sc_addr, @@ -753,6 +756,38 @@ def run(self): return ganache_proc, sifnoded_proc, ebrelayer_proc, rest_server_proc def write_vagrantenv_sh(self): + # Trace of test_utilities.py get_required_env_var/get_optional_env_var: + # + # BASEDIR (required), value=/home/jurez/work/projects/sif/sifnode/local + # BRIDGE_BANK_ADDRESS (optional), value=0x30753E4A8aad7F8597332E813735Def5dD395028 + # BRIDGE_BANK_ADDRESS (required), value=0x30753E4A8aad7F8597332E813735Def5dD395028 + # BRIDGE_REGISTRY_ADDRESS (required), value=0xf204a4Ef082f5c04bB89F7D5E6568B796096735a + # BRIDGE_TOKEN_ADDRESS (optional), value=0x82D50AD3C1091866E258Fd0f1a7cC9674609D254 + # BRIDGE_TOKEN_ADDRESS (required), value=0x82D50AD3C1091866E258Fd0f1a7cC9674609D254 + # CHAINDIR (required), 3x value + # CHAINNET (required), value=localnet + # DEPLOYMENT_NAME (optional), value=None + # ETHEREUM_ADDRESS (optional), value=None + # ETHEREUM_NETWORK (optional), value=None + # ETHEREUM_NETWORK_ID (optional), value=None + # ETHEREUM_WEBSOCKET_ADDRESS (required), value=ws://localhost:7545/ + # GANACHE_KEYS_FILE (optional), value=None + # HOME (required), value=/home/jurez + # MNEMONIC (required), value=future tattoo gesture artist tomato accuse chuckle polar ivory strategy rail flower apart virus burger rhythm either describe habit attend absurd aspect predict parent + # MONIKER (required), value=wandering-flower + # OPERATOR_ADDRESS (optional), value=None + # OPERATOR_PRIVATE_KEY (optional), value=None + # OPERATOR_PRIVATE_KEY (optional), value=c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3 + # ROWAN_SOURCE (optional), value=None + # ROWAN_SOURCE_KEY (optional), value=None + # SIFCHAIN_ADMIN_ACCOUNT (required), value=sif1896ner48vrg8m05k48ykc6yydlxc4yvm23hp5m + # SIFNODE (optional), value=None + # SMART_CONTRACTS_DIR (required), 2x value + # SMART_CONTRACT_ARTIFACT_DIR (optional), value=None + # SOLIDITY_JSON_PATH (optional), value=None + # TEST_INTEGRATION_DIR (required), value=/home/jurez/work/projects/sif/sifnode/local/test/integration + # VALIDATOR1_ADDR (optional), 3x value + # VALIDATOR1_PASSWORD (optional), 3x value env = dict_merge(self.state_vars, { # For running test/integration/execute_integration_tests_against_*.sh "TEST_INTEGRATION_DIR": project_dir("test/integration"), @@ -760,9 +795,12 @@ def write_vagrantenv_sh(self): "SMART_CONTRACTS_DIR": self.cmd.smart_contracts_dir, "datadir": self.data_dir, # Needed by test_rollback_chain.py that calls ganache_start.sh "GANACHE_KEYS_JSON": os.path.join(self.data_dir, "ganachekeys.json"), # Needed by test_rollback_chain.py that calls ganache_start.sh + "ETHEREUM_WEBSOCKET_ADDRESS": self.ethereum_websocket_address, # Needed by test_ebrelayer_replay.py (and possibly others) + "CHAINNET": self.chainnet, # Needed by test_ebrelayer_replay.py (and possibly others) }) vagrantenv_path = project_dir("test/integration/vagrantenv.sh") - self.cmd.write_text_file(vagrantenv_path, joinlines([f"export {k}=\"{v}\"" for k, v in env.items()])) + self.cmd.write_text_file(vagrantenv_path, joinlines(format_as_shell_env_vars(env))) + self.cmd.write_text_file(project_dir("test/integration/vagrantenv.json"), json.dumps(env)) def wait_for_sif_account(self, netdef_json, validator1_address): return self.cmd.execst(["python3", os.path.join(self.test_integration_dir, "src/py/wait_for_sif_account.py"), @@ -878,37 +916,3 @@ def main(argv): if __name__ == "__main__": main(sys.argv[1:]) - - -# Trace of test_utilities.py get_required_env_var/get_optional_env_var: -# -# BASEDIR (required), value=/home/jurez/work/projects/sif/sifnode/local -# BRIDGE_BANK_ADDRESS (optional), value=0x30753E4A8aad7F8597332E813735Def5dD395028 -# BRIDGE_BANK_ADDRESS (required), value=0x30753E4A8aad7F8597332E813735Def5dD395028 -# BRIDGE_REGISTRY_ADDRESS (required), value=0xf204a4Ef082f5c04bB89F7D5E6568B796096735a -# BRIDGE_TOKEN_ADDRESS (optional), value=0x82D50AD3C1091866E258Fd0f1a7cC9674609D254 -# BRIDGE_TOKEN_ADDRESS (required), value=0x82D50AD3C1091866E258Fd0f1a7cC9674609D254 -# CHAINDIR (required), 3x value -# CHAINNET (required), value=localnet -# DEPLOYMENT_NAME (optional), value=None -# ETHEREUM_ADDRESS (optional), value=None -# ETHEREUM_NETWORK (optional), value=None -# ETHEREUM_NETWORK_ID (optional), value=None -# ETHEREUM_WEBSOCKET_ADDRESS (required), value=ws://localhost:7545/ -# GANACHE_KEYS_FILE (optional), value=None -# HOME (required), value=/home/jurez -# MNEMONIC (required), value=future tattoo gesture artist tomato accuse chuckle polar ivory strategy rail flower apart virus burger rhythm either describe habit attend absurd aspect predict parent -# MONIKER (required), value=wandering-flower -# OPERATOR_ADDRESS (optional), value=None -# OPERATOR_PRIVATE_KEY (optional), value=None -# OPERATOR_PRIVATE_KEY (optional), value=c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3 -# ROWAN_SOURCE (optional), value=None -# ROWAN_SOURCE_KEY (optional), value=None -# SIFCHAIN_ADMIN_ACCOUNT (required), value=sif1896ner48vrg8m05k48ykc6yydlxc4yvm23hp5m -# SIFNODE (optional), value=None -# SMART_CONTRACTS_DIR (required), 2x value -# SMART_CONTRACT_ARTIFACT_DIR (optional), value=None -# SOLIDITY_JSON_PATH (optional), value=None -# TEST_INTEGRATION_DIR (required), value=/home/jurez/work/projects/sif/sifnode/local/test/integration -# VALIDATOR1_ADDR (optional), 3x value -# VALIDATOR1_PASSWORD (optional), 3x value diff --git a/test/integration/src/py/conftest.py b/test/integration/src/py/conftest.py index acd0cc0913..7bacc2bdcc 100644 --- a/test/integration/src/py/conftest.py +++ b/test/integration/src/py/conftest.py @@ -251,7 +251,7 @@ def ensure_relayer_restart(integration_dir, smart_contracts_dir): logging.info("restart ebrelayer after advancing wait blocks - avoids any interaction with replaying blocks") original_log_level = decrease_log_level(new_level=logging.WARNING) test_utilities.advance_n_ethereum_blocks(test_utilities.n_wait_blocks + 1, smart_contracts_dir) - test_utilities.get_shell_output(f"{integration_dir}/sifchain_start_ebrelayer.sh") + test_utilities.start_ebrelayer() force_log_level(original_log_level) diff --git a/test/integration/src/py/test_ebrelayer_replay.py b/test/integration/src/py/test_ebrelayer_replay.py index 8b27737e0b..5eb051e7b2 100644 --- a/test/integration/src/py/test_ebrelayer_replay.py +++ b/test/integration/src/py/test_ebrelayer_replay.py @@ -41,7 +41,7 @@ def test_transfer_eth_to_ceth_using_replay_blocks( ): starting_block = test_utilities.current_ethereum_block_number(smart_contracts_dir) logging.info("stopping ebrelayer") - test_utilities.get_shell_output("pkill -9 ebrelayer || true") + test_utilities.kill_ebrelayer() request, credentials = build_request(smart_contracts_dir, source_ethereum_address, solidity_json_path) logging.info("(no transactions should happen without a relayer)") logging.info(f"send {request.amount / 10 ** 18} eth ({request.amount} wei) to {request.sifchain_address}") diff --git a/test/integration/src/py/test_ebrelayer_restart.py b/test/integration/src/py/test_ebrelayer_restart.py index bb24278c92..2b497733ca 100644 --- a/test/integration/src/py/test_ebrelayer_restart.py +++ b/test/integration/src/py/test_ebrelayer_restart.py @@ -14,7 +14,6 @@ def test_ebrelayer_restart( basic_transfer_request: EthereumToSifchainTransferRequest, source_ethereum_address: str, - integration_dir, ): basic_transfer_request.ethereum_address = source_ethereum_address request, credentials = generate_minimal_test_account( @@ -23,7 +22,7 @@ def test_ebrelayer_restart( ) balance = test_utilities.get_sifchain_addr_balance(request.sifchain_address, request.sifnoded_node, "ceth") logging.info("restart ebrelayer normally, leaving the last block db in place") - test_utilities.get_shell_output(f"{integration_dir}/sifchain_start_ebrelayer.sh") + test_utilities.start_ebrelayer() test_utilities.advance_n_ethereum_blocks(test_utilities.n_wait_blocks * 2, request.smart_contracts_dir) time.sleep(5) assert balance == test_utilities.get_sifchain_addr_balance(request.sifchain_address, request.sifnoded_node, @@ -36,7 +35,6 @@ def test_ethereum_transactions_with_offline_relayer( smart_contracts_dir, source_ethereum_address, bridgebank_address, - integration_dir, ): logging.debug("need one transaction to make sure ebrelayer writes out relaydb") basic_transfer_request.ethereum_address = source_ethereum_address @@ -47,7 +45,7 @@ def test_ethereum_transactions_with_offline_relayer( logging.info("shut down ebrelayer") time.sleep(10) - test_utilities.get_shell_output(f"pkill -9 ebrelayer || true") + test_utilities.kill_ebrelayer() logging.info("prepare transactions to be sent while ebrelayer is offline") amount = 9000 @@ -77,7 +75,7 @@ def test_ethereum_transactions_with_offline_relayer( ) logging.debug(f"bulk result: {yarn_result}") logging.info("restart ebrelayer with outstanding locks on the ethereum side") - test_utilities.get_shell_output(f"{integration_dir}/sifchain_start_ebrelayer.sh") + test_utilities.start_ebrelayer() time.sleep(5) for _ in new_addresses: # ebrelayer only reads blocks if there are new blocks generated @@ -95,7 +93,6 @@ def test_sifchain_transactions_with_offline_relayer( rowan_source, smart_contracts_dir, source_ethereum_address, - integration_dir, ): basic_transfer_request.ethereum_address = source_ethereum_address request, credentials = generate_test_account( @@ -107,7 +104,7 @@ def test_sifchain_transactions_with_offline_relayer( ) logging.info("shut down ebrelayer") time.sleep(10) - test_utilities.get_shell_output(f"pkill -9 ebrelayer || true") + test_utilities.kill_ebrelayer() logging.info("prepare transactions to be sent while ebrelayer is offline") amount = 9000 @@ -136,7 +133,7 @@ def test_sifchain_transactions_with_offline_relayer( time.sleep(5) logging.info("restart ebrelayer") - test_utilities.get_shell_output(f"{integration_dir}/sifchain_start_ebrelayer.sh") + test_utilities.start_ebrelayer() time.sleep(15) test_utilities.advance_n_ethereum_blocks(test_utilities.n_wait_blocks * 2, request.smart_contracts_dir) for a in new_eth_addrs: diff --git a/test/integration/src/py/test_utilities.py b/test/integration/src/py/test_utilities.py index e69c9d6aad..f3273e850c 100644 --- a/test/integration/src/py/test_utilities.py +++ b/test/integration/src/py/test_utilities.py @@ -97,15 +97,26 @@ def print_error_message(error_message): raise Exception(error_message) -def get_required_env_var(name, why: str = "by the system"): +def get_env_var(name): result = os.environ.get(name) + if result is None: + tmp = os.environ.get("VAGRANT_ENV_JSON") + if tmp: + import json + with open(tmp, "rt") as f: + env = json.loads(f.read()) + result = env.get(name) + return result + +def get_required_env_var(name, why: str = "by the system"): + result = get_env_var(name) if not result: print_error_message(f"{name} env var is required {why}") return result def get_optional_env_var(name: str, default_value: str): - result = os.environ.get(name) + result = get_env_var(name) return result if result else default_value @@ -167,6 +178,15 @@ def run_yarn_command(command_line): raise Exception(f"json error from command:\n{command_line}\noutput:\n{lines}\noriginal exception: {e}") +def kill_ebrelayer(): + return get_shell_output("pkill -9 ebrelayer || true") + + +def start_ebrelayer(): + integration_dir = get_required_env_var("TEST_INTEGRATION_DIR") + return get_shell_output(f"{integration_dir}/sifchain_start_ebrelayer.sh") + + # converts a key to a sif address. def get_user_account(user, network_password): command_line = "yes " + network_password + f" | {sifnoded_binary} keys show " + user + " -a" From 11a1e651d2d03e086ad167ac3ddaef5d85e61828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 14 Aug 2021 11:50:32 +0200 Subject: [PATCH 22/35] Proof of concept: snapshots --- .../execute_integration_tests_jure.sh | 19 +++++ test/integration/make.py | 78 ++++++++++++++----- test/integration/project | 13 ++++ 3 files changed, 90 insertions(+), 20 deletions(-) create mode 100755 test/integration/execute_integration_tests_jure.sh mode change 100644 => 100755 test/integration/make.py create mode 100755 test/integration/project diff --git a/test/integration/execute_integration_tests_jure.sh b/test/integration/execute_integration_tests_jure.sh new file mode 100755 index 0000000000..b9b41fe263 --- /dev/null +++ b/test/integration/execute_integration_tests_jure.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# this script can be run only against a test chain. It relies on tight control over +# ganache and knows where ganache stores its data. + +set -e + +. $(dirname $0)/vagrantenv.sh +. $(dirname $0)/shell_utilities.sh + +loglevel=${LOG_LEVEL:-DEBUG} + +logecho $0 starting + +python3 -m pytest -olog_level=$loglevel -v -olog_file=/tmp/log.txt -v \ + ${TEST_INTEGRATION_PY_DIR}/test_random_currency_roundtrip.py + +# ${TEST_INTEGRATION_PY_DIR}/test_ebrelayer_restart.py::test_ethereum_transactions_with_offline_relayer +# ${TEST_INTEGRATION_PY_DIR}/test_jure.py diff --git a/test/integration/make.py b/test/integration/make.py old mode 100644 new mode 100755 index fc1efc5cd3..9a167ef195 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -105,20 +105,34 @@ def mktempdir(self): def mktempfile(self): return exactly_one(stdout_lines(self.execst(["mktemp"]))) - def tar_create(self, path, tar_path, compression=None): - comp_opts = {"gz": "z"} - comp = comp_opts[compression] if compression in comp_opts else "" + def __tar_compression_option(self, tarfile): + filename = os.path.basename(tarfile).lower() + if filename.endswith(".tar"): + return "" + elif filename.endswith(".tar.gz"): + return "z" + else: + raise ValueError(f"Unknown extension for tar file: {tarfile}") + + def tar_create(self, path, tarfile): + comp = self.__tar_compression_option(tarfile) # tar on 9p filesystem reports "file shrank by ... bytes" and exits with errorcode 1 tar_quirks = True if tar_quirks: tmpdir = self.mktempdir() try: shutil.copytree(path, tmpdir, dirs_exist_ok=True) - self.execst(["tar", "cf" + comp, tar_path, "."], cwd=tmpdir) + self.execst(["tar", "cf" + comp, tarfile, "."], cwd=tmpdir) finally: self.rmdir(tmpdir) else: - self.execst(["tar", "cf" + comp, tar_path, "."], cwd=path) + self.execst(["tar", "cf" + comp, tarfile, "."], cwd=path) + + def tar_extract(self, tarfile, path): + comp = self.__tar_compression_option(tarfile) + if not self.exists(path): + self.mkdir(path) + self.execst(["tar", "xf" + comp, tarfile], cwd=path) class Ganache(Command): @@ -516,15 +530,15 @@ def stack_save_snapshot(self): self.cmd.mkdir(snapshots_dir) # TODO self.cmd.rmdir(snapshots_dir) # ui/chains/peggy/snapshot.sh: # mkdir -p ui/chains/peggy/relayerdb - self.cmd.tar_create(project_dir("ui/chains/peggy/relayerdb"), os.path.join(snapshots_dir, "peggy.tar.gz"), compression="gz") + self.cmd.tar_create(project_dir("ui/chains/peggy/relayerdb"), os.path.join(snapshots_dir, "peggy.tar.gz")) # mkdir -p smart-contracts/build - self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(snapshots_dir, "peggy_build.tar.gz"), compression="gz") + self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(snapshots_dir, "peggy_build.tar.gz")) # ui/chains/sif/snapshot.sh: - self.cmd.tar_create(self.sifnoded_path, os.path.join(snapshots_dir, "sif.tar.gz"), compression="gz") + self.cmd.tar_create(self.sifnoded_path, os.path.join(snapshots_dir, "sif.tar.gz")) # ui/chains/etc/snapshot.sh: - self.cmd.tar_create(self.ganache_db_path, os.path.join(snapshots_dir, "eth.tar.gz"), compression="gz") + self.cmd.tar_create(self.ganache_db_path, os.path.join(snapshots_dir, "eth.tar.gz")) def stack_push(self): # ui/scripts/stack-push.sh @@ -822,9 +836,10 @@ def create_snapshot(self, snapshot_name): raise Exception(f"Directory '{named_snapshot_dir}' already exists") self.cmd.mkdir(named_snapshot_dir) ganache_db_path = self.state_vars["GANACHE_DB_DIR"] - self.cmd.tar_create(ganache_db_path, os.path.join(named_snapshot_dir, "ganache.tar.gz"), compression="gz") - self.cmd.tar_create(project_dir("deploy/networks"), os.path.join(named_snapshot_dir, "networks.tar.gz"), compression="gz") - self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(named_snapshot_dir, "smart-contracts.tar.gz"), compression="gz") + self.cmd.tar_create(ganache_db_path, os.path.join(named_snapshot_dir, "ganache.tar.gz")) + self.cmd.tar_create(project_dir("test/integration/relayerdb"), os.path.join(named_snapshot_dir, "relayerdb.tar.gz")) + self.cmd.tar_create(project_dir("deploy/networks"), os.path.join(named_snapshot_dir, "networks.tar.gz")) + self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(named_snapshot_dir, "smart-contracts.tar.gz")) self.cmd.write_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"), json.dumps(self.state_vars, indent=4)) def restart_processes(self): @@ -858,8 +873,27 @@ def restart_processes(self): def restore_snapshot(self, snapshot_name): named_snapshot_dir = os.path.join(self.snapshots_dir, snapshot_name) - self.state_vars = json.loads(self.cmd.read_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"))) + state_vars = json.loads(self.cmd.read_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"))) + + def extract(tarfile, path): + self.cmd.rmdir(path) + self.cmd.mkdir(path) + self.cmd.tar_extract(os.path.join(named_snapshot_dir, tarfile), path) + + ganache_db_dir = self.cmd.mktempdir() + extract("ganache.tar.gz", ganache_db_dir) + relayerdb_dir = project_dir("test/integration/relayerdb") + extract("relayerdb.tar.gz", relayerdb_dir) + deploy_networks_dir = project_dir("deploy/networks") + extract("networks.tar.gz", deploy_networks_dir) + smart_contracts_build_dir = project_dir("smart-contracts/build") + extract("smart-contracts.tar.gz", smart_contracts_build_dir) + + state_vars["GANACHE_DB_DIR"] = ganache_db_dir + self.state_vars = state_vars + self.write_vagrantenv_sh() + return self.restart_processes() def cleanup_and_reset_state(): # git checkout 4cb7322b6b282babd93a0d0aedda837c9134e84e deploy @@ -885,6 +919,12 @@ def cleanup_and_reset_state(): # cmd.rmdir(project_dir("smart-contracts/.openzeppelin")) time.sleep(3) +def killall(processes): + # TODO Order - ebrelayer, sifnoded, ganache + for p in processes: + if p is not None: + p.kill() + p.wait() def main(argv): logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(message)s") @@ -897,19 +937,17 @@ def main(argv): elif what == "create_snapshot": snapshot_name = argv[1] cleanup_and_reset_state() - cmd = Integrator() it_playbook = IntegrationTestsPlaybook(cmd) processes = it_playbook.run() - for p in processes: - if p is not None: - p.kill() - p.wait() - processes1 = it_playbook.restart_processes() + killall(processes) + # processes1 = it_playbook.restart_processes() it_playbook.create_snapshot(snapshot_name) elif what == "restore_snapshot": snapshot_name = argv[1] it_playbook = IntegrationTestsPlaybook(cmd) - it_playbook.restore_snapshot(snapshot_name) + processes = it_playbook.restore_snapshot(snapshot_name) + input("Press ENTER to exit...") + killall(processes) else: raise Exception("Missing/unknown command") diff --git a/test/integration/project b/test/integration/project new file mode 100755 index 0000000000..1b5ce43fd7 --- /dev/null +++ b/test/integration/project @@ -0,0 +1,13 @@ +#!/bin/python3 + +import os +import sys + +basedir = os.path.abspath(os.path.join(os.path.normpath(os.path.join(__file__, *([os.path.pardir] * 3))))) +venv_dir = os.path.join(basedir, "venv") +test_integration_dir = os.path.join(basedir, "test/integration") + +sys.path = sys.path + [os.path.join(venv_dir, "lib/python3.6/site-packages"), os.path.join(test_integration_dir)] + +import make +make.main(sys.argv[1:]) From 80de4bbffa8ce1c7dd62c7a1ac3beb4f2a9fbe06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 16 Aug 2021 18:56:21 +0200 Subject: [PATCH 23/35] Refactor test_random_currency_roundtrip to use snapshots --- .../execute_integration_tests_jure.sh | 5 +- test/integration/make.py | 111 ++++---- test/integration/src/py/conftest.py | 236 ++++++++++++++++++ test/integration/src/py/integrator.py | 14 ++ test/integration/src/py/test_jure.py | 21 ++ .../py/test_random_currency_roundtrip_jure.py | 137 ++++++++++ 6 files changed, 477 insertions(+), 47 deletions(-) create mode 100644 test/integration/src/py/integrator.py create mode 100644 test/integration/src/py/test_jure.py create mode 100644 test/integration/src/py/test_random_currency_roundtrip_jure.py diff --git a/test/integration/execute_integration_tests_jure.sh b/test/integration/execute_integration_tests_jure.sh index b9b41fe263..f7299c90db 100755 --- a/test/integration/execute_integration_tests_jure.sh +++ b/test/integration/execute_integration_tests_jure.sh @@ -13,7 +13,4 @@ loglevel=${LOG_LEVEL:-DEBUG} logecho $0 starting python3 -m pytest -olog_level=$loglevel -v -olog_file=/tmp/log.txt -v \ - ${TEST_INTEGRATION_PY_DIR}/test_random_currency_roundtrip.py - -# ${TEST_INTEGRATION_PY_DIR}/test_ebrelayer_restart.py::test_ethereum_transactions_with_offline_relayer -# ${TEST_INTEGRATION_PY_DIR}/test_jure.py + ${TEST_INTEGRATION_PY_DIR}/test_random_currency_roundtrip_jure.py diff --git a/test/integration/make.py b/test/integration/make.py index 9a167ef195..3a0f9fbfac 100755 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -39,10 +39,10 @@ def http_get(url): with urllib.request.urlopen(url) as r: return r.read() -def popen(args, env=None): +def popen(args, env=None, cwd=None): if env: env = dict_merge(os.environ, env) - return subprocess.Popen(args, env=env) + return subprocess.Popen(args, env=env, cwd=cwd) def dict_merge(*dicts): result = {} @@ -216,7 +216,7 @@ def __init__(self): def ebrelayer_init(self, ethereum_private_key, tendermind_node, web3_provider, bridge_registry_contract_address, validator_moniker, validator_mnemonic, chain_id, gas=None, gas_prices=None, node=None, keyring_backend=None, - sign_with=None): + sign_with=None, cwd=None): env = {"ETHEREUM_PRIVATE_KEY": ethereum_private_key} args = ["ebrelayer", "init", tendermind_node, web3_provider, bridge_registry_contract_address, validator_moniker, " ".join(validator_mnemonic), "--chain-id={}".format(chain_id)] + \ @@ -225,7 +225,7 @@ def ebrelayer_init(self, ethereum_private_key, tendermind_node, web3_provider, b (["--node", node] if node is not None else []) + \ (["--keyring-backend", keyring_backend] if keyring_backend is not None else []) + \ (["--from", sign_with] if sign_with is not None else []) - return popen(args, env=env) + return popen(args, env=env, cwd=cwd) def sif_wait_up(self, host, port): while True: @@ -690,9 +690,8 @@ def run(self): self.cmd.rmdir(networks_dir) # networks_dir has many directories without write permission, so change those before deleting it self.cmd.mkdir(networks_dir) self.cmd.execst(["rake", f"genesis:network:scaffold[{self.chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) - netdef = exactly_one(yaml_load(self.cmd.read_text_file(project_dir(networks_dir, "network-definition.yml")))) - netdef_json = os.path.join(self.data_dir, "netdef.json") - self.cmd.write_text_file(netdef_json, json.dumps(netdef)) + + netdef, netdef_json = self.process_netdef(networks_dir) validator_moniker = netdef["moniker"] validator1_address = netdef["address"] @@ -723,14 +722,8 @@ def run(self): # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh # This script is also called from tests - while not self.cmd.tcp_probe_connect("localhost", 26657): - time.sleep(1) - self.wait_for_sif_account(netdef_json, validator1_address) - time.sleep(10) - self.remove_and_add_sifnoded_keys(validator_moniker, validator_mnemonic) - ebrelayer_proc = self.cmd.ebrelayer_init(ebrelayer_ethereum_private_key, self.tcp_url, - self.ethereum_websocket_address, bridge_registry_sc_addr, validator_moniker, validator_mnemonic, - self.chainnet, node=self.tcp_url, keyring_backend="test", sign_with=validator_moniker) + ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator_moniker, validator_mnemonic, + ebrelayer_ethereum_private_key, bridge_registry_sc_addr) vagrantenv_path = project_dir("test/integration/vagrantenv.sh") self.state_vars = { @@ -829,6 +822,28 @@ def remove_and_add_sifnoded_keys(self, validator_moniker, validator_mnemonic): self.cmd.sifnoded_keys_add([validator_moniker, "--keyring-backend", "test", "--recover"], stdin=[" ".join(validator_mnemonic)]) + def process_netdef(self, networks_dir): + # networks_dir = deploy/networks + # File deploy/networks/network-definition.yml is created by "rake genesis:network:scaffold" + # We read it and convert to test/integration/vagrant/data/netdef.json + netdef = exactly_one(yaml_load(self.cmd.read_text_file(project_dir(networks_dir, "network-definition.yml")))) + netdef_json = os.path.join(self.data_dir, "netdef.json") + self.cmd.write_text_file(netdef_json, json.dumps(netdef)) + return netdef, netdef_json + + def run_ebrelayer(self, netdef_json, validator1_address, validator_moniker, validator_mnemonic, + ebrelayer_ethereum_private_key, bridge_registry_sc_addr): + while not self.cmd.tcp_probe_connect("localhost", 26657): + time.sleep(1) + self.wait_for_sif_account(netdef_json, validator1_address) + time.sleep(10) + self.remove_and_add_sifnoded_keys(validator_moniker, validator_mnemonic) # Creates ~/.sifnoded/keyring-tests/xxxx.address + ebrelayer_proc = self.cmd.ebrelayer_init(ebrelayer_ethereum_private_key, self.tcp_url, + self.ethereum_websocket_address, bridge_registry_sc_addr, validator_moniker, validator_mnemonic, + self.chainnet, node=self.tcp_url, keyring_backend="test", sign_with=validator_moniker, + cwd=self.test_integration_dir) + return ebrelayer_proc + def create_snapshot(self, snapshot_name): self.cmd.mkdir(self.snapshots_dir) named_snapshot_dir = os.path.join(self.snapshots_dir, snapshot_name) @@ -840,15 +855,45 @@ def create_snapshot(self, snapshot_name): self.cmd.tar_create(project_dir("test/integration/relayerdb"), os.path.join(named_snapshot_dir, "relayerdb.tar.gz")) self.cmd.tar_create(project_dir("deploy/networks"), os.path.join(named_snapshot_dir, "networks.tar.gz")) self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(named_snapshot_dir, "smart-contracts.tar.gz")) + self.cmd.tar_create(self.cmd.get_user_home(".sifnoded"), os.path.join(named_snapshot_dir, "sifnoded.tar.gz")) self.cmd.write_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"), json.dumps(self.state_vars, indent=4)) + def restore_snapshot(self, snapshot_name): + named_snapshot_dir = os.path.join(self.snapshots_dir, snapshot_name) + state_vars = json.loads(self.cmd.read_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"))) + + def extract(tarfile, path): + self.cmd.rmdir(path) + self.cmd.mkdir(path) + self.cmd.tar_extract(os.path.join(named_snapshot_dir, tarfile), path) + + ganache_db_dir = self.cmd.mktempdir() + extract("ganache.tar.gz", ganache_db_dir) + relayerdb_dir = project_dir("test/integration/relayerdb") + extract("relayerdb.tar.gz", relayerdb_dir) + deploy_networks_dir = project_dir("deploy/networks") + extract("networks.tar.gz", deploy_networks_dir) + smart_contracts_build_dir = project_dir("smart-contracts/build") + extract("smart-contracts.tar.gz", smart_contracts_build_dir) + + state_vars["GANACHE_DB_DIR"] = ganache_db_dir + self.state_vars = state_vars + self.write_vagrantenv_sh() + self.cmd.mkdir(self.data_dir) + + return self.restart_processes() + def restart_processes(self): block_time = None ganache_db_path = self.state_vars["GANACHE_DB_DIR"] - account_keys_path = None + account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") # TODO this is in test/integration/vagrant/data, which is supposed to be cleared + ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=self.ganache_mnemonic, network_id=self.network_id, port=7545, db=ganache_db_path, account_keys_path=account_keys_path) + self.cmd.wait_for_file(account_keys_path) # Created by ganache-cli + time.sleep(2) + validator_moniker = self.state_vars["MONIKER"] networks_dir = project_dir("deploy/networks") chaindir = os.path.join(networks_dir, f"validators/{self.chainnet}/{validator_moniker}") @@ -865,36 +910,14 @@ def restart_processes(self): ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] ebrelayer_ethereum_private_key = ganache_keys["private_keys"][ebrelayer_ethereum_addr] - ebrelayer_proc = self.cmd.ebrelayer_init(ebrelayer_ethereum_private_key, self.tcp_url, - self.ethereum_websocket_address, bridge_registry_sc_addr, validator_moniker, validator_mnemonic, - self.chainnet, node=self.tcp_url, keyring_backend="test", sign_with=validator_moniker) + netdef, netdef_json = self.process_netdef(networks_dir) + validator1_address = netdef["address"] + assert validator1_address == self.state_vars["VALIDATOR1_ADDR"] + ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator_moniker, validator_mnemonic, + ebrelayer_ethereum_private_key, bridge_registry_sc_addr) return ganache_proc, sifnoded_proc, ebrelayer_proc, None - def restore_snapshot(self, snapshot_name): - named_snapshot_dir = os.path.join(self.snapshots_dir, snapshot_name) - state_vars = json.loads(self.cmd.read_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"))) - - def extract(tarfile, path): - self.cmd.rmdir(path) - self.cmd.mkdir(path) - self.cmd.tar_extract(os.path.join(named_snapshot_dir, tarfile), path) - - ganache_db_dir = self.cmd.mktempdir() - extract("ganache.tar.gz", ganache_db_dir) - relayerdb_dir = project_dir("test/integration/relayerdb") - extract("relayerdb.tar.gz", relayerdb_dir) - deploy_networks_dir = project_dir("deploy/networks") - extract("networks.tar.gz", deploy_networks_dir) - smart_contracts_build_dir = project_dir("smart-contracts/build") - extract("smart-contracts.tar.gz", smart_contracts_build_dir) - - state_vars["GANACHE_DB_DIR"] = ganache_db_dir - self.state_vars = state_vars - self.write_vagrantenv_sh() - - return self.restart_processes() - def cleanup_and_reset_state(): # git checkout 4cb7322b6b282babd93a0d0aedda837c9134e84e deploy # pkill node; pkill ebrelayer; pkill sifnoded; rm -rvf $HOME/.sifnoded; rm -rvf ./vagrant/data; mkdir vagrant/data @@ -939,6 +962,8 @@ def main(argv): cleanup_and_reset_state() it_playbook = IntegrationTestsPlaybook(cmd) processes = it_playbook.run() + # Give processes some time to settle, for example relayerdb must init and create its "relayerdb" + time.sleep(45) killall(processes) # processes1 = it_playbook.restart_processes() it_playbook.create_snapshot(snapshot_name) diff --git a/test/integration/src/py/conftest.py b/test/integration/src/py/conftest.py index 7bacc2bdcc..252fda1c90 100644 --- a/test/integration/src/py/conftest.py +++ b/test/integration/src/py/conftest.py @@ -339,3 +339,239 @@ def restore_default_rescue_location( transfer_request=basic_transfer_request, credentials=sifchain_admin_account_credentials ) + + +import importlib.util +import os +import threading +import logging + +basedir = os.path.abspath(os.path.join(__file__, *([os.path.pardir] * 5))) +path = os.path.join(basedir, "test/integration/make.py") +spec = importlib.util.spec_from_file_location("module.name", path) +make_py_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(make_py_module) + +threadlocals = threading.local() + + +class IntegrationTestContext: + def __init__(self, snapshot_name): + self.cmd = make_py_module.Integrator() + self.it_playbook = make_py_module.IntegrationTestsPlaybook(self.cmd) + self.processes = self.it_playbook.restore_snapshot(snapshot_name) + + def get_required_var(self, varname): + return test_utilities.get_required_env_var(varname) + + def get_optional_var(self, varname, default_value): + return test_utilities.get_optional_env_var(varname, default_value) + + def env_or_truffle_artifact(self, contract_name, contract_env_var, smart_contract_artifact_dir, ethereum_network_id): + result = self.get_optional_var(contract_env_var, None) + return result if result else test_utilities.contract_address( + smart_contract_artifact_dir=smart_contract_artifact_dir, + contract_name=contract_name, + ethereum_network_id=ethereum_network_id + ) + + @property + def sifnoded_node(self): + return self.get_optional_var("SIFNODE", None) + + @property + def sifnode_base_dir(self): + return self.get_required_var("BASEDIR") + + @property + def smart_contracts_dir(self): + return self.get_optional_var("SMART_CONTRACTS_DIR", os.path.join(self.sifnode_base_dir, "smart-contracts")) + + @property + def smart_contract_artifact_dir(self): + result = self.get_optional_var("SMART_CONTRACT_ARTIFACT_DIR", None) + return result if result else os.path.join(self.smart_contracts_dir, "build/contracts") + + @property + def bridgebank_address(self): + return self.env_or_truffle_artifact("BridgeBank", "BRIDGE_BANK_ADDRESS", self.smart_contract_artifact_dir, + self.ethereum_network_id) + + @property + def is_ropsten_testnet(self): + """if sifnode_clinode is set, we're talking to ropsten/sandpit""" + return bool(self.sifnoded_node) + + @property + def ethereum_network_id(self): + result = self.get_optional_var("ETHEREUM_NETWORK_ID", None) + if result: + return result + else: + return 3 if self.is_ropsten_testnet else 5777 + + @property + def bridgetoken_address(self): + return self.env_or_truffle_artifact("BridgeToken", "BRIDGE_TOKEN_ADDRESS", self.smart_contract_artifact_dir, + self.ethereum_network_id) + + @property + def ethereum_network(self): + return self.get_optional_var("ETHEREUM_NETWORK", "") + + @property + def chain_id(self): + return self.get_optional_var("DEPLOYMENT_NAME", "localnet") + + @property + def is_ganache(self): + """true if we're using ganache""" + return not self.ethereum_network + + @property + def sifchain_fees_int(self): + return 200000 + + @property + def sifchain_fees(self): + """returns a string suitable for passing to sifnoded""" + return f"{self.sifchain_fees_int}rowan" + + @property + def solidity_json_path(self): + return self.get_optional_var("SOLIDITY_JSON_PATH", f"{self.smart_contracts_dir}/build/contracts") + + @property + def ganache_owner_account(self): + return test_utilities.ganache_owner_account(self.smart_contracts_dir) + + @property + def source_ethereum_address(self): + """ + Account with some starting eth that can be transferred out. + + Our test wallet can only use one address/privatekey combination, + so if you set OPERATOR_ACCOUNT you have to set ETHEREUM_PRIVATE_KEY to the operator private key + """ + addr = self.get_optional_var("ETHEREUM_ADDRESS", "") + if addr: + logging.debug("using ETHEREUM_ADDRESS provided for source_ethereum_address") + return addr + if self.is_ropsten_testnet: + # Ropsten requires that you manually set the ETHEREUM_ADDRESS environment variable + assert addr + result = self.ganache_owner_account + logging.debug(f"Using source_ethereum_address {result} from ganache_owner_account. (Set ETHEREUM_ADDRESS env var to set it manually)") + assert result + return result + + @property + def validator_address(self): + return self.get_optional_var("VALIDATOR1_ADDR", None) + + @property + def validator_password(self): + return self.get_optional_var("VALIDATOR1_PASSWORD", None) + + @property + def rowan_source(self): + """A sifchain address or key that has rowan and can send that rowan to other address""" + result = self.get_optional_var("ROWAN_SOURCE", None) + if result: + return result + if self.is_ropsten_testnet: + assert result + else: + result = self.validator_address + assert result + return result + + @property + def sifnoded_homedir(self): + if self.is_ropsten_testnet: + base = self.get_required_var("HOME") + else: + base = self.get_required_var("CHAINDIR") + result = f"""{base}/.sifnoded""" + return result + + @property + def ganache_keys_file(self): + return self.get_optional_var("GANACHE_KEYS_FILE", + os.path.join(self.sifnode_base_dir, "test/integration/vagrant/data/ganachekeys.json")) + + @property + def operator_address(self): + return self.get_optional_var("OPERATOR_ADDRESS", test_utilities.ganache_owner_account(self.smart_contracts_dir)) + + @property + def operator_private_key(self): + result = self.get_optional_var( + "OPERATOR_PRIVATE_KEY", + test_utilities.ganache_private_key(self.ganache_keys_file, self.operator_address) + ) + return result + + def set_operator_private_key_env_var(self): + os.environ["OPERATOR_PRIVATE_KEY"] = self.operator_private_key + + @property + def rowan_source_integrationtest_env_credentials(self): + """ + Creates a SifchaincliCredentials with all the fields filled in + to transfer rowan from an account that already has rowan. + """ + return test_utilities.SifchaincliCredentials( + keyring_backend="file" if self.is_ganache else "test", + keyring_passphrase=self.validator_password, + from_key=self.rowan_source, + sifnoded_homedir=self.sifnoded_homedir + ) + + def rowan_source_integrationtest_env_transfer_request(self, basic_transfer_request): + """ + Creates a EthereumToSifchainTransferRequest with all the generic fields filled in + for a transfer of rowan from an account that already has rowan. + """ + result: test_utilities.EthereumToSifchainTransferRequest = copy.deepcopy(basic_transfer_request) + result.sifchain_address = self.rowan_source + result.sifchain_symbol = "rowan" + return result + + @property + def basic_transfer_request(self): + """ + Creates a EthereumToSifchainTransferRequest with all the generic fields filled in. + """ + return test_utilities.EthereumToSifchainTransferRequest( + smart_contracts_dir=self.smart_contracts_dir, + ethereum_private_key_env_var="ETHEREUM_PRIVATE_KEY", + bridgebank_address=self.bridgebank_address, + bridgetoken_address=self.bridgetoken_address, + ethereum_network=self.ethereum_network, + sifnoded_node=self.sifnoded_node, + manual_block_advance=self.is_ganache, + chain_id=self.chain_id, + sifchain_fees=self.sifchain_fees, + solidity_json_path=self.solidity_json_path) + + +@pytest.fixture(scope="function") +def with_snapshot(snapshot_name): + make_py_module.cleanup_and_reset_state() + ctx = IntegrationTestContext(snapshot_name) + logging.info("Started processes: {}".format(repr(ctx.processes))) + threadlocals.ctx = ctx + os.environ["VAGRANT_ENV_JSON"] = make_py_module.project_dir("test/integration/vagrantenv.json") # TODO HACK + print("Before with_snapshot()") + yield + print("After with_snapshot()") + make_py_module.killall(ctx.processes) # TODO Ensure this is called even in the case of exception + logging.info("Terminated processes: {}".format(repr(ctx.processes))) + del threadlocals.ctx + +@pytest.fixture +def ctx(snapshot_name): + print("Before test_context()") + yield threadlocals.ctx + print("After test_context()") diff --git a/test/integration/src/py/integrator.py b/test/integration/src/py/integrator.py new file mode 100644 index 0000000000..ea2014a9db --- /dev/null +++ b/test/integration/src/py/integrator.py @@ -0,0 +1,14 @@ +import logging +import pytest + + +class Integrator: + def hello(self): + print("Hello!") + logging.debug("Hello [debug]") + +cmd = Integrator() + +@pytest.fixture +def integration_env(): + return cmd diff --git a/test/integration/src/py/test_jure.py b/test/integration/src/py/test_jure.py new file mode 100644 index 0000000000..ada017ce92 --- /dev/null +++ b/test/integration/src/py/test_jure.py @@ -0,0 +1,21 @@ +import logging +import pytest +from integrator import cmd + +@pytest.fixture(autouse=True) +def around(): + logging.info("Before...") + logging.info("Before (debug)...") + yield + logging.info("After...") + +def test_jure1(): + print("Print something") + logging.debug("Debug message") + logging.info("Info message") + logging.warning("Warning message") + logging.error("Error message") + logging.info("Running test_jure1()...") + cmd.hello() + assert False + # test_ebrelayer_restart.py::test_ethereum_transactions_with_offline_relayer diff --git a/test/integration/src/py/test_random_currency_roundtrip_jure.py b/test/integration/src/py/test_random_currency_roundtrip_jure.py new file mode 100644 index 0000000000..9dfc821bd6 --- /dev/null +++ b/test/integration/src/py/test_random_currency_roundtrip_jure.py @@ -0,0 +1,137 @@ +import logging + +import pytest +import burn_lock_functions +import test_utilities +from burn_lock_functions import EthereumToSifchainTransferRequest +from pytest_utilities import generate_test_account +from test_utilities import get_required_env_var, get_shell_output, amount_in_wei, \ + SifchaincliCredentials + +# smart_contracts_dir = get_required_env_var("SMART_CONTRACTS_DIR") +# bridgebank_address = get_required_env_var("BRIDGE_BANK_ADDRESS") +# bridgetoken_address = get_required_env_var("BRIDGE_TOKEN_ADDRESS") + +smart_contracts_dir = None +bridgebank_address = None +bridgetoken_address = None + +# @pytest.mark.skip +@pytest.mark.usefixtures("with_snapshot") +@pytest.mark.parametrize("snapshot_name", ["s1"]) +def test_simple(ctx): + ctx.set_operator_private_key_env_var() + basic_transfer_request = ctx.basic_transfer_request + source_ethereum_address = ctx.source_ethereum_address + rowan_source_integrationtest_env_credentials = ctx.rowan_source_integrationtest_env_credentials + rowan_source_integrationtest_env_transfer_request = ctx.rowan_source_integrationtest_env_transfer_request(basic_transfer_request) + new_currency_symbol = ("a" + get_shell_output("uuidgen").replace("-", ""))[:4] + ethereum_network = ctx.ethereum_network + solidity_json_path = ctx.solidity_json_path + + global smart_contracts_dir + global bridgebank_address + global bridgetoken_address + smart_contracts_dir = get_required_env_var("SMART_CONTRACTS_DIR") + bridgebank_address = get_required_env_var("BRIDGE_BANK_ADDRESS") + bridgetoken_address = get_required_env_var("BRIDGE_TOKEN_ADDRESS") + + do_currency_test( + new_currency_symbol, + basic_transfer_request, + source_ethereum_address, + rowan_source_integrationtest_env_credentials, + rowan_source_integrationtest_env_transfer_request, + ethereum_network, + solidity_json_path=solidity_json_path + ) + +def do_currency_test( + new_currency_symbol, + basic_transfer_request: EthereumToSifchainTransferRequest, + source_ethereum_address: str, + rowan_source_integrationtest_env_credentials: SifchaincliCredentials, + rowan_source_integrationtest_env_transfer_request: EthereumToSifchainTransferRequest, + ethereum_network, + solidity_json_path, +): + amount = amount_in_wei(9) + logging.info(f"create new currency") + new_currency = test_utilities.create_new_currency( + amount, + new_currency_symbol, + new_currency_symbol, + 18, + smart_contracts_dir=smart_contracts_dir, + bridgebank_address=bridgebank_address, + solidity_json_path=solidity_json_path + ) + + logging.info(f"create test account to use with new currency {new_currency_symbol}") + basic_transfer_request.ethereum_address = source_ethereum_address + request, credentials = generate_test_account( + basic_transfer_request, + rowan_source_integrationtest_env_transfer_request, + rowan_source_integrationtest_env_credentials, + target_ceth_balance=10 ** 17, + target_rowan_balance=10 ** 18 + ) + test_amount = 39000 + logging.info(f"transfer some of the new currency {new_currency_symbol} to the test sifchain address") + request.ethereum_symbol = new_currency["newtoken_address"] + request.sifchain_symbol = ("c" + new_currency["newtoken_symbol"]).lower() + request.amount = test_amount + burn_lock_functions.transfer_ethereum_to_sifchain(request) + + logging.info("send some new currency to ethereum") + request.ethereum_address, _ = test_utilities.create_ethereum_address( + smart_contracts_dir, ethereum_network + ) + request.amount = test_amount - 1 + burn_lock_functions.transfer_sifchain_to_ethereum(request, credentials) + + +@pytest.mark.skip +@pytest.mark.usefixtures("with_snapshot") +@pytest.mark.parametrize("snapshot_name", ["s1"]) +@pytest.mark.usefixtures("operator_private_key") +def test_transfer_tokens_with_some_currency( + basic_transfer_request: EthereumToSifchainTransferRequest, + source_ethereum_address: str, + rowan_source_integrationtest_env_credentials: SifchaincliCredentials, + rowan_source_integrationtest_env_transfer_request: EthereumToSifchainTransferRequest, + ethereum_network, + solidity_json_path, +): + new_currency_symbol = ("a" + get_shell_output("uuidgen").replace("-", ""))[:4] + do_currency_test( + new_currency_symbol, + basic_transfer_request, + source_ethereum_address, + rowan_source_integrationtest_env_credentials, + rowan_source_integrationtest_env_transfer_request, + ethereum_network, + solidity_json_path=solidity_json_path + ) + + +@pytest.mark.skip +@pytest.mark.usefixtures("operator_private_key") +def test_three_letter_currency_with_capitals_in_name( + basic_transfer_request: EthereumToSifchainTransferRequest, + source_ethereum_address: str, + rowan_source_integrationtest_env_credentials: SifchaincliCredentials, + rowan_source_integrationtest_env_transfer_request: EthereumToSifchainTransferRequest, + ethereum_network, + solidity_json_path, +): + new_currency_symbol = ("F" + get_shell_output("uuidgen").replace("-", ""))[:3] + do_currency_test( + new_currency_symbol, + basic_transfer_request, + source_ethereum_address, + rowan_source_integrationtest_env_credentials, + rowan_source_integrationtest_env_transfer_request, + ethereum_network, + solidity_json_path=solidity_json_path + ) From 78ecc9c42b3858afe2fc3545d17c943e46d2d4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 23 Aug 2021 20:40:33 +0200 Subject: [PATCH 24/35] Cleanup --- .../py/test_random_currency_roundtrip_jure.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/test/integration/src/py/test_random_currency_roundtrip_jure.py b/test/integration/src/py/test_random_currency_roundtrip_jure.py index 9dfc821bd6..7a99962a7a 100644 --- a/test/integration/src/py/test_random_currency_roundtrip_jure.py +++ b/test/integration/src/py/test_random_currency_roundtrip_jure.py @@ -12,9 +12,9 @@ # bridgebank_address = get_required_env_var("BRIDGE_BANK_ADDRESS") # bridgetoken_address = get_required_env_var("BRIDGE_TOKEN_ADDRESS") -smart_contracts_dir = None -bridgebank_address = None -bridgetoken_address = None +# smart_contracts_dir = None +# bridgebank_address = None +# bridgetoken_address = None # @pytest.mark.skip @pytest.mark.usefixtures("with_snapshot") @@ -26,34 +26,36 @@ def test_simple(ctx): rowan_source_integrationtest_env_credentials = ctx.rowan_source_integrationtest_env_credentials rowan_source_integrationtest_env_transfer_request = ctx.rowan_source_integrationtest_env_transfer_request(basic_transfer_request) new_currency_symbol = ("a" + get_shell_output("uuidgen").replace("-", ""))[:4] - ethereum_network = ctx.ethereum_network - solidity_json_path = ctx.solidity_json_path + # ethereum_network = ctx.ethereum_network + # solidity_json_path = ctx.solidity_json_path - global smart_contracts_dir - global bridgebank_address - global bridgetoken_address - smart_contracts_dir = get_required_env_var("SMART_CONTRACTS_DIR") - bridgebank_address = get_required_env_var("BRIDGE_BANK_ADDRESS") - bridgetoken_address = get_required_env_var("BRIDGE_TOKEN_ADDRESS") + # global smart_contracts_dir + # global bridgebank_address + # global bridgetoken_address + # smart_contracts_dir = get_required_env_var("SMART_CONTRACTS_DIR") + # bridgebank_address = get_required_env_var("BRIDGE_BANK_ADDRESS") + # bridgetoken_address = get_required_env_var("BRIDGE_TOKEN_ADDRESS") + + # assert ctx.smart_contracts_dir == smart_contracts_dir + # assert ctx.bridgebank_address == bridgebank_address + # assert ctx.bridgetoken_address == bridgetoken_address do_currency_test( + ctx, new_currency_symbol, basic_transfer_request, source_ethereum_address, rowan_source_integrationtest_env_credentials, rowan_source_integrationtest_env_transfer_request, - ethereum_network, - solidity_json_path=solidity_json_path ) def do_currency_test( + ctx, new_currency_symbol, basic_transfer_request: EthereumToSifchainTransferRequest, source_ethereum_address: str, rowan_source_integrationtest_env_credentials: SifchaincliCredentials, rowan_source_integrationtest_env_transfer_request: EthereumToSifchainTransferRequest, - ethereum_network, - solidity_json_path, ): amount = amount_in_wei(9) logging.info(f"create new currency") @@ -62,9 +64,9 @@ def do_currency_test( new_currency_symbol, new_currency_symbol, 18, - smart_contracts_dir=smart_contracts_dir, - bridgebank_address=bridgebank_address, - solidity_json_path=solidity_json_path + smart_contracts_dir=ctx.smart_contracts_dir, + bridgebank_address=ctx.bridgebank_address, + solidity_json_path=ctx.solidity_json_path ) logging.info(f"create test account to use with new currency {new_currency_symbol}") @@ -85,7 +87,7 @@ def do_currency_test( logging.info("send some new currency to ethereum") request.ethereum_address, _ = test_utilities.create_ethereum_address( - smart_contracts_dir, ethereum_network + ctx.smart_contracts_dir, ctx.ethereum_network ) request.amount = test_amount - 1 burn_lock_functions.transfer_sifchain_to_ethereum(request, credentials) From 4a8d2f0c61213541bf33803184d62cd45ed1c481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 23 Aug 2021 22:03:37 +0200 Subject: [PATCH 25/35] Cleanup --- .../py/test_random_currency_roundtrip_jure.py | 112 +++++------------- 1 file changed, 29 insertions(+), 83 deletions(-) diff --git a/test/integration/src/py/test_random_currency_roundtrip_jure.py b/test/integration/src/py/test_random_currency_roundtrip_jure.py index 7a99962a7a..02ee91a975 100644 --- a/test/integration/src/py/test_random_currency_roundtrip_jure.py +++ b/test/integration/src/py/test_random_currency_roundtrip_jure.py @@ -5,57 +5,16 @@ import test_utilities from burn_lock_functions import EthereumToSifchainTransferRequest from pytest_utilities import generate_test_account -from test_utilities import get_required_env_var, get_shell_output, amount_in_wei, \ - SifchaincliCredentials +from test_utilities import get_shell_output, amount_in_wei, SifchaincliCredentials -# smart_contracts_dir = get_required_env_var("SMART_CONTRACTS_DIR") -# bridgebank_address = get_required_env_var("BRIDGE_BANK_ADDRESS") -# bridgetoken_address = get_required_env_var("BRIDGE_TOKEN_ADDRESS") - -# smart_contracts_dir = None -# bridgebank_address = None -# bridgetoken_address = None - -# @pytest.mark.skip -@pytest.mark.usefixtures("with_snapshot") -@pytest.mark.parametrize("snapshot_name", ["s1"]) -def test_simple(ctx): - ctx.set_operator_private_key_env_var() - basic_transfer_request = ctx.basic_transfer_request - source_ethereum_address = ctx.source_ethereum_address - rowan_source_integrationtest_env_credentials = ctx.rowan_source_integrationtest_env_credentials - rowan_source_integrationtest_env_transfer_request = ctx.rowan_source_integrationtest_env_transfer_request(basic_transfer_request) - new_currency_symbol = ("a" + get_shell_output("uuidgen").replace("-", ""))[:4] - # ethereum_network = ctx.ethereum_network - # solidity_json_path = ctx.solidity_json_path - - # global smart_contracts_dir - # global bridgebank_address - # global bridgetoken_address - # smart_contracts_dir = get_required_env_var("SMART_CONTRACTS_DIR") - # bridgebank_address = get_required_env_var("BRIDGE_BANK_ADDRESS") - # bridgetoken_address = get_required_env_var("BRIDGE_TOKEN_ADDRESS") - - # assert ctx.smart_contracts_dir == smart_contracts_dir - # assert ctx.bridgebank_address == bridgebank_address - # assert ctx.bridgetoken_address == bridgetoken_address - - do_currency_test( - ctx, - new_currency_symbol, - basic_transfer_request, - source_ethereum_address, - rowan_source_integrationtest_env_credentials, - rowan_source_integrationtest_env_transfer_request, - ) def do_currency_test( - ctx, - new_currency_symbol, - basic_transfer_request: EthereumToSifchainTransferRequest, - source_ethereum_address: str, - rowan_source_integrationtest_env_credentials: SifchaincliCredentials, - rowan_source_integrationtest_env_transfer_request: EthereumToSifchainTransferRequest, + ctx, + new_currency_symbol, + basic_transfer_request: EthereumToSifchainTransferRequest, + source_ethereum_address: str, + rowan_source_integrationtest_env_credentials: SifchaincliCredentials, + rowan_source_integrationtest_env_transfer_request: EthereumToSifchainTransferRequest, ): amount = amount_in_wei(9) logging.info(f"create new currency") @@ -66,8 +25,7 @@ def do_currency_test( 18, smart_contracts_dir=ctx.smart_contracts_dir, bridgebank_address=ctx.bridgebank_address, - solidity_json_path=ctx.solidity_json_path - ) + solidity_json_path=ctx.solidity_json_path) logging.info(f"create test account to use with new currency {new_currency_symbol}") basic_transfer_request.ethereum_address = source_ethereum_address @@ -75,9 +33,8 @@ def do_currency_test( basic_transfer_request, rowan_source_integrationtest_env_transfer_request, rowan_source_integrationtest_env_credentials, - target_ceth_balance=10 ** 17, - target_rowan_balance=10 ** 18 - ) + target_ceth_balance=10**17, + target_rowan_balance=10**18) test_amount = 39000 logging.info(f"transfer some of the new currency {new_currency_symbol} to the test sifchain address") request.ethereum_symbol = new_currency["newtoken_address"] @@ -87,53 +44,42 @@ def do_currency_test( logging.info("send some new currency to ethereum") request.ethereum_address, _ = test_utilities.create_ethereum_address( - ctx.smart_contracts_dir, ctx.ethereum_network - ) + ctx.smart_contracts_dir, ctx.ethereum_network) request.amount = test_amount - 1 burn_lock_functions.transfer_sifchain_to_ethereum(request, credentials) -@pytest.mark.skip @pytest.mark.usefixtures("with_snapshot") @pytest.mark.parametrize("snapshot_name", ["s1"]) -@pytest.mark.usefixtures("operator_private_key") -def test_transfer_tokens_with_some_currency( - basic_transfer_request: EthereumToSifchainTransferRequest, - source_ethereum_address: str, - rowan_source_integrationtest_env_credentials: SifchaincliCredentials, - rowan_source_integrationtest_env_transfer_request: EthereumToSifchainTransferRequest, - ethereum_network, - solidity_json_path, -): +def test_transfer_tokens_with_some_currency(ctx): + ctx.set_operator_private_key_env_var() # TODO Instead of @pytest.mark.usefixtures("operator_private_key") + basic_transfer_request = ctx.basic_transfer_request + source_ethereum_address = ctx.source_ethereum_address + rowan_source_integrationtest_env_credentials = ctx.rowan_source_integrationtest_env_credentials + rowan_source_integrationtest_env_transfer_request = ctx.rowan_source_integrationtest_env_transfer_request(basic_transfer_request) new_currency_symbol = ("a" + get_shell_output("uuidgen").replace("-", ""))[:4] do_currency_test( + ctx, new_currency_symbol, basic_transfer_request, source_ethereum_address, rowan_source_integrationtest_env_credentials, - rowan_source_integrationtest_env_transfer_request, - ethereum_network, - solidity_json_path=solidity_json_path - ) + rowan_source_integrationtest_env_transfer_request) -@pytest.mark.skip -@pytest.mark.usefixtures("operator_private_key") -def test_three_letter_currency_with_capitals_in_name( - basic_transfer_request: EthereumToSifchainTransferRequest, - source_ethereum_address: str, - rowan_source_integrationtest_env_credentials: SifchaincliCredentials, - rowan_source_integrationtest_env_transfer_request: EthereumToSifchainTransferRequest, - ethereum_network, - solidity_json_path, -): +@pytest.mark.usefixtures("with_snapshot") +@pytest.mark.parametrize("snapshot_name", ["s1"]) +def test_three_letter_currency_with_capitals_in_name(ctx): + ctx.set_operator_private_key_env_var() # TODO Instead of @pytest.mark.usefixtures("operator_private_key") + basic_transfer_request = ctx.basic_transfer_request + source_ethereum_address = ctx.source_ethereum_address + rowan_source_integrationtest_env_credentials = ctx.rowan_source_integrationtest_env_credentials + rowan_source_integrationtest_env_transfer_request = ctx.rowan_source_integrationtest_env_transfer_request(basic_transfer_request) new_currency_symbol = ("F" + get_shell_output("uuidgen").replace("-", ""))[:3] do_currency_test( + ctx, new_currency_symbol, basic_transfer_request, source_ethereum_address, rowan_source_integrationtest_env_credentials, - rowan_source_integrationtest_env_transfer_request, - ethereum_network, - solidity_json_path=solidity_json_path - ) + rowan_source_integrationtest_env_transfer_request) From 5a87b6995f254313e15630f62870fec786695571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 24 Aug 2021 15:21:49 +0200 Subject: [PATCH 26/35] WIP --- test/integration/.gitignore | 3 +- test/integration/make.py | 151 +++++++++++++++++++++++------------- 2 files changed, 100 insertions(+), 54 deletions(-) diff --git a/test/integration/.gitignore b/test/integration/.gitignore index f46c60a72b..169e47fdba 100644 --- a/test/integration/.gitignore +++ b/test/integration/.gitignore @@ -1,5 +1,6 @@ vagrantenv.sh +vagrantenv.json /__pycache__/ /watcher_pids.txt sandpit.log -../../smart-contracts/combined.log \ No newline at end of file +../../smart-contracts/combined.log diff --git a/test/integration/make.py b/test/integration/make.py index 3a0f9fbfac..98e3f4dfe5 100755 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -59,6 +59,7 @@ def format_as_shell_env_vars(env, export=True): class Command: def execst(self, args, cwd=None, env=None, stdin=None, binary=False, pipe=True, check_exit=True): + logging.debug(f"execst(): args={repr(args)}, cwd={repr(cwd)}") if stdin is not None: if type(stdin) == list: stdin = "".join([line + "\n" for line in stdin]) @@ -164,16 +165,23 @@ def sifnoded_generate_deterministic_account(self, name, mnemonic): res = self.execst(args, stdin=stdin) return yaml_load(res[1])[0] - def sifnoded_keys_show(self, name, bech=None, keyring_backend=None): - keyring_backend = "test" + def sifnoded_keys_show(self, name, bech=None, keyring_backend=None, home=None): + keyring_backend = keyring_backend or "test" args = ["sifnoded", "keys", "show", name] + \ (["--bech", bech] if bech else []) + \ - (["--keyring-backend={}".format(keyring_backend)] if keyring_backend else []) + (["--keyring-backend={}".format(keyring_backend)] if keyring_backend else []) + \ + (["--home", home] if home else []) res = self.execst(args) return yaml_load(res[1]) - def sifnoded_keys_add(self, args, stdin=None): - return yaml_load(self.execst(["sifnoded", "keys", "add"] + args, stdin=stdin)[1]) + def sifnoded_keys_add(self, moniker, mnemonic): + args = ["sifnoded", "keys", "add", moniker, "--keyring-backend", "test", "--recover"] + stdin = [" ".join(mnemonic)] + return yaml_load(self.execst(args, stdin=stdin)[1]) + + def sifnoded_keys_add_1(self, mnemonic): + args = ["sifnoded", "keys", "add", mnemonic, "--keyring-backend", "test"] + return yaml_load(self.execst(args, stdin=["y"])[1]) def sifnoded_add_genesis_account(self, address, tokens): tokens_str = ",".join([sif_format_amount(amount, denom) for amount, denom in tokens]) @@ -216,7 +224,7 @@ def __init__(self): def ebrelayer_init(self, ethereum_private_key, tendermind_node, web3_provider, bridge_registry_contract_address, validator_moniker, validator_mnemonic, chain_id, gas=None, gas_prices=None, node=None, keyring_backend=None, - sign_with=None, cwd=None): + sign_with=None, symbol_translator_file=None, relayerdb_path=None, cwd=None): env = {"ETHEREUM_PRIVATE_KEY": ethereum_private_key} args = ["ebrelayer", "init", tendermind_node, web3_provider, bridge_registry_contract_address, validator_moniker, " ".join(validator_mnemonic), "--chain-id={}".format(chain_id)] + \ @@ -224,7 +232,9 @@ def ebrelayer_init(self, ethereum_private_key, tendermind_node, web3_provider, b (["--gas-prices", sif_format_amount(*gas_prices)] if gas_prices is not None else []) + \ (["--node", node] if node is not None else []) + \ (["--keyring-backend", keyring_backend] if keyring_backend is not None else []) + \ - (["--from", sign_with] if sign_with is not None else []) + (["--from", sign_with] if sign_with is not None else []) + \ + (["--symbol-translator-file", symbol_translator_file] if symbol_translator_file else []) + \ + (["--relayerdb-path", relayerdb_path] if relayerdb_path else []) return popen(args, env=env, cwd=cwd) def sif_wait_up(self, host, port): @@ -267,8 +277,8 @@ def split(lines): result[k] = v return result - def build_smart_contracts_for_integration_tests(self): - self.execst(["make", "clean-smartcontracts"], cwd=self.smart_contracts_dir) + def install_smart_contracts_dependencies(self): + self.execst(["make", "clean-smartcontracts"], cwd=self.smart_contracts_dir) # = rm -rf build .openzeppelin self.yarn(["install"], cwd=self.smart_contracts_dir) def _check_env_vs_file(self, env, env_path): @@ -334,6 +344,7 @@ def truffle_exec(self, script_name, *script_args, env=None): # Maybe: self.cmd.yarn(["integrationtest:setTokenLockBurnLimit", str(amount)]) self.execst(["npx", "truffle", "exec", script_path] + list(script_args), env=env, cwd=self.smart_contracts_dir) + # TODO setTokenLockBurnLimit is gone, possibly replaced by bulkSetTokenLockBurnLimit def set_token_lock_burn_limit(self, update_address, amount, ethereum_private_key, infura_project_id, local_provider): env = { "ETHEREUM_PRIVATE_KEY": ethereum_private_key, @@ -342,6 +353,7 @@ def set_token_lock_burn_limit(self, update_address, amount, ethereum_private_key "LOCAL_PROVIDER": local_provider, } # Needs: ETHEREUM_PRIVATE_KEY, INFURA_PROJECT_ID, LOCAL_PROVIDER, UPDATE_ADDRESS + # TODO script is no longer there! self.truffle_exec("setTokenLockBurnLimit", str(amount), env=env) @@ -612,13 +624,16 @@ def __init__(self, cmd): self.ganache_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", "crumble", "sweet", "treat"] + def make_go_binaries(self): + # make go binaries (TODO Makefile needs to be trimmed down, especially "find") + self.cmd.execst(["make"], cwd=self.test_integration_dir, env={"BASEDIR": project_dir()}) + def run(self): self.cmd.mkdir(self.data_dir) - # make go binaries (a lot of nonsense!) - self.cmd.execst(["make"], cwd=self.test_integration_dir, env={"BASEDIR": project_dir()}) + self.make_go_binaries() - self.cmd.build_smart_contracts_for_integration_tests() + self.cmd.install_smart_contracts_dependencies() if self.using_ganache_gui: ebrelayer_ethereum_addr = "0x8e2bE12daDbCcbf7c98DBb59f98f22DFF0eF3F2c" @@ -664,32 +679,44 @@ def run(self): bridge_token_sc_addr, bridge_registry_sc_addr, bridge_bank_sc_addr = \ self.cmd.get_bridge_smart_contract_addresses(self.network_id) - # # Proof of concept: restart ganache-cli - # time.sleep(10) - # ganache_proc.kill() - # ganache_proc = self.cmd.start_ganache_cli(block_time=block_time, host="0.0.0.0", mnemonic=validator_mnemonic, - # network_id=self.network_id, port=7545, db=ganache_db_path, account_keys_path=account_keys_path) - - # TODO This should be last (after return from setup_sifchain.sh) - burn_limits = [ - [NULL_ADDRESS, 31*10**18], - [bridge_token_sc_addr, 10**25], - ] - env_file_vars = self.cmd.primitive_parse_env_file(env_file) - for address, amount in burn_limits: - self.cmd.set_token_lock_burn_limit( - address, - amount, - env_file_vars["ETHEREUM_PRIVATE_KEY"], # != ebrelayer_ethereum_private_key - env_file_vars["INFURA_PROJECT_ID"], - env_file_vars["LOCAL_PROVIDER"], # for web3.js to connect to ganache - ) + # # TODO This should be last (after return from setup_sifchain.sh) + # burn_limits = [ + # [NULL_ADDRESS, 31*10**18], + # [bridge_token_sc_addr, 10**25], + # ] + # env_file_vars = self.cmd.primitive_parse_env_file(env_file) + # for address, amount in burn_limits: + # self.cmd.set_token_lock_burn_limit( + # address, + # amount, + # env_file_vars["ETHEREUM_PRIVATE_KEY"], # != ebrelayer_ethereum_private_key + # env_file_vars["INFURA_PROJECT_ID"], + # env_file_vars["LOCAL_PROVIDER"], # for web3.js to connect to ganache + # ) # test/integration/setup_sifchain.sh: networks_dir = project_dir("deploy/networks") self.cmd.rmdir(networks_dir) # networks_dir has many directories without write permission, so change those before deleting it self.cmd.mkdir(networks_dir) - self.cmd.execst(["rake", f"genesis:network:scaffold[{self.chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) + # Old: + # self.cmd.execst(["rake", f"genesis:network:scaffold[{self.chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) + # New: + # sifgen network create localnet 1 $NETWORKDIR 192.168.1.2 $NETWORKDIR/network-definition.yml --keyring-backend test \ + # --mint-amount 999999000000000000000000000rowan,1370000000000000000ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE + mint_amount = [[999999 * 10**21, "rowan"], [137 * 10**16, "ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE"]] + validator_count = 1 + seed_ip_address = "192.168.1.2" + + # Old call (no longer works either): + # sifgen network create localnet 1 /mnt/shared/work/projects/sif/sifnode/local-tmp/my/deploy/rake/../networks \ + # 192.168.1.2 /mnt/shared/work/projects/sif/sifnode/local-tmp/my/deploy/rake/../networks/network-definition.yml \ + # --keyring-backend file + # self.cmd.execst(["sifgen", "network", "create", "localnet", str(validator_count), networks_dir, seed_ip_address, + # os.path.join(networks_dir, "network-definition.yml"), "--keyring-backend", "file"]) + + self.cmd.execst(["sifgen", "network", "create", "localnet", str(validator_count), networks_dir, seed_ip_address, + os.path.join(networks_dir, "network-definition.yml"), "--keyring-backend", "file", "--mint-amount", + ",".join([sif_format_amount(*x) for x in mint_amount])]) netdef, netdef_json = self.process_netdef(networks_dir) @@ -698,22 +725,34 @@ def run(self): validator1_password = netdef["password"] validator_mnemonic = netdef["mnemonic"].split(" ") chaindir = os.path.join(networks_dir, f"validators/{self.chainnet}/{validator_moniker}") + sifnoded_home = os.path.join(chaindir, ".sifnoded") # SIFNODED_LOG=$datadir/logs/sifnoded.log + # now we have to add the validator key to the test keyring so the tests can send rowan from validator1 + # echo "$MNEMONIC" | sifnoded keys add $MONIKER --keyring-backend test --recover + self.cmd.sifnoded_keys_add(validator_moniker, validator_mnemonic) + # valoper=$(sifnoded keys show -a --bech val $MONIKER --home $CHAINDIR/.sifnoded --keyring-backend test) + valoper = self.cmd.sifnoded_keys_show(validator_moniker, bech="val", keyring_backend="test", home=sifnoded_home)[0]["address"] + # sifnoded add-genesis-validators $valoper --home $CHAINDIR/.sifnoded + self.cmd.execst(["sifnoded", "add-genesis-validators", valoper, "--home", sifnoded_home]) + # test/integration/sifchain_start_daemon.sh: - sifchaind_home = os.path.join(chaindir, ".sifnoded") + # whitelisted_validator=$(yes $VALIDATOR1_PASSWORD | sifnoded keys show --keyring-backend file -a --bech val \ + # $MONIKER --home $CHAINDIR/.sifnoded) whitelisted_validator = exactly_one(stdout_lines(self.cmd.execst(["sifnoded", "keys", "show", - "--keyring-backend", "file", "-a", "--bech", "val", validator_moniker, "--home", sifchaind_home], + "--keyring-backend", "file", "-a", "--bech", "val", validator_moniker, "--home", sifnoded_home], stdin=[validator1_password]))) log.info(f"Whitelisted validator: {whitelisted_validator}") - self.cmd.execst(["sifnoded", "add-genesis-validators", whitelisted_validator, "--home", sifchaind_home]) - adminuser_addr = json.loads(self.cmd.execst(["sifnoded", "keys", "add", "sifnodeadmin", "--keyring-backend", - "test", "--output", "json"], stdin=["y"])[1])["address"] + self.cmd.execst(["sifnoded", "add-genesis-validators", whitelisted_validator, "--home", sifnoded_home]) + adminuser_addr = self.cmd.sifnoded_keys_add_1("sifnodeadmin")["address"] self.cmd.execst(["sifnoded", "add-genesis-account", adminuser_addr, sif_format_amount(10**20, "rowan"), - "--home", sifchaind_home], pipe=False) - self.cmd.execst(["sifnoded", "set-genesis-oracle-admin", adminuser_addr, "--home", sifchaind_home], pipe=False) + "--home", sifnoded_home], pipe=False) + self.cmd.execst(["sifnoded", "set-genesis-oracle-admin", adminuser_addr, "--home", sifnoded_home], pipe=False) sifnoded_proc = popen(["sifnoded", "start", "--minimum-gas-prices", sif_format_amount(0.5, "rowan"), - "--rpc.laddr", self.tcp_url, "--home", sifchaind_home]) + "--rpc.laddr", self.tcp_url, "--home", sifnoded_home]) + # TODO + # sifnoded set-genesis-whitelister-admin $adminuser --home $CHAINDIR/.sifnoded + # sifnoded set-gen-denom-whitelist $SCRIPT_DIR/whitelisted-denoms.json --home $CHAINDIR/.sifnoded # TODO Process exits immediately with returncode 1 # TODO Why does it not stop start-integration-env.sh? @@ -722,8 +761,9 @@ def run(self): # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh # This script is also called from tests + relayer_db_path = os.path.join(self.test_integration_dir, "sifchainrelayerdb") ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator_moniker, validator_mnemonic, - ebrelayer_ethereum_private_key, bridge_registry_sc_addr) + ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path) vagrantenv_path = project_dir("test/integration/vagrantenv.sh") self.state_vars = { @@ -757,6 +797,7 @@ def run(self): "MNEMONIC": " ".join(validator_mnemonic), "CHAINDIR": os.path.join(networks_dir, "validators", self.chainnet, validator_moniker), "SIFCHAIN_ADMIN_ACCOUNT": adminuser_addr, # Needed by test_peggy_fees.py (via conftest.py) + "EBRELAYERDB_DB": relayer_db_path, # Created by sifchain_run_ebrelayer.sh, does not appear to be used anywhere at the moment } self.write_vagrantenv_sh() @@ -819,8 +860,7 @@ def remove_and_add_sifnoded_keys(self, validator_moniker, validator_mnemonic): # during tests that restart ebrelayer # res = self.cmd.execst(["sifnoded", "keys", "delete", moniker, "--keyring-backend", "test"], stdin=["y"]) self.cmd.execst(["sifnoded", "keys", "delete", validator_moniker, "--keyring-backend", "test"], stdin=["y"], check_exit=False) - self.cmd.sifnoded_keys_add([validator_moniker, "--keyring-backend", "test", "--recover"], - stdin=[" ".join(validator_mnemonic)]) + self.cmd.sifnoded_keys_add(validator_moniker, validator_mnemonic) def process_netdef(self, networks_dir): # networks_dir = deploy/networks @@ -832,7 +872,7 @@ def process_netdef(self, networks_dir): return netdef, netdef_json def run_ebrelayer(self, netdef_json, validator1_address, validator_moniker, validator_mnemonic, - ebrelayer_ethereum_private_key, bridge_registry_sc_addr): + ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path): while not self.cmd.tcp_probe_connect("localhost", 26657): time.sleep(1) self.wait_for_sif_account(netdef_json, validator1_address) @@ -841,7 +881,8 @@ def run_ebrelayer(self, netdef_json, validator1_address, validator_moniker, vali ebrelayer_proc = self.cmd.ebrelayer_init(ebrelayer_ethereum_private_key, self.tcp_url, self.ethereum_websocket_address, bridge_registry_sc_addr, validator_moniker, validator_mnemonic, self.chainnet, node=self.tcp_url, keyring_backend="test", sign_with=validator_moniker, - cwd=self.test_integration_dir) + symbol_translator_file=os.path.join(self.test_integration_dir, "config/symbol_translator.json"), + relayerdb_path=relayer_db_path, cwd=self.test_integration_dir) return ebrelayer_proc def create_snapshot(self, snapshot_name): @@ -850,9 +891,8 @@ def create_snapshot(self, snapshot_name): if self.cmd.exists(named_snapshot_dir): raise Exception(f"Directory '{named_snapshot_dir}' already exists") self.cmd.mkdir(named_snapshot_dir) - ganache_db_path = self.state_vars["GANACHE_DB_DIR"] - self.cmd.tar_create(ganache_db_path, os.path.join(named_snapshot_dir, "ganache.tar.gz")) - self.cmd.tar_create(project_dir("test/integration/relayerdb"), os.path.join(named_snapshot_dir, "relayerdb.tar.gz")) + self.cmd.tar_create(self.state_vars["GANACHE_DB_DIR"], os.path.join(named_snapshot_dir, "ganache.tar.gz")) + self.cmd.tar_create(self.state_vars["EBRELAYER_DB"], os.path.join(named_snapshot_dir, "sifchainrelayerdb.tar.gz")) self.cmd.tar_create(project_dir("deploy/networks"), os.path.join(named_snapshot_dir, "networks.tar.gz")) self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(named_snapshot_dir, "smart-contracts.tar.gz")) self.cmd.tar_create(self.cmd.get_user_home(".sifnoded"), os.path.join(named_snapshot_dir, "sifnoded.tar.gz")) @@ -868,15 +908,17 @@ def extract(tarfile, path): self.cmd.tar_extract(os.path.join(named_snapshot_dir, tarfile), path) ganache_db_dir = self.cmd.mktempdir() + relayer_db_path = self.state_vars["RELAYERDB_DIR"] # TODO use /tmp + assert os.path.normpath(relayer_db_path) == os.path.normpath(os.path.join(self.test_integration_dir, "sifchainrelayerdb")) extract("ganache.tar.gz", ganache_db_dir) - relayerdb_dir = project_dir("test/integration/relayerdb") - extract("relayerdb.tar.gz", relayerdb_dir) + extract("sifchainrelayerdb.tar.gz", relayer_db_path) deploy_networks_dir = project_dir("deploy/networks") extract("networks.tar.gz", deploy_networks_dir) smart_contracts_build_dir = project_dir("smart-contracts/build") extract("smart-contracts.tar.gz", smart_contracts_build_dir) state_vars["GANACHE_DB_DIR"] = ganache_db_dir + state_vars["EBRELAYER_DB"] = relayer_db_path self.state_vars = state_vars self.write_vagrantenv_sh() self.cmd.mkdir(self.data_dir) @@ -913,8 +955,9 @@ def restart_processes(self): netdef, netdef_json = self.process_netdef(networks_dir) validator1_address = netdef["address"] assert validator1_address == self.state_vars["VALIDATOR1_ADDR"] + relayer_db_path = self.state_vars["EBRELAYER_DB"] ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator_moniker, validator_mnemonic, - ebrelayer_ethereum_private_key, bridge_registry_sc_addr) + ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path) return ganache_proc, sifnoded_proc, ebrelayer_proc, None @@ -927,7 +970,7 @@ def cleanup_and_reset_state(): cmd.execst(["pkill", "sifnoded"], check_exit=False) # rm -rvf /tmp/tmp.xxxx (ganache DB, unique for every run) - cmd.rmdir(project_dir("test/integration/relayerdb")) + cmd.rmdir(project_dir("test/integration/sifchainrelayerdb")) # TODO move to /tmp cmd.rmdir(project_dir("smart-contracts/build")) cmd.rmdir(project_dir("test/integration/vagrant/data")) @@ -940,6 +983,8 @@ def cleanup_and_reset_state(): # cmd,rm(project_dir("smart-contracts/.env")) # cmd.rmdir(project_dir("deploy/networks")) # cmd.rmdir(project_dir("smart-contracts/.openzeppelin")) + + # rmdir ~/.cache/yarn time.sleep(3) def killall(processes): From e8152eef12fec19b3b61b69e7e4df4ecb464e883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 25 Aug 2021 06:51:14 +0200 Subject: [PATCH 27/35] Bug workarounds --- test/integration/make.py | 61 ++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 98e3f4dfe5..8e1f6fb2ce 100755 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -181,7 +181,7 @@ def sifnoded_keys_add(self, moniker, mnemonic): def sifnoded_keys_add_1(self, mnemonic): args = ["sifnoded", "keys", "add", mnemonic, "--keyring-backend", "test"] - return yaml_load(self.execst(args, stdin=["y"])[1]) + return exactly_one(yaml_load(self.execst(args, stdin=["y"])[1])) def sifnoded_add_genesis_account(self, address, tokens): tokens_str = ",".join([sif_format_amount(amount, denom) for amount, denom in tokens]) @@ -494,6 +494,7 @@ def stack_save_snapshot(self): "LOCAL_PROVIDER": "http://localhost:7545", } + # NOTE: this probably doesn't work anymore since setTokenLockBurnLimit.js was replaced burn_limits = [ [NULL_ADDRESS, 31 * 10 ** 18], [bridge_token_address, 10 ** 25], @@ -614,7 +615,7 @@ def __init__(self, cmd): self.network_name = "develop" self.network_id = 5777 self.using_ganache_gui = False - self.snapshots_dir = self.cmd.get_user_home(".sifnode-snapshots") + self.peruser_storage_dir = self.cmd.get_user_home(".sifnode-integration") self.state_vars = {} self.test_integration_dir = project_dir("test/integration") self.data_dir = project_dir("test/integration/vagrant/data") @@ -713,9 +714,9 @@ def run(self): # --keyring-backend file # self.cmd.execst(["sifgen", "network", "create", "localnet", str(validator_count), networks_dir, seed_ip_address, # os.path.join(networks_dir, "network-definition.yml"), "--keyring-backend", "file"]) - + # TODO Most likely, this should be "--keyring-backend file" self.cmd.execst(["sifgen", "network", "create", "localnet", str(validator_count), networks_dir, seed_ip_address, - os.path.join(networks_dir, "network-definition.yml"), "--keyring-backend", "file", "--mint-amount", + os.path.join(networks_dir, "network-definition.yml"), "--keyring-backend", "test", "--mint-amount", ",".join([sif_format_amount(*x) for x in mint_amount])]) netdef, netdef_json = self.process_netdef(networks_dir) @@ -729,34 +730,40 @@ def run(self): # SIFNODED_LOG=$datadir/logs/sifnoded.log # now we have to add the validator key to the test keyring so the tests can send rowan from validator1 - # echo "$MNEMONIC" | sifnoded keys add $MONIKER --keyring-backend test --recover self.cmd.sifnoded_keys_add(validator_moniker, validator_mnemonic) - # valoper=$(sifnoded keys show -a --bech val $MONIKER --home $CHAINDIR/.sifnoded --keyring-backend test) valoper = self.cmd.sifnoded_keys_show(validator_moniker, bech="val", keyring_backend="test", home=sifnoded_home)[0]["address"] - # sifnoded add-genesis-validators $valoper --home $CHAINDIR/.sifnoded self.cmd.execst(["sifnoded", "add-genesis-validators", valoper, "--home", sifnoded_home]) - # test/integration/sifchain_start_daemon.sh: - # whitelisted_validator=$(yes $VALIDATOR1_PASSWORD | sifnoded keys show --keyring-backend file -a --bech val \ - # $MONIKER --home $CHAINDIR/.sifnoded) - whitelisted_validator = exactly_one(stdout_lines(self.cmd.execst(["sifnoded", "keys", "show", - "--keyring-backend", "file", "-a", "--bech", "val", validator_moniker, "--home", sifnoded_home], - stdin=[validator1_password]))) - log.info(f"Whitelisted validator: {whitelisted_validator}") - self.cmd.execst(["sifnoded", "add-genesis-validators", whitelisted_validator, "--home", sifnoded_home]) + try: + # Probable bug in test/integration/sifchain_start_daemon.sh: + # whitelisted_validator=$(yes $VALIDATOR1_PASSWORD | sifnoded keys show --keyring-backend file -a \ + # --bech val $MONIKER --home $CHAINDIR/.sifnoded) + whitelisted_validator = exactly_one(stdout_lines(self.cmd.execst(["sifnoded", "keys", "show", + "--keyring-backend", "file", "-a", "--bech", "val", validator_moniker, "--home", sifnoded_home], + stdin=[validator1_password]))) + assert False + log.info(f"Whitelisted validator: {whitelisted_validator}") + self.cmd.execst(["sifnoded", "add-genesis-validators", whitelisted_validator, "--home", sifnoded_home]) + except: + log.error("Failed to get whitelisted validator (probable bug)", exc_info=True) + assert True adminuser_addr = self.cmd.sifnoded_keys_add_1("sifnodeadmin")["address"] self.cmd.execst(["sifnoded", "add-genesis-account", adminuser_addr, sif_format_amount(10**20, "rowan"), "--home", sifnoded_home], pipe=False) self.cmd.execst(["sifnoded", "set-genesis-oracle-admin", adminuser_addr, "--home", sifnoded_home], pipe=False) + self.cmd.execst(["sifnoded", "set-genesis-whitelister-admin", adminuser_addr, "--home", sifnoded_home]) + self.cmd.execst(["sifnoded", "set-gen-denom-whitelist", os.path.join(self.test_integration_dir, + "whitelisted-denoms.json"), "--home", sifnoded_home]) + + # Start sifnoded sifnoded_proc = popen(["sifnoded", "start", "--minimum-gas-prices", sif_format_amount(0.5, "rowan"), "--rpc.laddr", self.tcp_url, "--home", sifnoded_home]) - # TODO - # sifnoded set-genesis-whitelister-admin $adminuser --home $CHAINDIR/.sifnoded - # sifnoded set-gen-denom-whitelist $SCRIPT_DIR/whitelisted-denoms.json --home $CHAINDIR/.sifnoded + + # TODO: should we wait for sifnoded to come up before continuing? If so, how do we do it? # TODO Process exits immediately with returncode 1 # TODO Why does it not stop start-integration-env.sh? - rest_server_proc = popen(["sifnoded", "rest-server", "--laddr", "tcp://0.0.0.0:1317"]) # TODO cwd + # rest_server_proc = popen(["sifnoded", "rest-server", "--laddr", "tcp://0.0.0.0:1317"]) # TODO cwd # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh # This script is also called from tests @@ -797,11 +804,11 @@ def run(self): "MNEMONIC": " ".join(validator_mnemonic), "CHAINDIR": os.path.join(networks_dir, "validators", self.chainnet, validator_moniker), "SIFCHAIN_ADMIN_ACCOUNT": adminuser_addr, # Needed by test_peggy_fees.py (via conftest.py) - "EBRELAYERDB_DB": relayer_db_path, # Created by sifchain_run_ebrelayer.sh, does not appear to be used anywhere at the moment + "EBRELAYER_DB": relayer_db_path, # Created by sifchain_run_ebrelayer.sh, does not appear to be used anywhere at the moment } self.write_vagrantenv_sh() - return ganache_proc, sifnoded_proc, ebrelayer_proc, rest_server_proc + return ganache_proc, sifnoded_proc, ebrelayer_proc def write_vagrantenv_sh(self): # Trace of test_utilities.py get_required_env_var/get_optional_env_var: @@ -885,9 +892,13 @@ def run_ebrelayer(self, netdef_json, validator1_address, validator_moniker, vali relayerdb_path=relayer_db_path, cwd=self.test_integration_dir) return ebrelayer_proc + def create_own_dirs(self): + self.cmd.mkdir(self.peruser_storage_dir) + self.cmd.mkdir(os.path.join(self.peruser_storage_dir, "snapshots")) + def create_snapshot(self, snapshot_name): - self.cmd.mkdir(self.snapshots_dir) - named_snapshot_dir = os.path.join(self.snapshots_dir, snapshot_name) + self.create_own_dirs() + named_snapshot_dir = os.path.join(self.peruser_storage_dir, "snapshots", snapshot_name) if self.cmd.exists(named_snapshot_dir): raise Exception(f"Directory '{named_snapshot_dir}' already exists") self.cmd.mkdir(named_snapshot_dir) @@ -899,7 +910,7 @@ def create_snapshot(self, snapshot_name): self.cmd.write_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"), json.dumps(self.state_vars, indent=4)) def restore_snapshot(self, snapshot_name): - named_snapshot_dir = os.path.join(self.snapshots_dir, snapshot_name) + named_snapshot_dir = os.path.join(self.peruser_storage_dir, "snapshots", snapshot_name) state_vars = json.loads(self.cmd.read_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"))) def extract(tarfile, path): @@ -959,7 +970,7 @@ def restart_processes(self): ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator_moniker, validator_mnemonic, ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path) - return ganache_proc, sifnoded_proc, ebrelayer_proc, None + return ganache_proc, sifnoded_proc, ebrelayer_proc def cleanup_and_reset_state(): # git checkout 4cb7322b6b282babd93a0d0aedda837c9134e84e deploy From b518cf450f69f2a9d89d3481c0de8bf8671f8fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 25 Aug 2021 10:04:33 +0200 Subject: [PATCH 28/35] Bug fixes and workarounds --- test/integration/make.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 8e1f6fb2ce..ffae509a55 100755 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -919,8 +919,8 @@ def extract(tarfile, path): self.cmd.tar_extract(os.path.join(named_snapshot_dir, tarfile), path) ganache_db_dir = self.cmd.mktempdir() - relayer_db_path = self.state_vars["RELAYERDB_DIR"] # TODO use /tmp - assert os.path.normpath(relayer_db_path) == os.path.normpath(os.path.join(self.test_integration_dir, "sifchainrelayerdb")) + relayer_db_path = state_vars["EBRELAYER_DB"] # TODO use /tmp + assert os.path.realpath(relayer_db_path) == os.path.realpath(os.path.join(self.test_integration_dir, "sifchainrelayerdb")) extract("ganache.tar.gz", ganache_db_dir) extract("sifchainrelayerdb.tar.gz", relayer_db_path) deploy_networks_dir = project_dir("deploy/networks") From 0f7579006772f382307a6fcca3ffb365adb18ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 26 Aug 2021 02:52:15 +0200 Subject: [PATCH 29/35] Bugfixes --- test/integration/src/py/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/integration/src/py/conftest.py b/test/integration/src/py/conftest.py index 252fda1c90..ef7590048b 100644 --- a/test/integration/src/py/conftest.py +++ b/test/integration/src/py/conftest.py @@ -522,10 +522,9 @@ def rowan_source_integrationtest_env_credentials(self): to transfer rowan from an account that already has rowan. """ return test_utilities.SifchaincliCredentials( - keyring_backend="file" if self.is_ganache else "test", + keyring_backend="test", keyring_passphrase=self.validator_password, - from_key=self.rowan_source, - sifnoded_homedir=self.sifnoded_homedir + from_key=self.rowan_source ) def rowan_source_integrationtest_env_transfer_request(self, basic_transfer_request): From bcd2aa06b156b3465f9891d8502876fcf1fa79f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 26 Aug 2021 04:09:08 +0200 Subject: [PATCH 30/35] Cleanup --- test/integration/make.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/integration/make.py b/test/integration/make.py index ffae509a55..4537a0ea89 100755 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -1029,6 +1029,19 @@ def main(argv): processes = it_playbook.restore_snapshot(snapshot_name) input("Press ENTER to exit...") killall(processes) + elif what == "fullclean": + cmd.execst(["chmod", "-R", "+w", cmd.get_user_home("go")]) + cmd.rmdir(cmd.get_user_home("go")) + cmd.mkdir(cmd.get_user_home("go")) + cmd.rmdir(cmd.get_user_home(".npm")) + cmd.rmdir(cmd.get_user_home(".npm-global")) + cmd.mkdir(cmd.get_user_home(".npm-global")) + cmd.rmdir(cmd.get_user_home(".cache/yarn")) + cmd.rmdir(cmd.get_user_home(".sifnoded")) + cmd.rmdir(cmd.get_user_home(".sifnode-integration")) + cmd.rmdir(project_dir("smart-contracts/node_modules")) + cmd.execst(["npm", "install", "-g", "ganache-cli", "dotenv", "yarn"], cwd=project_dir("smart-contracts")) + cmd.install_smart_contracts_dependencies() else: raise Exception("Missing/unknown command") From 3ea96c6d6bdcfa668b7bf8e257ad97903e20c3c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 26 Aug 2021 04:58:14 +0200 Subject: [PATCH 31/35] Cleanup --- test/integration/src/py/test_jure.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 test/integration/src/py/test_jure.py diff --git a/test/integration/src/py/test_jure.py b/test/integration/src/py/test_jure.py deleted file mode 100644 index ada017ce92..0000000000 --- a/test/integration/src/py/test_jure.py +++ /dev/null @@ -1,21 +0,0 @@ -import logging -import pytest -from integrator import cmd - -@pytest.fixture(autouse=True) -def around(): - logging.info("Before...") - logging.info("Before (debug)...") - yield - logging.info("After...") - -def test_jure1(): - print("Print something") - logging.debug("Debug message") - logging.info("Info message") - logging.warning("Warning message") - logging.error("Error message") - logging.info("Running test_jure1()...") - cmd.hello() - assert False - # test_ebrelayer_restart.py::test_ethereum_transactions_with_offline_relayer From 4b308042051e7d955a35340c9996d5df5a26b0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 26 Aug 2021 06:34:52 +0200 Subject: [PATCH 32/35] Cleanup --- ...ecute_integration_tests_with_snapshots.sh} | 0 test/integration/src/py/conftest.py | 237 ++---------------- .../src/py/integration_test_context.py | 210 ++++++++++++++++ ...ndom_currency_roundtrip_with_snapshots.py} | 0 4 files changed, 224 insertions(+), 223 deletions(-) rename test/integration/{execute_integration_tests_jure.sh => execute_integration_tests_with_snapshots.sh} (100%) create mode 100644 test/integration/src/py/integration_test_context.py rename test/integration/src/py/{test_random_currency_roundtrip_jure.py => test_random_currency_roundtrip_with_snapshots.py} (100%) diff --git a/test/integration/execute_integration_tests_jure.sh b/test/integration/execute_integration_tests_with_snapshots.sh similarity index 100% rename from test/integration/execute_integration_tests_jure.sh rename to test/integration/execute_integration_tests_with_snapshots.sh diff --git a/test/integration/src/py/conftest.py b/test/integration/src/py/conftest.py index ef7590048b..20a960a63f 100644 --- a/test/integration/src/py/conftest.py +++ b/test/integration/src/py/conftest.py @@ -1,8 +1,10 @@ import copy import logging import os +import threading import pytest +import integration_test_context import test_utilities from burn_lock_functions import decrease_log_level, force_log_level @@ -341,236 +343,25 @@ def restore_default_rescue_location( ) -import importlib.util -import os -import threading -import logging - -basedir = os.path.abspath(os.path.join(__file__, *([os.path.pardir] * 5))) -path = os.path.join(basedir, "test/integration/make.py") -spec = importlib.util.spec_from_file_location("module.name", path) -make_py_module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(make_py_module) - threadlocals = threading.local() - -class IntegrationTestContext: - def __init__(self, snapshot_name): - self.cmd = make_py_module.Integrator() - self.it_playbook = make_py_module.IntegrationTestsPlaybook(self.cmd) - self.processes = self.it_playbook.restore_snapshot(snapshot_name) - - def get_required_var(self, varname): - return test_utilities.get_required_env_var(varname) - - def get_optional_var(self, varname, default_value): - return test_utilities.get_optional_env_var(varname, default_value) - - def env_or_truffle_artifact(self, contract_name, contract_env_var, smart_contract_artifact_dir, ethereum_network_id): - result = self.get_optional_var(contract_env_var, None) - return result if result else test_utilities.contract_address( - smart_contract_artifact_dir=smart_contract_artifact_dir, - contract_name=contract_name, - ethereum_network_id=ethereum_network_id - ) - - @property - def sifnoded_node(self): - return self.get_optional_var("SIFNODE", None) - - @property - def sifnode_base_dir(self): - return self.get_required_var("BASEDIR") - - @property - def smart_contracts_dir(self): - return self.get_optional_var("SMART_CONTRACTS_DIR", os.path.join(self.sifnode_base_dir, "smart-contracts")) - - @property - def smart_contract_artifact_dir(self): - result = self.get_optional_var("SMART_CONTRACT_ARTIFACT_DIR", None) - return result if result else os.path.join(self.smart_contracts_dir, "build/contracts") - - @property - def bridgebank_address(self): - return self.env_or_truffle_artifact("BridgeBank", "BRIDGE_BANK_ADDRESS", self.smart_contract_artifact_dir, - self.ethereum_network_id) - - @property - def is_ropsten_testnet(self): - """if sifnode_clinode is set, we're talking to ropsten/sandpit""" - return bool(self.sifnoded_node) - - @property - def ethereum_network_id(self): - result = self.get_optional_var("ETHEREUM_NETWORK_ID", None) - if result: - return result - else: - return 3 if self.is_ropsten_testnet else 5777 - - @property - def bridgetoken_address(self): - return self.env_or_truffle_artifact("BridgeToken", "BRIDGE_TOKEN_ADDRESS", self.smart_contract_artifact_dir, - self.ethereum_network_id) - - @property - def ethereum_network(self): - return self.get_optional_var("ETHEREUM_NETWORK", "") - - @property - def chain_id(self): - return self.get_optional_var("DEPLOYMENT_NAME", "localnet") - - @property - def is_ganache(self): - """true if we're using ganache""" - return not self.ethereum_network - - @property - def sifchain_fees_int(self): - return 200000 - - @property - def sifchain_fees(self): - """returns a string suitable for passing to sifnoded""" - return f"{self.sifchain_fees_int}rowan" - - @property - def solidity_json_path(self): - return self.get_optional_var("SOLIDITY_JSON_PATH", f"{self.smart_contracts_dir}/build/contracts") - - @property - def ganache_owner_account(self): - return test_utilities.ganache_owner_account(self.smart_contracts_dir) - - @property - def source_ethereum_address(self): - """ - Account with some starting eth that can be transferred out. - - Our test wallet can only use one address/privatekey combination, - so if you set OPERATOR_ACCOUNT you have to set ETHEREUM_PRIVATE_KEY to the operator private key - """ - addr = self.get_optional_var("ETHEREUM_ADDRESS", "") - if addr: - logging.debug("using ETHEREUM_ADDRESS provided for source_ethereum_address") - return addr - if self.is_ropsten_testnet: - # Ropsten requires that you manually set the ETHEREUM_ADDRESS environment variable - assert addr - result = self.ganache_owner_account - logging.debug(f"Using source_ethereum_address {result} from ganache_owner_account. (Set ETHEREUM_ADDRESS env var to set it manually)") - assert result - return result - - @property - def validator_address(self): - return self.get_optional_var("VALIDATOR1_ADDR", None) - - @property - def validator_password(self): - return self.get_optional_var("VALIDATOR1_PASSWORD", None) - - @property - def rowan_source(self): - """A sifchain address or key that has rowan and can send that rowan to other address""" - result = self.get_optional_var("ROWAN_SOURCE", None) - if result: - return result - if self.is_ropsten_testnet: - assert result - else: - result = self.validator_address - assert result - return result - - @property - def sifnoded_homedir(self): - if self.is_ropsten_testnet: - base = self.get_required_var("HOME") - else: - base = self.get_required_var("CHAINDIR") - result = f"""{base}/.sifnoded""" - return result - - @property - def ganache_keys_file(self): - return self.get_optional_var("GANACHE_KEYS_FILE", - os.path.join(self.sifnode_base_dir, "test/integration/vagrant/data/ganachekeys.json")) - - @property - def operator_address(self): - return self.get_optional_var("OPERATOR_ADDRESS", test_utilities.ganache_owner_account(self.smart_contracts_dir)) - - @property - def operator_private_key(self): - result = self.get_optional_var( - "OPERATOR_PRIVATE_KEY", - test_utilities.ganache_private_key(self.ganache_keys_file, self.operator_address) - ) - return result - - def set_operator_private_key_env_var(self): - os.environ["OPERATOR_PRIVATE_KEY"] = self.operator_private_key - - @property - def rowan_source_integrationtest_env_credentials(self): - """ - Creates a SifchaincliCredentials with all the fields filled in - to transfer rowan from an account that already has rowan. - """ - return test_utilities.SifchaincliCredentials( - keyring_backend="test", - keyring_passphrase=self.validator_password, - from_key=self.rowan_source - ) - - def rowan_source_integrationtest_env_transfer_request(self, basic_transfer_request): - """ - Creates a EthereumToSifchainTransferRequest with all the generic fields filled in - for a transfer of rowan from an account that already has rowan. - """ - result: test_utilities.EthereumToSifchainTransferRequest = copy.deepcopy(basic_transfer_request) - result.sifchain_address = self.rowan_source - result.sifchain_symbol = "rowan" - return result - - @property - def basic_transfer_request(self): - """ - Creates a EthereumToSifchainTransferRequest with all the generic fields filled in. - """ - return test_utilities.EthereumToSifchainTransferRequest( - smart_contracts_dir=self.smart_contracts_dir, - ethereum_private_key_env_var="ETHEREUM_PRIVATE_KEY", - bridgebank_address=self.bridgebank_address, - bridgetoken_address=self.bridgetoken_address, - ethereum_network=self.ethereum_network, - sifnoded_node=self.sifnoded_node, - manual_block_advance=self.is_ganache, - chain_id=self.chain_id, - sifchain_fees=self.sifchain_fees, - solidity_json_path=self.solidity_json_path) - +@pytest.fixture +def ctx(snapshot_name): + logging.debug("Before ctx()") + yield threadlocals.ctx + logging.debug("After ctx()") @pytest.fixture(scope="function") def with_snapshot(snapshot_name): - make_py_module.cleanup_and_reset_state() - ctx = IntegrationTestContext(snapshot_name) + make = integration_test_context.make_py_module + make.cleanup_and_reset_state() + ctx = integration_test_context.IntegrationTestContext(snapshot_name) logging.info("Started processes: {}".format(repr(ctx.processes))) threadlocals.ctx = ctx - os.environ["VAGRANT_ENV_JSON"] = make_py_module.project_dir("test/integration/vagrantenv.json") # TODO HACK - print("Before with_snapshot()") + os.environ["VAGRANT_ENV_JSON"] = make.project_dir("test/integration/vagrantenv.json") # TODO HACK + logging.debug("Before with_snapshot()") yield - print("After with_snapshot()") - make_py_module.killall(ctx.processes) # TODO Ensure this is called even in the case of exception + logging.debug("After with_snapshot()") + make.killall(ctx.processes) # TODO Ensure this is called even in the case of exception logging.info("Terminated processes: {}".format(repr(ctx.processes))) del threadlocals.ctx - -@pytest.fixture -def ctx(snapshot_name): - print("Before test_context()") - yield threadlocals.ctx - print("After test_context()") diff --git a/test/integration/src/py/integration_test_context.py b/test/integration/src/py/integration_test_context.py new file mode 100644 index 0000000000..f5dbed4c4f --- /dev/null +++ b/test/integration/src/py/integration_test_context.py @@ -0,0 +1,210 @@ +import importlib.util +import os +import logging +import test_utilities + +basedir = os.path.abspath(os.path.join(__file__, *([os.path.pardir] * 5))) +path = os.path.join(basedir, "test/integration/make.py") +spec = importlib.util.spec_from_file_location("module.name", path) +make_py_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(make_py_module) + + +class IntegrationTestContext: + def __init__(self, snapshot_name): + self.cmd = make_py_module.Integrator() + self.it_playbook = make_py_module.IntegrationTestsPlaybook(self.cmd) + self.processes = self.it_playbook.restore_snapshot(snapshot_name) + + def get_required_var(self, varname): + return test_utilities.get_required_env_var(varname) + + def get_optional_var(self, varname, default_value): + return test_utilities.get_optional_env_var(varname, default_value) + + def env_or_truffle_artifact(self, contract_name, contract_env_var, smart_contract_artifact_dir, ethereum_network_id): + result = self.get_optional_var(contract_env_var, None) + return result if result else test_utilities.contract_address( + smart_contract_artifact_dir=smart_contract_artifact_dir, + contract_name=contract_name, + ethereum_network_id=ethereum_network_id + ) + + @property + def sifnoded_node(self): + return self.get_optional_var("SIFNODE", None) + + @property + def sifnode_base_dir(self): + return self.get_required_var("BASEDIR") + + @property + def smart_contracts_dir(self): + return self.get_optional_var("SMART_CONTRACTS_DIR", os.path.join(self.sifnode_base_dir, "smart-contracts")) + + @property + def smart_contract_artifact_dir(self): + result = self.get_optional_var("SMART_CONTRACT_ARTIFACT_DIR", None) + return result if result else os.path.join(self.smart_contracts_dir, "build/contracts") + + @property + def bridgebank_address(self): + return self.env_or_truffle_artifact("BridgeBank", "BRIDGE_BANK_ADDRESS", self.smart_contract_artifact_dir, + self.ethereum_network_id) + + @property + def is_ropsten_testnet(self): + """if sifnode_clinode is set, we're talking to ropsten/sandpit""" + return bool(self.sifnoded_node) + + @property + def ethereum_network_id(self): + result = self.get_optional_var("ETHEREUM_NETWORK_ID", None) + if result: + return result + else: + return 3 if self.is_ropsten_testnet else 5777 + + @property + def bridgetoken_address(self): + return self.env_or_truffle_artifact("BridgeToken", "BRIDGE_TOKEN_ADDRESS", self.smart_contract_artifact_dir, + self.ethereum_network_id) + + @property + def ethereum_network(self): + return self.get_optional_var("ETHEREUM_NETWORK", "") + + @property + def chain_id(self): + return self.get_optional_var("DEPLOYMENT_NAME", "localnet") + + @property + def is_ganache(self): + """true if we're using ganache""" + return not self.ethereum_network + + @property + def sifchain_fees_int(self): + return 200000 + + @property + def sifchain_fees(self): + """returns a string suitable for passing to sifnoded""" + return f"{self.sifchain_fees_int}rowan" + + @property + def solidity_json_path(self): + return self.get_optional_var("SOLIDITY_JSON_PATH", f"{self.smart_contracts_dir}/build/contracts") + + @property + def ganache_owner_account(self): + return test_utilities.ganache_owner_account(self.smart_contracts_dir) + + @property + def source_ethereum_address(self): + """ + Account with some starting eth that can be transferred out. + + Our test wallet can only use one address/privatekey combination, + so if you set OPERATOR_ACCOUNT you have to set ETHEREUM_PRIVATE_KEY to the operator private key + """ + addr = self.get_optional_var("ETHEREUM_ADDRESS", "") + if addr: + logging.debug("using ETHEREUM_ADDRESS provided for source_ethereum_address") + return addr + if self.is_ropsten_testnet: + # Ropsten requires that you manually set the ETHEREUM_ADDRESS environment variable + assert addr + result = self.ganache_owner_account + logging.debug(f"Using source_ethereum_address {result} from ganache_owner_account. (Set ETHEREUM_ADDRESS env var to set it manually)") + assert result + return result + + @property + def validator_address(self): + return self.get_optional_var("VALIDATOR1_ADDR", None) + + @property + def validator_password(self): + return self.get_optional_var("VALIDATOR1_PASSWORD", None) + + @property + def rowan_source(self): + """A sifchain address or key that has rowan and can send that rowan to other address""" + result = self.get_optional_var("ROWAN_SOURCE", None) + if result: + return result + if self.is_ropsten_testnet: + assert result + else: + result = self.validator_address + assert result + return result + + @property + def sifnoded_homedir(self): + if self.is_ropsten_testnet: + base = self.get_required_var("HOME") + else: + base = self.get_required_var("CHAINDIR") + result = f"""{base}/.sifnoded""" + return result + + @property + def ganache_keys_file(self): + return self.get_optional_var("GANACHE_KEYS_FILE", + os.path.join(self.sifnode_base_dir, "test/integration/vagrant/data/ganachekeys.json")) + + @property + def operator_address(self): + return self.get_optional_var("OPERATOR_ADDRESS", test_utilities.ganache_owner_account(self.smart_contracts_dir)) + + @property + def operator_private_key(self): + result = self.get_optional_var( + "OPERATOR_PRIVATE_KEY", + test_utilities.ganache_private_key(self.ganache_keys_file, self.operator_address) + ) + return result + + def set_operator_private_key_env_var(self): + os.environ["OPERATOR_PRIVATE_KEY"] = self.operator_private_key + + @property + def rowan_source_integrationtest_env_credentials(self): + """ + Creates a SifchaincliCredentials with all the fields filled in + to transfer rowan from an account that already has rowan. + """ + return test_utilities.SifchaincliCredentials( + keyring_backend="test", + keyring_passphrase=self.validator_password, + from_key=self.rowan_source + ) + + def rowan_source_integrationtest_env_transfer_request(self, basic_transfer_request): + """ + Creates a EthereumToSifchainTransferRequest with all the generic fields filled in + for a transfer of rowan from an account that already has rowan. + """ + result: test_utilities.EthereumToSifchainTransferRequest = copy.deepcopy(basic_transfer_request) + result.sifchain_address = self.rowan_source + result.sifchain_symbol = "rowan" + return result + + @property + def basic_transfer_request(self): + """ + Creates a EthereumToSifchainTransferRequest with all the generic fields filled in. + """ + return test_utilities.EthereumToSifchainTransferRequest( + smart_contracts_dir=self.smart_contracts_dir, + ethereum_private_key_env_var="ETHEREUM_PRIVATE_KEY", + bridgebank_address=self.bridgebank_address, + bridgetoken_address=self.bridgetoken_address, + ethereum_network=self.ethereum_network, + sifnoded_node=self.sifnoded_node, + manual_block_advance=self.is_ganache, + chain_id=self.chain_id, + sifchain_fees=self.sifchain_fees, + solidity_json_path=self.solidity_json_path) diff --git a/test/integration/src/py/test_random_currency_roundtrip_jure.py b/test/integration/src/py/test_random_currency_roundtrip_with_snapshots.py similarity index 100% rename from test/integration/src/py/test_random_currency_roundtrip_jure.py rename to test/integration/src/py/test_random_currency_roundtrip_with_snapshots.py From aa287d7da394be8800bbfb97e22508a1f120924e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 26 Aug 2021 12:52:33 +0200 Subject: [PATCH 33/35] Restore ~/.sifnoded for 'test' keyring --- test/integration/make.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/integration/make.py b/test/integration/make.py index 4537a0ea89..f6821f5a32 100755 --- a/test/integration/make.py +++ b/test/integration/make.py @@ -927,6 +927,7 @@ def extract(tarfile, path): extract("networks.tar.gz", deploy_networks_dir) smart_contracts_build_dir = project_dir("smart-contracts/build") extract("smart-contracts.tar.gz", smart_contracts_build_dir) + extract("sifnoded.tar.gz", self.cmd.get_user_home(".sifnoded")) # Needed for "--keyring-backend test" state_vars["GANACHE_DB_DIR"] = ganache_db_dir state_vars["EBRELAYER_DB"] = relayer_db_path @@ -984,9 +985,7 @@ def cleanup_and_reset_state(): cmd.rmdir(project_dir("test/integration/sifchainrelayerdb")) # TODO move to /tmp cmd.rmdir(project_dir("smart-contracts/build")) cmd.rmdir(project_dir("test/integration/vagrant/data")) - - # Not sure if this is needed too - cmd.rmdir(cmd.get_user_home(".sifnoded")) + cmd.rmdir(cmd.get_user_home(".sifnoded")) # Probably needed for "--keyring-backend test" # Additional cleanup (not neccessary to make it work) # cmd.rm(project_dir("smart-contracts/combined.log")) From ce967a76cff6e506ebcc813f24b98d9f5d08e76d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 26 Aug 2021 19:06:37 +0200 Subject: [PATCH 34/35] Cleanup / fixes --- .../execute_integration_tests_with_snapshots.sh | 3 ++- .../integration/src/py/integration_test_context.py | 1 + test/integration/src/py/integrator.py | 14 -------------- 3 files changed, 3 insertions(+), 15 deletions(-) delete mode 100644 test/integration/src/py/integrator.py diff --git a/test/integration/execute_integration_tests_with_snapshots.sh b/test/integration/execute_integration_tests_with_snapshots.sh index f7299c90db..25e56f00fc 100755 --- a/test/integration/execute_integration_tests_with_snapshots.sh +++ b/test/integration/execute_integration_tests_with_snapshots.sh @@ -12,5 +12,6 @@ loglevel=${LOG_LEVEL:-DEBUG} logecho $0 starting +# To run these tests you must first create a snapshot with "make.py create_snapshot s1" python3 -m pytest -olog_level=$loglevel -v -olog_file=/tmp/log.txt -v \ - ${TEST_INTEGRATION_PY_DIR}/test_random_currency_roundtrip_jure.py + ${TEST_INTEGRATION_PY_DIR}/test_random_currency_roundtrip_with_snapshots.py diff --git a/test/integration/src/py/integration_test_context.py b/test/integration/src/py/integration_test_context.py index f5dbed4c4f..03d0215f62 100644 --- a/test/integration/src/py/integration_test_context.py +++ b/test/integration/src/py/integration_test_context.py @@ -1,4 +1,5 @@ import importlib.util +import copy import os import logging import test_utilities diff --git a/test/integration/src/py/integrator.py b/test/integration/src/py/integrator.py deleted file mode 100644 index ea2014a9db..0000000000 --- a/test/integration/src/py/integrator.py +++ /dev/null @@ -1,14 +0,0 @@ -import logging -import pytest - - -class Integrator: - def hello(self): - print("Hello!") - logging.debug("Hello [debug]") - -cmd = Integrator() - -@pytest.fixture -def integration_env(): - return cmd From 77e4266cf7e95cc36db4217ad474ac9308801f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 31 Aug 2021 05:44:31 +0200 Subject: [PATCH 35/35] Add deprecation comments --- test/integration/src/py/conftest.py | 6 ++++++ test/integration/src/py/integration_test_context.py | 6 ++++++ test/integration/src/py/test_utilities.py | 8 ++++---- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/test/integration/src/py/conftest.py b/test/integration/src/py/conftest.py index 20a960a63f..b2923de6f7 100644 --- a/test/integration/src/py/conftest.py +++ b/test/integration/src/py/conftest.py @@ -180,12 +180,18 @@ def is_ganache(ethereum_network): return not ethereum_network +# Deprecated: sifnoded accepts --gas-prices=0.5rowan along with --gas-adjustment=1.5 instead of a fixed fee. +# Using those parameters is the best way to have the fees set robustly after the .42 upgrade. +# See https://github.com/Sifchain/sifnode/pull/1802#discussion_r697403408 @pytest.fixture def sifchain_fees(sifchain_fees_int): """returns a string suitable for passing to sifnoded""" return f"{sifchain_fees_int}rowan" +# Deprecated: sifnoded accepts --gas-prices=0.5rowan along with --gas-adjustment=1.5 instead of a fixed fee. +# Using those parameters is the best way to have the fees set robustly after the .42 upgrade. +# See https://github.com/Sifchain/sifnode/pull/1802#discussion_r697403408 @pytest.fixture def sifchain_fees_int(): return 200000 diff --git a/test/integration/src/py/integration_test_context.py b/test/integration/src/py/integration_test_context.py index 03d0215f62..211ea8f7a0 100644 --- a/test/integration/src/py/integration_test_context.py +++ b/test/integration/src/py/integration_test_context.py @@ -84,10 +84,16 @@ def is_ganache(self): """true if we're using ganache""" return not self.ethereum_network + # Deprecated: sifnoded accepts --gas-prices=0.5rowan along with --gas-adjustment=1.5 instead of a fixed fee. + # Using those parameters is the best way to have the fees set robustly after the .42 upgrade. + # See https://github.com/Sifchain/sifnode/pull/1802#discussion_r697403408 @property def sifchain_fees_int(self): return 200000 + # Deprecated: sifnoded accepts --gas-prices=0.5rowan along with --gas-adjustment=1.5 instead of a fixed fee. + # Using those parameters is the best way to have the fees set robustly after the .42 upgrade. + # See https://github.com/Sifchain/sifnode/pull/1802#discussion_r697403408 @property def sifchain_fees(self): """returns a string suitable for passing to sifnoded""" diff --git a/test/integration/src/py/test_utilities.py b/test/integration/src/py/test_utilities.py index f3273e850c..3e22ca5ad1 100644 --- a/test/integration/src/py/test_utilities.py +++ b/test/integration/src/py/test_utilities.py @@ -27,7 +27,7 @@ class EthereumToSifchainTransferRequest: ethereum_network: str = "" # mainnet, ropsten, http:// for localnet amount: int = 0 ceth_amount: int = 0 - sifchain_fees: str = "" + sifchain_fees: str = "" # Deprecated, see https://github.com/Sifchain/sifnode/pull/1802#discussion_r697403408 smart_contracts_dir: str = "" ethereum_chain_id: str = "5777" chain_id: str = "localnet" # cosmos chain id @@ -377,7 +377,7 @@ def send_from_sifchain_to_sifchain_cmd( keyring_backend_entry = f"--keyring-backend {credentials.keyring_backend}" if credentials.keyring_backend else "" chain_id_entry = f"--chain-id {transfer_request.chain_id}" if transfer_request.chain_id else "" node = f"--node {transfer_request.sifnoded_node}" if transfer_request.sifnoded_node else "" - sifchain_fees_entry = f"--fees {transfer_request.sifchain_fees}" if transfer_request.sifchain_fees else "" + sifchain_fees_entry = f"--fees {transfer_request.sifchain_fees}" if transfer_request.sifchain_fees else "" # Deprecated, see https://github.com/Sifchain/sifnode/pull/1802#discussion_r697403408 home_entry = f"--home {credentials.sifnoded_homedir}" if credentials.sifnoded_homedir else "" cmd = " ".join([ yes_entry, @@ -421,7 +421,7 @@ def send_from_sifchain_to_ethereum_cmd( yes_entry = f"yes {credentials.keyring_passphrase} | " if credentials.keyring_passphrase else "" keyring_backend_entry = f"--keyring-backend {credentials.keyring_backend}" if credentials.keyring_backend else "" node = f"--node {transfer_request.sifnoded_node}" if transfer_request.sifnoded_node else "" - sifchain_fees_entry = f"--fees {transfer_request.sifchain_fees}" if transfer_request.sifchain_fees else "" + sifchain_fees_entry = f"--fees {transfer_request.sifchain_fees}" if transfer_request.sifchain_fees else "" # Deprecated, see https://github.com/Sifchain/sifnode/pull/1802#discussion_r697403408 direction = "lock" if transfer_request.sifchain_symbol == "rowan" else "burn" home_entry = f"--home {credentials.sifnoded_homedir}" if credentials.sifnoded_homedir else "" from_entry = f"--from {credentials.from_key} " if credentials.from_key else "" @@ -736,7 +736,7 @@ def build_sifchain_command( node_entry = f"--node {transfer_request.sifnoded_node}" if transfer_request.sifnoded_node else "" home_entry = f"--home {credentials.sifnoded_homedir}" if credentials.sifnoded_homedir else "" from_entry = f"--from {credentials.from_key} " if credentials.from_key else "" - sifchain_fees_entry = f"--fees {transfer_request.sifchain_fees}" if transfer_request.sifchain_fees else "" + sifchain_fees_entry = f"--fees {transfer_request.sifchain_fees}" if transfer_request.sifchain_fees else "" # Deprecated, see https://github.com/Sifchain/sifnode/pull/1802#discussion_r697403408 return " ".join([ yes_entry, command_contents,