From 96559bebcf98fcd2243cab67dbdae9cfce6c94cd Mon Sep 17 00:00:00 2001 From: cyberthirst Date: Mon, 14 Oct 2024 22:21:43 +0200 Subject: [PATCH 01/70] chore[ci]: add auto-labeling workflow (#4276) auto-label new issues with "needs triage" --- .github/ISSUE_TEMPLATE/bug.md | 1 + .github/ISSUE_TEMPLATE/vip.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index a48e240a23..88f418fff4 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -1,6 +1,7 @@ --- name: Bug Report about: Any general feedback or bug reports about the Vyper Compiler. No new features proposals. +labels: ["needs triage"] --- ### Version Information diff --git a/.github/ISSUE_TEMPLATE/vip.md b/.github/ISSUE_TEMPLATE/vip.md index b35a1e7c23..d32a8ac3de 100644 --- a/.github/ISSUE_TEMPLATE/vip.md +++ b/.github/ISSUE_TEMPLATE/vip.md @@ -1,6 +1,7 @@ --- name: Vyper Improvement Proposal (VIP) about: This is the suggested template for new VIPs. +labels: ["needs triage"] --- ## Simple Summary "If you can't explain it simply, you don't understand it well enough." Provide a simplified and layman-accessible explanation of the VIP. From 33b9f8365e5e4e621bfef0aa9c58055bc6cf5754 Mon Sep 17 00:00:00 2001 From: cyberthirst Date: Mon, 14 Oct 2024 22:26:33 +0200 Subject: [PATCH 02/70] fix[test]: fix some clamper tests (#4300) was running some tests against ivy and found these failing in ivy (but ivy reported the true reason of the failure) --- tests/functional/codegen/features/test_clampers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/codegen/features/test_clampers.py b/tests/functional/codegen/features/test_clampers.py index 1adffcf29a..b82a771962 100644 --- a/tests/functional/codegen/features/test_clampers.py +++ b/tests/functional/codegen/features/test_clampers.py @@ -429,7 +429,7 @@ def foo(b: int128[6][1][2]) -> int128[6][1][2]: c = get_contract(code) with tx_failed(): - _make_tx(env, c.address, "foo(int128[6][1][2]])", values) + _make_tx(env, c.address, "foo(int128[6][1][2])", values) @pytest.mark.parametrize("value", [0, 1, -1, 2**127 - 1, -(2**127)]) @@ -453,7 +453,7 @@ def test_int128_dynarray_clamper_failing(env, tx_failed, get_contract, bad_value # ensure the invalid value is detected at all locations in the array code = """ @external -def foo(b: int128[5]) -> int128[5]: +def foo(b: DynArray[int128, 5]) -> DynArray[int128, 5]: return b """ From 6bcdb00e1c9faaf940395b44756d4970919fd58c Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Mon, 14 Oct 2024 23:55:18 +0200 Subject: [PATCH 03/70] feat[lang]: support top level `"abi"` key in json interfaces (#4279) some frameworks produce json ABI files with `"abi"` as a top-level key. this decreases a bit of friction for using those ABI files. --- .../unit/cli/vyper_json/test_compile_json.py | 24 +++++++++++++++++++ vyper/compiler/input_bundle.py | 2 ++ 2 files changed, 26 insertions(+) diff --git a/tests/unit/cli/vyper_json/test_compile_json.py b/tests/unit/cli/vyper_json/test_compile_json.py index ef3284cd15..5da98cf20f 100644 --- a/tests/unit/cli/vyper_json/test_compile_json.py +++ b/tests/unit/cli/vyper_json/test_compile_json.py @@ -295,3 +295,27 @@ def test_relative_import_paths(input_json): input_json["sources"]["contracts/potato/baz/potato.vy"] = {"content": "from . import baz"} input_json["sources"]["contracts/potato/footato.vy"] = {"content": "from baz import baz"} compile_from_input_dict(input_json) + + +def test_compile_json_with_abi_top(make_input_bundle): + stream = """ +{ + "abi": [ + { + "name": "validate", + "inputs": [ + { "name": "creator", "type": "address" }, + { "name": "token", "type": "address" }, + { "name": "amount_per_second", "type": "uint256" }, + { "name": "reason", "type": "bytes" } + ], + "outputs": [{ "name": "max_stream_life", "type": "uint256" }] + } + ] +} + """ + code = """ +from . import stream + """ + input_bundle = make_input_bundle({"stream.json": stream, "code.vy": code}) + vyper.compiler.compile_code(code, input_bundle=input_bundle) diff --git a/vyper/compiler/input_bundle.py b/vyper/compiler/input_bundle.py index a928989393..c9eeded3cf 100644 --- a/vyper/compiler/input_bundle.py +++ b/vyper/compiler/input_bundle.py @@ -52,6 +52,8 @@ class ABIInput(CompilerInput): def try_parse_abi(file_input: FileInput) -> CompilerInput: try: s = json.loads(file_input.source_code) + if isinstance(s, dict) and "abi" in s: + s = s["abi"] return ABIInput(**asdict(file_input), abi=s) except (ValueError, TypeError): return file_input From 990a6fac43a0571fe5f94324da6d1bc61c08eba2 Mon Sep 17 00:00:00 2001 From: tserg <8017125+tserg@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:57:19 +0800 Subject: [PATCH 04/70] feat[lang]: support flags from imported interfaces (#4253) this commit allows flag types to be imported from `.vyi` interface files. --------- Co-authored-by: Charles Cooper --- .../codegen/modules/test_interface_imports.py | 28 +++++++++ vyper/semantics/types/module.py | 57 ++++++++++--------- 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/tests/functional/codegen/modules/test_interface_imports.py b/tests/functional/codegen/modules/test_interface_imports.py index c0fae6496c..3f0f8cb010 100644 --- a/tests/functional/codegen/modules/test_interface_imports.py +++ b/tests/functional/codegen/modules/test_interface_imports.py @@ -58,3 +58,31 @@ def foo() -> bool: c = get_contract(main, input_bundle=input_bundle) assert c.foo() is True + + +def test_import_interface_flags(make_input_bundle, get_contract): + ifaces = """ +flag Foo: + BOO + MOO + POO + +interface IFoo: + def foo() -> Foo: nonpayable + """ + + contract = """ +import ifaces + +implements: ifaces + +@external +def foo() -> ifaces.Foo: + return ifaces.Foo.POO + """ + + input_bundle = make_input_bundle({"ifaces.vyi": ifaces}) + + c = get_contract(contract, input_bundle=input_bundle) + + assert c.foo() == 4 diff --git a/vyper/semantics/types/module.py b/vyper/semantics/types/module.py index d6cc50a2ea..dabeaf21b6 100644 --- a/vyper/semantics/types/module.py +++ b/vyper/semantics/types/module.py @@ -21,7 +21,7 @@ from vyper.semantics.types.base import TYPE_T, VyperType, is_type_t from vyper.semantics.types.function import ContractFunctionT from vyper.semantics.types.primitives import AddressT -from vyper.semantics.types.user import EventT, StructT, _UserType +from vyper.semantics.types.user import EventT, FlagT, StructT, _UserType from vyper.utils import OrderedSet if TYPE_CHECKING: @@ -45,27 +45,29 @@ def __init__( functions: dict, events: dict, structs: dict, + flags: dict, ) -> None: validate_unique_method_ids(list(functions.values())) - members = functions | events | structs + members = functions | events | structs | flags # sanity check: by construction, there should be no duplicates. - assert len(members) == len(functions) + len(events) + len(structs) + assert len(members) == len(functions) + len(events) + len(structs) + len(flags) super().__init__(functions) - self._helper = VyperType(events | structs) + self._helper = VyperType(events | structs | flags) self._id = _id self._helper._id = _id self.functions = functions self.events = events self.structs = structs + self.flags = flags self.decl_node = decl_node def get_type_member(self, attr, node): - # get an event or struct from this interface + # get an event, struct or flag from this interface return TYPE_T(self._helper.get_member(attr, node)) @property @@ -159,12 +161,14 @@ def _from_lists( interface_name: str, decl_node: Optional[vy_ast.VyperNode], function_list: list[tuple[str, ContractFunctionT]], - event_list: list[tuple[str, EventT]], - struct_list: list[tuple[str, StructT]], + event_list: Optional[list[tuple[str, EventT]]] = None, + struct_list: Optional[list[tuple[str, StructT]]] = None, + flag_list: Optional[list[tuple[str, FlagT]]] = None, ) -> "InterfaceT": - functions = {} - events = {} - structs = {} + functions: dict[str, ContractFunctionT] = {} + events: dict[str, EventT] = {} + structs: dict[str, StructT] = {} + flags: dict[str, FlagT] = {} seen_items: dict = {} @@ -175,19 +179,20 @@ def _mark_seen(name, item): raise NamespaceCollision(msg, item.decl_node, prev_decl=prev_decl) seen_items[name] = item - for name, function in function_list: - _mark_seen(name, function) - functions[name] = function + def _process(dst_dict, items): + if items is None: + return - for name, event in event_list: - _mark_seen(name, event) - events[name] = event + for name, item in items: + _mark_seen(name, item) + dst_dict[name] = item - for name, struct in struct_list: - _mark_seen(name, struct) - structs[name] = struct + _process(functions, function_list) + _process(events, event_list) + _process(structs, struct_list) + _process(flags, flag_list) - return cls(interface_name, decl_node, functions, events, structs) + return cls(interface_name, decl_node, functions, events, structs, flags) @classmethod def from_json_abi(cls, name: str, abi: dict) -> "InterfaceT": @@ -214,8 +219,7 @@ def from_json_abi(cls, name: str, abi: dict) -> "InterfaceT": for item in [i for i in abi if i.get("type") == "event"]: events.append((item["name"], EventT.from_abi(item))) - structs: list = [] # no structs in json ABI (as of yet) - return cls._from_lists(name, None, functions, events, structs) + return cls._from_lists(name, None, functions, events) @classmethod def from_ModuleT(cls, module_t: "ModuleT") -> "InterfaceT": @@ -247,8 +251,9 @@ def from_ModuleT(cls, module_t: "ModuleT") -> "InterfaceT": # these are accessible via import, but they do not show up # in the ABI json structs = [(node.name, node._metadata["struct_type"]) for node in module_t.struct_defs] + flags = [(node.name, node._metadata["flag_type"]) for node in module_t.flag_defs] - return cls._from_lists(module_t._id, module_t.decl_node, funcs, events, structs) + return cls._from_lists(module_t._id, module_t.decl_node, funcs, events, structs, flags) @classmethod def from_InterfaceDef(cls, node: vy_ast.InterfaceDef) -> "InterfaceT": @@ -265,11 +270,7 @@ def from_InterfaceDef(cls, node: vy_ast.InterfaceDef) -> "InterfaceT": ) functions.append((func_ast.name, ContractFunctionT.from_InterfaceDef(func_ast))) - # no structs or events in InterfaceDefs - events: list = [] - structs: list = [] - - return cls._from_lists(node.name, node, functions, events, structs) + return cls._from_lists(node.name, node, functions) # Datatype to store all module information. From 8f9a8cac49aafb3fbc9dde78f0f6125c390c32f0 Mon Sep 17 00:00:00 2001 From: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:40:11 +0200 Subject: [PATCH 05/70] refactor[venom]: move commutative instruction set (#4307) this commit moves the commutative instruction set from `venom_to_assembly.py` into `basicblock.py` and adds a convenience method in `IRInstruction`. --- vyper/venom/basicblock.py | 6 ++++++ vyper/venom/venom_to_assembly.py | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index 799dcfb33b..eb4f7e67ba 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -82,6 +82,8 @@ CFG_ALTERING_INSTRUCTIONS = frozenset(["jmp", "djmp", "jnz"]) +COMMUTATIVE_INSTRUCTIONS = frozenset(["add", "mul", "smul", "or", "xor", "and", "eq"]) + if TYPE_CHECKING: from vyper.venom.function import IRFunction @@ -235,6 +237,10 @@ def __init__( def is_volatile(self) -> bool: return self.opcode in VOLATILE_INSTRUCTIONS + @property + def is_commutative(self) -> bool: + return self.opcode in COMMUTATIVE_INSTRUCTIONS + @property def is_bb_terminator(self) -> bool: return self.opcode in BB_TERMINATORS diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index ee4b83201d..264ec35eee 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -103,9 +103,6 @@ ] ) -COMMUTATIVE_INSTRUCTIONS = frozenset(["add", "mul", "smul", "or", "xor", "and", "eq"]) - - _REVERT_POSTAMBLE = ["_sym___revert", "JUMPDEST", *PUSH(0), "DUP1", "REVERT"] @@ -431,7 +428,7 @@ def _generate_evm_for_instruction( # the same variable, however, before a jump that is not possible self._stack_reorder(assembly, stack, list(target_stack)) - if opcode in COMMUTATIVE_INSTRUCTIONS: + if inst.is_commutative: cost_no_swap = self._stack_reorder([], stack, operands, dry_run=True) operands[-1], operands[-2] = operands[-2], operands[-1] cost_with_swap = self._stack_reorder([], stack, operands, dry_run=True) From 039d36926cf19916ada19f36b7e8ee9b09c1eec5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 15 Oct 2024 13:03:50 -0400 Subject: [PATCH 06/70] fix[ci]: fix commithash calculation for pypi release (#4309) there is a mismatch between the commit hash in the binary of the github release vs the pypi release. for example, ```bash ~ $ vyper --version # pipx install vyper==0.4.0 0.4.0+commit.e9db8d9 ``` ```bash ~ $ .vvm/vyper-0.4.0 --version 0.4.0+commit.e9db8d9f ``` this is due to how git computes the shorthash. when checkout is run for release-pypi.yml, it doesn't fetch the full commit history, and so there are fewer commits, so `git rev-parse --short HEAD` returns a smaller fingerprint for the commit hash. this commit amends the pypi release checkout step so that it matches the github release workflow. it also adds a debug step to the relevant workflows so that we can debug the commit hash during the github action. --- .github/workflows/build.yml | 8 ++++++++ .github/workflows/release-pypi.yml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7cb8de830c..515be87c25 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,10 @@ jobs: # need to fetch unshallow so that setuptools_scm can infer the version fetch-depth: 0 + # debug + - name: Git shorthash + run: git rev-parse --short HEAD + - name: Python uses: actions/setup-python@v5 with: @@ -60,6 +64,10 @@ jobs: # need to fetch unshallow so that setuptools_scm can infer the version fetch-depth: 0 + # debug + - name: Git shorthash + run: git rev-parse --short HEAD + - name: Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index d09aeb9adc..1511c61e51 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -20,6 +20,14 @@ jobs: steps: - uses: actions/checkout@v4 + with: + # fetch unshallow so commit hash matches github release. + # see https://github.com/vyperlang/vyper/blob/8f9a8cac49aafb3fbc9dde78f0f6125c390c32f0/.github/workflows/build.yml#L27-L32 + fetch-depth: 0 + + # debug + - name: Git shorthash + run: git rev-parse --short HEAD - name: Python uses: actions/setup-python@v5 From b3ea6630d6bbe05168768ba162252e2e79ee2478 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 16 Oct 2024 21:34:40 +0300 Subject: [PATCH 07/70] fix[venom]: add `make_ssa` pass after algebraic optimizations (#4292) This commit adds a `MakeSSA` pass after algebraic optimisations. Makes `StoreElimination` pass skip `phi` instructions and adds a test case that would fail without this step. The `MakeSSA` pass here results in smaller code so we are keeping it in for now. Resolves GH issue #4288 --------- Co-authored-by: Charles Cooper --- .../venom/test_algebraic_optimizer.py | 27 +++++++++++++++++++ vyper/venom/__init__.py | 7 +++++ vyper/venom/analysis/cfg.py | 5 +++- vyper/venom/passes/sccp/sccp.py | 9 +++---- vyper/venom/passes/store_elimination.py | 7 ++--- 5 files changed, 46 insertions(+), 9 deletions(-) diff --git a/tests/unit/compiler/venom/test_algebraic_optimizer.py b/tests/unit/compiler/venom/test_algebraic_optimizer.py index 39008649ea..00ccb0684a 100644 --- a/tests/unit/compiler/venom/test_algebraic_optimizer.py +++ b/tests/unit/compiler/venom/test_algebraic_optimizer.py @@ -1,5 +1,6 @@ import pytest +import vyper from vyper.venom.analysis import IRAnalysesCache from vyper.venom.basicblock import IRBasicBlock, IRLabel from vyper.venom.context import IRContext @@ -176,3 +177,29 @@ def test_offsets(): offset_count += 1 assert offset_count == 3 + + +# Test the case of https://github.com/vyperlang/vyper/issues/4288 +def test_ssa_after_algebraic_optimization(): + code = """ +@internal +def _do_math(x: uint256) -> uint256: + value: uint256 = x + result: uint256 = 0 + + if (x >> 128 != 0): + x >>= 128 + if (x >> 64 != 0): + x >>= 64 + + if 1 < value: + result = 1 + + return result + +@external +def run() -> uint256: + return self._do_math(10) + """ + + vyper.compile_code(code, output_formats=["bytecode"]) diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index 310147baa7..bf3115b4dd 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -55,6 +55,13 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: StoreElimination(ac, fn).run_pass() SimplifyCFGPass(ac, fn).run_pass() AlgebraicOptimizationPass(ac, fn).run_pass() + # NOTE: MakeSSA is after algebraic optimization it currently produces + # smaller code by adding some redundant phi nodes. This is not a + # problem for us, but we need to be aware of it, and should be + # removed when the dft pass is fixed to produce the smallest code + # without making the code generation more expensive by running + # MakeSSA again. + MakeSSA(ac, fn).run_pass() BranchOptimizationPass(ac, fn).run_pass() RemoveUnusedVariablesPass(ac, fn).run_pass() diff --git a/vyper/venom/analysis/cfg.py b/vyper/venom/analysis/cfg.py index e4f130bc18..90b18b353c 100644 --- a/vyper/venom/analysis/cfg.py +++ b/vyper/venom/analysis/cfg.py @@ -32,7 +32,10 @@ def analyze(self) -> None: in_bb.add_cfg_out(bb) def invalidate(self): - from vyper.venom.analysis import DominatorTreeAnalysis, LivenessAnalysis + from vyper.venom.analysis import DFGAnalysis, DominatorTreeAnalysis, LivenessAnalysis self.analyses_cache.invalidate_analysis(DominatorTreeAnalysis) self.analyses_cache.invalidate_analysis(LivenessAnalysis) + + # be conservative - assume cfg invalidation invalidates dfg + self.analyses_cache.invalidate_analysis(DFGAnalysis) diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index 7966863081..19d373f81a 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -332,19 +332,18 @@ def _replace_constants(self, inst: IRInstruction): def _fix_phi_nodes(self): # fix basic blocks whose cfg in was changed # maybe this should really be done in _visit_phi - needs_sort = False - for bb in self.fn.get_basic_blocks(): cfg_in_labels = OrderedSet(in_bb.label for in_bb in bb.cfg_in) + needs_sort = False for inst in bb.instructions: if inst.opcode != "phi": break needs_sort |= self._fix_phi_inst(inst, cfg_in_labels) - # move phi instructions to the top of the block - if needs_sort: - bb.instructions.sort(key=lambda inst: inst.opcode != "phi") + # move phi instructions to the top of the block + if needs_sort: + bb.instructions.sort(key=lambda inst: inst.opcode != "phi") def _fix_phi_inst(self, inst: IRInstruction, cfg_in_labels: OrderedSet): operands = [op for label, op in inst.phi_operands if label in cfg_in_labels] diff --git a/vyper/venom/passes/store_elimination.py b/vyper/venom/passes/store_elimination.py index 0ecd324e26..559205adc8 100644 --- a/vyper/venom/passes/store_elimination.py +++ b/vyper/venom/passes/store_elimination.py @@ -27,16 +27,17 @@ def run_pass(self): self.analyses_cache.invalidate_analysis(LivenessAnalysis) self.analyses_cache.invalidate_analysis(DFGAnalysis) - def _process_store(self, dfg, inst, var, new_var): + def _process_store(self, dfg, inst, var: IRVariable, new_var: IRVariable): """ Process store instruction. If the variable is only used by a load instruction, forward the variable to the load instruction. """ - uses = dfg.get_uses(var) + if any([inst.opcode == "phi" for inst in dfg.get_uses(new_var)]): + return + uses = dfg.get_uses(var) if any([inst.opcode == "phi" for inst in uses]): return - for use_inst in uses: for i, operand in enumerate(use_inst.operands): if operand == var: From 658f0c4bd8ebd80a6e98fb01121ad5801bdd6661 Mon Sep 17 00:00:00 2001 From: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:26:10 +0000 Subject: [PATCH 08/70] refactor[venom]: refactor sccp pass to use dfg (#4329) use `DFGAnalysis` in `SCCP` instead of duplicating the logic to compute uses in the `SCCP` itself. also use `OrderedSet` for var uses, this ensures we don't add the same instruction multiple times (as in `add %2 %2`) to a var's use set, and also enables a cheaper `remove_use()` implementation. --- vyper/venom/analysis/dfg.py | 18 ++++++++++-------- vyper/venom/passes/sccp/sccp.py | 28 +++++++--------------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/vyper/venom/analysis/dfg.py b/vyper/venom/analysis/dfg.py index 328ed47c72..f49b2ac6ac 100644 --- a/vyper/venom/analysis/dfg.py +++ b/vyper/venom/analysis/dfg.py @@ -1,5 +1,6 @@ from typing import Optional +from vyper.utils import OrderedSet from vyper.venom.analysis.analysis import IRAnalysesCache, IRAnalysis from vyper.venom.analysis.liveness import LivenessAnalysis from vyper.venom.basicblock import IRInstruction, IRVariable @@ -7,7 +8,7 @@ class DFGAnalysis(IRAnalysis): - _dfg_inputs: dict[IRVariable, list[IRInstruction]] + _dfg_inputs: dict[IRVariable, OrderedSet[IRInstruction]] _dfg_outputs: dict[IRVariable, IRInstruction] def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): @@ -16,19 +17,19 @@ def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): self._dfg_outputs = dict() # return uses of a given variable - def get_uses(self, op: IRVariable) -> list[IRInstruction]: - return self._dfg_inputs.get(op, []) + def get_uses(self, op: IRVariable) -> OrderedSet[IRInstruction]: + return self._dfg_inputs.get(op, OrderedSet()) # the instruction which produces this variable. def get_producing_instruction(self, op: IRVariable) -> Optional[IRInstruction]: return self._dfg_outputs.get(op) def add_use(self, op: IRVariable, inst: IRInstruction): - uses = self._dfg_inputs.setdefault(op, []) - uses.append(inst) + uses = self._dfg_inputs.setdefault(op, OrderedSet()) + uses.add(inst) def remove_use(self, op: IRVariable, inst: IRInstruction): - uses = self._dfg_inputs.get(op, []) + uses: OrderedSet = self._dfg_inputs.get(op, OrderedSet()) uses.remove(inst) @property @@ -48,10 +49,11 @@ def analyze(self): res = inst.get_outputs() for op in operands: - inputs = self._dfg_inputs.setdefault(op, []) - inputs.append(inst) + inputs = self._dfg_inputs.setdefault(op, OrderedSet()) + inputs.add(inst) for op in res: # type: ignore + assert isinstance(op, IRVariable) self._dfg_outputs[op] = inst def as_graph(self) -> str: diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index 19d373f81a..d85e09c9b4 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -5,7 +5,7 @@ from vyper.exceptions import CompilerPanic, StaticAssertionException from vyper.utils import OrderedSet -from vyper.venom.analysis import CFGAnalysis, DominatorTreeAnalysis, IRAnalysesCache +from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, DominatorTreeAnalysis, IRAnalysesCache from vyper.venom.basicblock import ( IRBasicBlock, IRInstruction, @@ -51,7 +51,7 @@ class SCCP(IRPass): fn: IRFunction dom: DominatorTreeAnalysis - uses: dict[IRVariable, OrderedSet[IRInstruction]] + dfg: DFGAnalysis lattice: Lattice work_list: list[WorkListItem] cfg_in_exec: dict[IRBasicBlock, OrderedSet[IRBasicBlock]] @@ -67,7 +67,7 @@ def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): def run_pass(self): self.fn = self.function self.dom = self.analyses_cache.request_analysis(DominatorTreeAnalysis) - self._compute_uses() + self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) self._calculate_sccp(self.fn.entry) self._propagate_constants() @@ -75,6 +75,8 @@ def run_pass(self): self.analyses_cache.force_analysis(CFGAnalysis) self._fix_phi_nodes() + self.analyses_cache.invalidate_analysis(DFGAnalysis) + def _calculate_sccp(self, entry: IRBasicBlock): """ This method is the main entry point for the SCCP algorithm. It @@ -92,7 +94,7 @@ def _calculate_sccp(self, entry: IRBasicBlock): self.work_list.append(FlowWorkItem(dummy, entry)) # Initialize the lattice with TOP values for all variables - for v in self.uses.keys(): + for v in self.dfg._dfg_outputs: self.lattice[v] = LatticeEnum.TOP # Iterate over the work list until it is empty @@ -258,25 +260,9 @@ def _eval(self, inst) -> LatticeItem: return ret # type: ignore def _add_ssa_work_items(self, inst: IRInstruction): - for target_inst in self._get_uses(inst.output): # type: ignore + for target_inst in self.dfg.get_uses(inst.output): # type: ignore self.work_list.append(SSAWorkListItem(target_inst)) - def _compute_uses(self): - """ - This method computes the uses for each variable in the IR. - It iterates over the dominator tree and collects all the - instructions that use each variable. - """ - self.uses = {} - for bb in self.dom.dfs_walk: - for var, insts in bb.get_uses().items(): - self._get_uses(var).update(insts) - - def _get_uses(self, var: IRVariable): - if var not in self.uses: - self.uses[var] = OrderedSet() - return self.uses[var] - def _propagate_constants(self): """ This method iterates over the IR and replaces constant values From 063f1900a1b2b32bf8d0f091027dd0fb16e99de8 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Sat, 26 Oct 2024 01:57:30 +0200 Subject: [PATCH 09/70] docs[venom]: expand venom docs (#4314) expand venom docs. document venom instructions and structure of a venom program. --------- Co-authored-by: Charles Cooper --- vyper/venom/README.md | 313 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) diff --git a/vyper/venom/README.md b/vyper/venom/README.md index 5d98b22dd6..4e4f5ca3d1 100644 --- a/vyper/venom/README.md +++ b/vyper/venom/README.md @@ -160,3 +160,316 @@ A number of passes that are planned to be implemented, or are implemented for im ### Function inlining ### Load-store elimination + +--- + +## Structure of a venom program + +### IRContext +An `IRContext` consists of multiple `IRFunctions`, with one designated as the main entry point of the program. +Additionally, the `IRContext` maintains its own representation of the data segment. + +### IRFunction +An `IRFunction` is composed of a name and multiple `IRBasicBlocks`, with one marked as the entry point to the function. + +### IRBasicBlock +An `IRBasicBlock` contains a label and a sequence of `IRInstructions`. +Each `IRBasicBlock` has a single entry point and exit point. +The exit point must be one of the following terminator instructions: +- `jmp` +- `djmp` +- `jnz` +- `ret` +- `return` +- `stop` +- `exit` + +Normalized basic blocks cannot have multiple predecessors and successors. It has either one (or zero) predecessors and potentially multiple successors or vice versa. + +### IRInstruction +An `IRInstruction` consists of an opcode, a list of operands, and an optional return value. +An operand can be a label, a variable, or a literal. + +By convention, variables have a `%-` prefix, e.g. `%1` is a valid variable. However, the prefix is not required. + +## Instructions +To enable Venom IR in Vyper, use the `--experimental-codegen` flag. To view the Venom IR output, use `-f bb_runtime` for the runtime code, or `-f bb` to see the deploy code. To get a dot file (for use e.g. with `xdot -`), use `-f cfg` or `-f cfg_runtime`. + +Assembly can be inspected with `-f asm`, whereas an opcode view of the final bytecode can be seen with `-f opcodes` or `-f opcodes_runtime`, respectively. + +### Special instructions + +- `invoke` + - ``` + invoke offset, label + ``` + - Causes control flow to jump to a function denoted by the `label`. + - Return values are passed in the return buffer at the `offset` address. + - Used for internal functions. + - Effectively translates to `JUMP`, and marks the call site as a valid return destination (for callee to jump back to) by `JUMPDEST`. +- `alloca` + - ``` + out = alloca size, offset + ``` + - Allocates memory of a given `size` at a given `offset` in memory. + - The output is the offset value itself. + - Because the SSA form does not allow changing values of registers, handling mutable variables can be tricky. The `alloca` instruction is meant to simplify that. + +- `palloca` + - ``` + out = palloca size, offset + ``` + - Like the `alloca` instruction but only used for parameters of internal functions which are passed by memory. +- `iload` + - ``` + out = iload offset + ``` + - Loads value at an immutable section of memory denoted by `offset` into `out` variable. + - The operand can be either a literal, which is a statically computed offset, or a variable. + - Essentially translates to `MLOAD` on an immutable section of memory. So, for example + ``` + %op = 12 + %out = iload %op + ``` + could compile into `PUSH1 12 _mem_deploy_end ADD MLOAD`. + - When `offset` is a literal the location is computed statically during compilation from assembly to bytecode. +- `istore` + - ``` + istore offset value + ``` + - Represents a store into immutable section of memory. + - Like in `iload`, the offset operand can be a literal. + - Essentially translates to `MSTORE` on an immutable section of memory. For example, + ``` + %op = 12 + istore 24 %op + ``` + could compile to + `PUSH1 12 PUSH1 24 _mem_deploy_end ADD MSTORE`. +- `phi` + - ``` + out = phi %var_a, label_a, %var_b, label_b + ``` + - Because in SSA form each variable is assigned just once, it is tricky to handle that variables may be assigned to something different based on which program path was taken. + - Therefore, we use `phi` instructions. They are are magic instructions, used in basic blocks where the control flow path merges. + - In this example, essentially the `out` variable is set to `%var_a` if the program entered the current block from `label_a` or to `%var_b` when it went through `label_b`. +- `offset` + - ``` + ret = offset label, op + ``` + - Statically compute offset before compiling into bytecode. Useful for `mstore`, `mload` and such. + - Basically `label` + `op`. + - The `asm` output could show something like `_OFST _sym_ label`. +- `param` + - ``` + out = param + ``` + - The `param` instruction is used to represent function arguments passed by the stack. + - We assume the argument is on the stack and the `param` instruction is used to ensure we represent the argument by the `out` variable. +- `store` + - ``` + out = op + ``` + - Store variable value or literal into `out` variable. +- `dbname` + - ``` + dbname label + ``` + - Mark memory with a `label` in the data segment so it can be referenced. +- `db` + - ``` + db data + ``` + - Store `data` into data segment. +- `dloadbytes` + - Alias for `codecopy` for legacy reasons. May be removed in future versions. + - Translates to `CODECOPY`. +- `ret` + - ``` + ret op + ``` + - Represents return from an internal call. + - Jumps to a location given by `op`. + - If `op` is a label it can effectively translate into `op JUMP`. +- `exit` + - ``` + exit + ``` + - Similar to `stop`, but used for constructor exit. The assembler is expected to jump to a special initcode sequence which returns the runtime code. + - Might translate to something like `_sym__ctor_exit JUMP`. +- `sha3_64` + - ``` + out = sha3_64 x y + ``` + - Shortcut to access the `SHA3` EVM opcode where `out` is the result. + - Essentially translates to + ``` + PUSH y PUSH FREE_VAR_SPACE MSTORE + PUSH x PUSH FREE_VAR_SPACE2 MSTORE + PUSH 64 PUSH FREE_VAR_SPACE SHA3 + ``` + where `FREE_VAR_SPACE` and `FREE_VAR_SPACE2` are locations reserved by the compiler, set to 0 and 32 respectively. + +- `assert` + - ``` + assert op + ``` + - Assert that `op` is zero. If it is not, revert. + - Calls that terminate this way receive a gas refund. + - For example + ``` + %op = 13 + assert %op + ``` + could compile to + `PUSH1 13 ISZERO _sym___revert JUMPI`. +- `assert_unreachable` + - ``` + assert_unreachable op + ``` + - Check that `op` is zero. If it is not, terminate with `0xFE` ("INVALID" opcode). + - Calls that end this way do not receive a gas refund. + - Could translate to `op reachable JUMPI INVALID reachable JUMPDEST`. + - For example + ``` + %op = 13 + assert_unreachable %op + ``` + could compile to + ``` + PUSH1 13 _sym_reachable1 JUMPI + INVALID + _sym_reachable1 JUMPDEST + ``` +- `log` + - ``` + log offset, size, [topic] * topic_count , topic_count + ``` + - Corresponds to the `LOGX` instruction in EVM. + - Depending on the `topic_count` value (which can be only from 0 to 4) translates to `LOG0` ... `LOG4`. + - The rest of the operands correspond to the `LOGX` instructions. + - For example + ``` + log %53, 32, 64, %56, 2 + ``` + could translate to: + ``` + %56, 64, 32, %53 LOG2 + ``` +- `nop` + - ``` + nop + ``` + - No operation, does nothing. +- `offset` + - ``` + %2 = offset %1 label1 + - Similar to `add`, but takes a label as the second argument. If the first argument is a literal, the addition will get optimized at assembly time. + +### Jump instructions + +- `jmp` + - ``` + jmp label + ``` + - Unconditional jump to code denoted by given `label`. + - Translates to `label JUMP`. +- `jnz` + - ``` + jnz label1, label2, op + ``` + - A conditional jump depending on the value of `op`. + - Jumps to `label2` when `op` is not zero, otherwise jumps to `label1`. + - For example + ``` + %op = 15 + jnz label1, label2, %op + ``` + could translate to: `PUSH1 15 label2 JUMPI label1 JUMP`. +- `djmp` + - ``` + djmp %var, label1, label2, label3, ... + ``` + - Dynamic jump to an address specified by the variable operand, constrained to the provided labels. + - Accepts a variable number of labels. + - The target is not a fixed label but rather a value stored in a variable, making the jump dynamic. + - The jump target can be any of the provided labels. + - Translates to `JUMP`. + +### EVM instructions + +The following instructions map one-to-one with [EVM instructions](https://www.evm.codes/). +Operands correspond to stack inputs in the same order. Stack outputs are the instruction's output. +Instructions have the same effects. +- `return` +- `revert` +- `coinbase` +- `calldatasize` +- `calldatacopy` +- `mcopy` +- `calldataload` +- `gas` +- `gasprice` +- `gaslimit` +- `chainid` +- `address` +- `origin` +- `number` +- `extcodesize` +- `extcodehash` +- `extcodecopy` +- `returndatasize` +- `returndatacopy` +- `callvalue` +- `selfbalance` +- `sload` +- `sstore` +- `mload` +- `mstore` +- `tload` +- `tstore` +- `timestamp` +- `caller` +- `blockhash` +- `selfdestruct` +- `signextend` +- `stop` +- `shr` +- `shl` +- `sar` +- `and` +- `xor` +- `or` +- `add` +- `sub` +- `mul` +- `div` +- `smul` +- `sdiv` +- `mod` +- `smod` +- `exp` +- `addmod` +- `mulmod` +- `eq` +- `iszero` +- `not` +- `lt` +- `gt` +- `slt` +- `sgt` +- `create` +- `create2` +- `msize` +- `balance` +- `call` +- `staticcall` +- `delegatecall` +- `codesize` +- `basefee` +- `blobhash` +- `blobbasefee` +- `prevrandao` +- `difficulty` +- `invalid` +- `sha3` From 3fba8c735e1ab124e928609296f70606261e7113 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 29 Oct 2024 09:32:31 -0400 Subject: [PATCH 10/70] refactor[venom]: update translator for `deploy` instruction (#4318) this commit updates `ir_node_to_venom.py` for the `deploy` instruction to properly emit the `exit` instruction. it also removes places in the translator where the `exit` instruction was improperly emitted. misc: - update style to continue early in the loop rather than nest. --- vyper/venom/function.py | 22 ++++++++++++---------- vyper/venom/ir_node_to_venom.py | 6 ++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/vyper/venom/function.py b/vyper/venom/function.py index fb0dabc99a..2f2d477460 100644 --- a/vyper/venom/function.py +++ b/vyper/venom/function.py @@ -195,17 +195,19 @@ def chain_basic_blocks(self) -> None: """ bbs = list(self.get_basic_blocks()) for i, bb in enumerate(bbs): - if not bb.is_terminated: - if i < len(bbs) - 1: - # TODO: revisit this. When contructor calls internal functions they - # are linked to the last ctor block. Should separate them before this - # so we don't have to handle this here - if bbs[i + 1].label.value.startswith("internal"): - bb.append_instruction("stop") - else: - bb.append_instruction("jmp", bbs[i + 1].label) + if bb.is_terminated: + continue + + if i < len(bbs) - 1: + # TODO: revisit this. When contructor calls internal functions + # they are linked to the last ctor block. Should separate them + # before this so we don't have to handle this here + if bbs[i + 1].label.value.startswith("internal"): + bb.append_instruction("stop") else: - bb.append_instruction("exit") + bb.append_instruction("jmp", bbs[i + 1].label) + else: + bb.append_instruction("stop") def copy(self): new = IRFunction(self.name) diff --git a/vyper/venom/ir_node_to_venom.py b/vyper/venom/ir_node_to_venom.py index e30f27f480..02a9f4d1f7 100644 --- a/vyper/venom/ir_node_to_venom.py +++ b/vyper/venom/ir_node_to_venom.py @@ -255,6 +255,7 @@ def _convert_ir_bb(fn, ir, symbols): elif ir.value == "deploy": ctx.ctor_mem_size = ir.args[0].value ctx.immutables_len = ir.args[2].value + fn.get_basic_block().append_instruction("exit") return None elif ir.value == "seq": if len(ir.args) == 0: @@ -398,10 +399,7 @@ def _convert_ir_bb(fn, ir, symbols): bb = IRBasicBlock(label, fn) fn.append_basic_block(bb) code = ir.args[2] - if code.value == "pass": - bb.append_instruction("exit") - else: - _convert_ir_bb(fn, code, symbols) + _convert_ir_bb(fn, code, symbols) elif ir.value == "exit_to": args = _convert_ir_bb_list(fn, ir.args[1:], symbols) var_list = args From fcddb70b6a796757709fa68a36f86ce729d18f77 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 29 Oct 2024 09:33:13 -0400 Subject: [PATCH 11/70] feat[venom]: reduce legacy opts when venom is enabled (#4336) reduce legacy IR optimization when venom is enabled. the use of `IRnode._optimized` was there to decide if it was safe to inline an IRnode expression or it required a `with` statement; with venom, we don't need to inline the expressions, since the venom optimizer is more powerful. this leads to a 50% improvement in AST -> IRnode generation, which is a ~15% performance improvement in overall end-to-end compile time. --- vyper/codegen/ir_node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vyper/codegen/ir_node.py b/vyper/codegen/ir_node.py index 6f9eb0359b..ff721fafcb 100644 --- a/vyper/codegen/ir_node.py +++ b/vyper/codegen/ir_node.py @@ -6,7 +6,7 @@ from typing import Any, List, Optional, Union import vyper.ast as vy_ast -from vyper.compiler.settings import VYPER_COLOR_OUTPUT +from vyper.compiler.settings import VYPER_COLOR_OUTPUT, get_global_settings from vyper.evm.address_space import AddrSpace from vyper.evm.opcodes import get_ir_opcodes from vyper.exceptions import CodegenPanic, CompilerPanic @@ -426,6 +426,10 @@ def is_pointer(self) -> bool: @property # probably could be cached_property but be paranoid def _optimized(self): + if get_global_settings().experimental_codegen: + # in venom pipeline, we don't need to inline constants. + return self + # TODO figure out how to fix this circular import from vyper.ir.optimizer import optimize From 471a556f617e550a7882b138a0fdd75dabf34279 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:14:07 +0100 Subject: [PATCH 12/70] refactor[ux]: add `venom` as `experimental-codegen` alias (#4337) Make "venom" an alternative alias to the "experimental_codegen" flag. Add the alias to cli, pre-parser, and json parser. --- tests/unit/ast/test_pre_parser.py | 28 +++++++++++++++ .../unit/cli/vyper_json/test_compile_json.py | 36 +++++++++++++++++++ vyper/ast/pre_parser.py | 4 +-- vyper/cli/vyper_compile.py | 1 + vyper/cli/vyper_json.py | 6 ++++ vyper/compiler/settings.py | 2 +- 6 files changed, 74 insertions(+), 3 deletions(-) diff --git a/tests/unit/ast/test_pre_parser.py b/tests/unit/ast/test_pre_parser.py index 4190725f7e..5d3f30481c 100644 --- a/tests/unit/ast/test_pre_parser.py +++ b/tests/unit/ast/test_pre_parser.py @@ -191,6 +191,25 @@ def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_ve assert compiler_data.settings == compiler_data_settings +pragma_venom = [ + """ + #pragma venom + """, + """ + #pragma experimental-codegen + """, +] + + +@pytest.mark.parametrize("code", pragma_venom) +def test_parse_venom_pragma(code): + pre_parse_result = pre_parse(code) + assert pre_parse_result.settings.experimental_codegen is True + + compiler_data = CompilerData(code) + assert compiler_data.settings.experimental_codegen is True + + invalid_pragmas = [ # evm-versionnn """ @@ -218,6 +237,15 @@ def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_ve # pragma evm-version cancun # pragma evm-version shanghai """, + # duplicate setting of venom + """ + #pragma venom + #pragma experimental-codegen + """, + """ + #pragma venom + #pragma venom + """, ] diff --git a/tests/unit/cli/vyper_json/test_compile_json.py b/tests/unit/cli/vyper_json/test_compile_json.py index 5da98cf20f..7802ee7955 100644 --- a/tests/unit/cli/vyper_json/test_compile_json.py +++ b/tests/unit/cli/vyper_json/test_compile_json.py @@ -9,6 +9,7 @@ compile_json, exc_handler_to_dict, get_inputs, + get_settings, ) from vyper.compiler import OUTPUT_FORMATS, compile_code, compile_from_file_input from vyper.compiler.input_bundle import JSONInputBundle @@ -319,3 +320,38 @@ def test_compile_json_with_abi_top(make_input_bundle): """ input_bundle = make_input_bundle({"stream.json": stream, "code.vy": code}) vyper.compiler.compile_code(code, input_bundle=input_bundle) + + +def test_compile_json_with_experimental_codegen(): + code = { + "language": "Vyper", + "sources": {"foo.vy": {"content": "@external\ndef foo() -> bool:\n return True"}}, + "settings": { + "evmVersion": "cancun", + "optimize": "gas", + "venom": True, + "search_paths": [], + "outputSelection": {"*": ["ast"]}, + }, + } + + settings = get_settings(code) + assert settings.experimental_codegen is True + + +def test_compile_json_with_both_venom_aliases(): + code = { + "language": "Vyper", + "sources": {"foo.vy": {"content": ""}}, + "settings": { + "evmVersion": "cancun", + "optimize": "gas", + "experimentalCodegen": False, + "venom": False, + "search_paths": [], + "outputSelection": {"*": ["ast"]}, + }, + } + with pytest.raises(JSONError) as e: + get_settings(code) + assert e.value.args[0] == "both experimentalCodegen and venom cannot be set" diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index 07ba1d2d0d..5d2abcf645 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -265,10 +265,10 @@ def pre_parse(code: str) -> PreParseResult: if evm_version not in EVM_VERSIONS: raise StructureException(f"Invalid evm version: `{evm_version}`", start) settings.evm_version = evm_version - elif pragma.startswith("experimental-codegen"): + elif pragma.startswith("experimental-codegen") or pragma.startswith("venom"): if settings.experimental_codegen is not None: raise StructureException( - "pragma experimental-codegen specified twice!", start + "pragma experimental-codegen/venom specified twice!", start ) settings.experimental_codegen = True elif pragma.startswith("enable-decimals"): diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index 5999aed178..fde35f781e 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -176,6 +176,7 @@ def _parse_args(argv): parser.add_argument("-o", help="Set the output path", dest="output_path") parser.add_argument( "--experimental-codegen", + "--venom", help="The compiler use the new IR codegen. This is an experimental feature.", action="store_true", dest="experimental_codegen", diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index 7d0e6064af..9fcdf27baf 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -253,7 +253,13 @@ def get_settings(input_dict: dict) -> Settings: evm_version = get_evm_version(input_dict) optimize = input_dict["settings"].get("optimize") + experimental_codegen = input_dict["settings"].get("experimentalCodegen") + if experimental_codegen is None: + experimental_codegen = input_dict["settings"].get("venom") + elif input_dict["settings"].get("venom") is not None: + raise JSONError("both experimentalCodegen and venom cannot be set") + if isinstance(optimize, bool): # bool optimization level for backwards compatibility warnings.warn( diff --git a/vyper/compiler/settings.py b/vyper/compiler/settings.py index 7c20e03906..a8e28c1ed1 100644 --- a/vyper/compiler/settings.py +++ b/vyper/compiler/settings.py @@ -77,7 +77,7 @@ def as_cli(self): if self.optimize is not None: ret.append(" --optimize " + str(self.optimize)) if self.experimental_codegen is True: - ret.append(" --experimental-codegen") + ret.append(" --venom") if self.evm_version is not None: ret.append(" --evm-version " + self.evm_version) if self.debug is True: From 53f623b83d718fad78453142462736d99cdc6515 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 5 Nov 2024 09:17:02 -0500 Subject: [PATCH 13/70] fix[ci]: fix README encoding in `setup.py` (#4348) the call to `setup()` would fail on windows when there are unicode characters in `README.md`, because files are apparently opened with encoding `cp1252` by default on windows. this commit ensures the file is opened with `utf-8` encoding. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6e48129cba..5d6bd1db3a 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ extras_require["dev"] = extras_require["dev"] + extras_require["test"] + extras_require["lint"] -with open("README.md", "r") as f: +with open("README.md", "r", encoding="utf-8") as f: long_description = f.read() From 0abcf452b29f5348cb14233fd9a8444224392184 Mon Sep 17 00:00:00 2001 From: Rafael Abuawad Date: Tue, 5 Nov 2024 10:34:52 -0400 Subject: [PATCH 14/70] feat[docs]: add Telegram badge to README.md (#4342) --------- Co-authored-by: Charles Cooper --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bcaa50b570..84c2948ceb 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Build Status](https://github.com/vyperlang/vyper/workflows/Test/badge.svg)](https://github.com/vyperlang/vyper/actions/workflows/test.yml) [![Documentation Status](https://readthedocs.org/projects/vyper/badge/?version=latest)](http://docs.vyperlang.org/en/latest/?badge=latest "ReadTheDocs") [![Discord](https://img.shields.io/discord/969926564286459934.svg?label=%23vyper)](https://discord.gg/6tw7PTM7C2) +[![Telegram](https://img.shields.io/badge/Vyperholics🐍-Telegram-blue)](https://t.me/vyperlang) [![PyPI](https://badge.fury.io/py/vyper.svg)](https://pypi.org/project/vyper "PyPI") [![Docker](https://img.shields.io/docker/cloud/build/vyperlang/vyper)](https://hub.docker.com/r/vyperlang/vyper "DockerHub") From fee16e657abf9ab944c10ead75d5ce112fbf3a67 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 12 Nov 2024 10:31:56 +0400 Subject: [PATCH 15/70] feat[venom]: make cfg scheduler "stack aware" (#4356) this is a step towards making the cfg traversal order more "stack aware". previously we would blindly try to remove `iszero` instructions before a `jnz`. now, the heuristic is based on which basic block has a smaller sized dependency on this basic block. only if the two cfg_out blocks have the same sized dependency, then we try to remove the iszero. this creates a slight improvement to codesize due to fewer stack operations at `jnz` boundaries. --- .../compiler/venom/test_branch_optimizer.py | 6 ++-- vyper/venom/passes/branch_optimization.py | 30 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/tests/unit/compiler/venom/test_branch_optimizer.py b/tests/unit/compiler/venom/test_branch_optimizer.py index 82dff4777d..4c46127e1d 100644 --- a/tests/unit/compiler/venom/test_branch_optimizer.py +++ b/tests/unit/compiler/venom/test_branch_optimizer.py @@ -22,9 +22,9 @@ def test_simple_jump_case(): jnz_input = bb.append_instruction("iszero", op3) bb.append_instruction("jnz", jnz_input, br1.label, br2.label) - br1.append_instruction("add", op3, 10) + br1.append_instruction("add", op3, p1) br1.append_instruction("stop") - br2.append_instruction("add", op3, p1) + br2.append_instruction("add", op3, 10) br2.append_instruction("stop") term_inst = bb.instructions[-1] @@ -47,6 +47,6 @@ def test_simple_jump_case(): # Test that the dfg is updated correctly dfg = ac.request_analysis(DFGAnalysis) - assert dfg is old_dfg, "DFG should not be invalidated by BranchOptimizationPass" + assert dfg is not old_dfg, "DFG should be invalidated by BranchOptimizationPass" assert term_inst in dfg.get_uses(op3), "jnz not using the new condition" assert term_inst not in dfg.get_uses(jnz_input), "jnz still using the old condition" diff --git a/vyper/venom/passes/branch_optimization.py b/vyper/venom/passes/branch_optimization.py index d5b0ed9809..920dc5e431 100644 --- a/vyper/venom/passes/branch_optimization.py +++ b/vyper/venom/passes/branch_optimization.py @@ -1,4 +1,5 @@ -from vyper.venom.analysis import DFGAnalysis +from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, LivenessAnalysis +from vyper.venom.basicblock import IRInstruction from vyper.venom.passes.base_pass import IRPass @@ -14,17 +15,30 @@ def _optimize_branches(self) -> None: if term_inst.opcode != "jnz": continue - prev_inst = self.dfg.get_producing_instruction(term_inst.operands[0]) - if prev_inst.opcode == "iszero": + fst, snd = bb.cfg_out + + fst_liveness = fst.instructions[0].liveness + snd_liveness = snd.instructions[0].liveness + + cost_a, cost_b = len(fst_liveness), len(snd_liveness) + + cond = term_inst.operands[0] + prev_inst = self.dfg.get_producing_instruction(cond) + if cost_a >= cost_b and prev_inst.opcode == "iszero": new_cond = prev_inst.operands[0] term_inst.operands = [new_cond, term_inst.operands[2], term_inst.operands[1]] - - # Since the DFG update is simple we do in place to avoid invalidating the DFG - # and having to recompute it (which is expensive(er)) - self.dfg.remove_use(prev_inst.output, term_inst) - self.dfg.add_use(new_cond, term_inst) + elif cost_a > cost_b: + new_cond = fn.get_next_variable() + inst = IRInstruction("iszero", [term_inst.operands[0]], output=new_cond) + bb.insert_instruction(inst, index=-1) + term_inst.operands = [new_cond, term_inst.operands[2], term_inst.operands[1]] def run_pass(self): + self.liveness = self.analyses_cache.request_analysis(LivenessAnalysis) + self.cfg = self.analyses_cache.request_analysis(CFGAnalysis) self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) self._optimize_branches() + + self.analyses_cache.invalidate_analysis(LivenessAnalysis) + self.analyses_cache.invalidate_analysis(CFGAnalysis) From 48cb39bed2c3fd5403f64e4c89854f6c645231c1 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 12 Nov 2024 11:27:39 +0400 Subject: [PATCH 16/70] feat[venom]: improve liveness computation (#4330) traversing in reverse topsort order improves stack scheduling slightly this commit also adds a topsort method to CFGAnalysis, and speeds it up by only checking the terminator instruction instead of iterating over all the instructions in every basic block. additional refactors: - move dfs order calculation from domtree to cfg analysis. - remove unnecessary calculation of domtree in sccp - remove redundant IRFunction.compute_reachability - change cfg_out order - refactor shared phi fixup code - remove useless `__eq__()` and `__hash__()` for IRBasicBlock --- .../codegen/features/test_constructor.py | 4 -- .../codegen/types/test_dynamic_array.py | 2 - .../compiler/venom/test_branch_optimizer.py | 3 +- tests/unit/compiler/venom/test_sccp.py | 9 +-- vyper/venom/analysis/cfg.py | 47 ++++++++++----- vyper/venom/analysis/dominators.py | 29 +++------- vyper/venom/analysis/liveness.py | 14 ++--- vyper/venom/basicblock.py | 37 ++++++++---- vyper/venom/function.py | 58 +++++-------------- vyper/venom/passes/sccp/sccp.py | 35 +---------- vyper/venom/passes/simplify_cfg.py | 4 +- vyper/venom/venom_to_assembly.py | 10 +++- 12 files changed, 110 insertions(+), 142 deletions(-) diff --git a/tests/functional/codegen/features/test_constructor.py b/tests/functional/codegen/features/test_constructor.py index 3b86fe3460..6cc7007bb2 100644 --- a/tests/functional/codegen/features/test_constructor.py +++ b/tests/functional/codegen/features/test_constructor.py @@ -1,7 +1,4 @@ -import pytest - from tests.evm_backends.base_env import _compile -from vyper.exceptions import StackTooDeep from vyper.utils import method_id @@ -169,7 +166,6 @@ def get_foo() -> uint256: assert c.get_foo() == 39 -@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression") def test_nested_dynamic_array_constructor_arg_2(env, get_contract): code = """ foo: int128 diff --git a/tests/functional/codegen/types/test_dynamic_array.py b/tests/functional/codegen/types/test_dynamic_array.py index 2a0f4e77e5..2f647ac38c 100644 --- a/tests/functional/codegen/types/test_dynamic_array.py +++ b/tests/functional/codegen/types/test_dynamic_array.py @@ -11,7 +11,6 @@ CompilerPanic, ImmutableViolation, OverflowException, - StackTooDeep, StateAccessViolation, TypeMismatch, ) @@ -737,7 +736,6 @@ def test_array_decimal_return3() -> DynArray[DynArray[decimal, 2], 2]: ] -@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression") def test_mult_list(get_contract): code = """ nest3: DynArray[DynArray[DynArray[uint256, 2], 2], 2] diff --git a/tests/unit/compiler/venom/test_branch_optimizer.py b/tests/unit/compiler/venom/test_branch_optimizer.py index 4c46127e1d..a96ed0709c 100644 --- a/tests/unit/compiler/venom/test_branch_optimizer.py +++ b/tests/unit/compiler/venom/test_branch_optimizer.py @@ -16,6 +16,7 @@ def test_simple_jump_case(): fn.append_basic_block(br2) p1 = bb.append_instruction("param") + p2 = bb.append_instruction("param") op1 = bb.append_instruction("store", p1) op2 = bb.append_instruction("store", 64) op3 = bb.append_instruction("add", op1, op2) @@ -24,7 +25,7 @@ def test_simple_jump_case(): br1.append_instruction("add", op3, p1) br1.append_instruction("stop") - br2.append_instruction("add", op3, 10) + br2.append_instruction("add", op3, p2) br2.append_instruction("stop") term_inst = bb.instructions[-1] diff --git a/tests/unit/compiler/venom/test_sccp.py b/tests/unit/compiler/venom/test_sccp.py index 375dfd5dac..0d46b61acd 100644 --- a/tests/unit/compiler/venom/test_sccp.py +++ b/tests/unit/compiler/venom/test_sccp.py @@ -167,8 +167,8 @@ def test_cont_phi_case(): assert sccp.lattice[IRVariable("%2")].value == 32 assert sccp.lattice[IRVariable("%3")].value == 64 assert sccp.lattice[IRVariable("%4")].value == 96 - assert sccp.lattice[IRVariable("%5", version=1)].value == 106 - assert sccp.lattice[IRVariable("%5", version=2)] == LatticeEnum.BOTTOM + assert sccp.lattice[IRVariable("%5", version=2)].value == 106 + assert sccp.lattice[IRVariable("%5", version=1)] == LatticeEnum.BOTTOM assert sccp.lattice[IRVariable("%5")].value == 2 @@ -207,8 +207,9 @@ def test_cont_phi_const_case(): assert sccp.lattice[IRVariable("%2")].value == 32 assert sccp.lattice[IRVariable("%3")].value == 64 assert sccp.lattice[IRVariable("%4")].value == 96 - assert sccp.lattice[IRVariable("%5", version=1)].value == 106 - assert sccp.lattice[IRVariable("%5", version=2)].value == 97 + # dependent on cfg traversal order + assert sccp.lattice[IRVariable("%5", version=2)].value == 106 + assert sccp.lattice[IRVariable("%5", version=1)].value == 97 assert sccp.lattice[IRVariable("%5")].value == 2 diff --git a/vyper/venom/analysis/cfg.py b/vyper/venom/analysis/cfg.py index 90b18b353c..700fd73f26 100644 --- a/vyper/venom/analysis/cfg.py +++ b/vyper/venom/analysis/cfg.py @@ -1,6 +1,8 @@ +from typing import Iterator + from vyper.utils import OrderedSet from vyper.venom.analysis import IRAnalysis -from vyper.venom.basicblock import CFG_ALTERING_INSTRUCTIONS +from vyper.venom.basicblock import CFG_ALTERING_INSTRUCTIONS, IRBasicBlock class CFGAnalysis(IRAnalysis): @@ -8,28 +10,45 @@ class CFGAnalysis(IRAnalysis): Compute control flow graph information for each basic block in the function. """ + _dfs: OrderedSet[IRBasicBlock] + def analyze(self) -> None: fn = self.function + self._dfs = OrderedSet() + for bb in fn.get_basic_blocks(): bb.cfg_in = OrderedSet() bb.cfg_out = OrderedSet() bb.out_vars = OrderedSet() + bb.is_reachable = False for bb in fn.get_basic_blocks(): - assert len(bb.instructions) > 0, "Basic block should not be empty" - last_inst = bb.instructions[-1] - assert last_inst.is_bb_terminator, f"Last instruction should be a terminator {bb}" + assert bb.is_terminated - for inst in bb.instructions: - if inst.opcode in CFG_ALTERING_INSTRUCTIONS: - ops = inst.get_label_operands() - for op in ops: - fn.get_basic_block(op.value).add_cfg_in(bb) + term = bb.instructions[-1] + if term.opcode in CFG_ALTERING_INSTRUCTIONS: + ops = term.get_label_operands() + # order of cfg_out matters to performance! + for op in reversed(list(ops)): + next_bb = fn.get_basic_block(op.value) + bb.add_cfg_out(next_bb) + next_bb.add_cfg_in(bb) - # Fill in the "out" set for each basic block - for bb in fn.get_basic_blocks(): - for in_bb in bb.cfg_in: - in_bb.add_cfg_out(bb) + self._compute_dfs_r(self.function.entry) + + def _compute_dfs_r(self, bb): + if bb.is_reachable: + return + bb.is_reachable = True + + for out_bb in bb.cfg_out: + self._compute_dfs_r(out_bb) + + self._dfs.add(bb) + + @property + def dfs_walk(self) -> Iterator[IRBasicBlock]: + return iter(self._dfs) def invalidate(self): from vyper.venom.analysis import DFGAnalysis, DominatorTreeAnalysis, LivenessAnalysis @@ -37,5 +56,7 @@ def invalidate(self): self.analyses_cache.invalidate_analysis(DominatorTreeAnalysis) self.analyses_cache.invalidate_analysis(LivenessAnalysis) + self._dfs = None + # be conservative - assume cfg invalidation invalidates dfg self.analyses_cache.invalidate_analysis(DFGAnalysis) diff --git a/vyper/venom/analysis/dominators.py b/vyper/venom/analysis/dominators.py index e360df36b9..b60f9bdab9 100644 --- a/vyper/venom/analysis/dominators.py +++ b/vyper/venom/analysis/dominators.py @@ -1,3 +1,5 @@ +from functools import cached_property + from vyper.exceptions import CompilerPanic from vyper.utils import OrderedSet from vyper.venom.analysis import CFGAnalysis, IRAnalysis @@ -14,8 +16,6 @@ class DominatorTreeAnalysis(IRAnalysis): fn: IRFunction entry_block: IRBasicBlock - dfs_order: dict[IRBasicBlock, int] - dfs_walk: list[IRBasicBlock] dominators: dict[IRBasicBlock, OrderedSet[IRBasicBlock]] immediate_dominators: dict[IRBasicBlock, IRBasicBlock] dominated: dict[IRBasicBlock, OrderedSet[IRBasicBlock]] @@ -27,16 +27,13 @@ def analyze(self): """ self.fn = self.function self.entry_block = self.fn.entry - self.dfs_order = {} - self.dfs_walk = [] self.dominators = {} self.immediate_dominators = {} self.dominated = {} self.dominator_frontiers = {} - self.analyses_cache.request_analysis(CFGAnalysis) + self.cfg = self.analyses_cache.request_analysis(CFGAnalysis) - self._compute_dfs(self.entry_block, OrderedSet()) self._compute_dominators() self._compute_idoms() self._compute_df() @@ -131,21 +128,13 @@ def _intersect(self, bb1, bb2): bb2 = self.immediate_dominators[bb2] return bb1 - def _compute_dfs(self, entry: IRBasicBlock, visited): - """ - Depth-first search to compute the DFS order of the basic blocks. This - is used to compute the dominator tree. The sequence of basic blocks in - the DFS order is stored in `self.dfs_walk`. The DFS order of each basic - block is stored in `self.dfs_order`. - """ - visited.add(entry) - - for bb in entry.cfg_out: - if bb not in visited: - self._compute_dfs(bb, visited) + @cached_property + def dfs_walk(self) -> list[IRBasicBlock]: + return list(self.cfg.dfs_walk) - self.dfs_walk.append(entry) - self.dfs_order[entry] = len(self.dfs_walk) + @cached_property + def dfs_order(self) -> dict[IRBasicBlock, int]: + return {bb: idx for idx, bb in enumerate(self.dfs_walk)} def as_graph(self) -> str: """ diff --git a/vyper/venom/analysis/liveness.py b/vyper/venom/analysis/liveness.py index b5d65961b7..2ee28b9530 100644 --- a/vyper/venom/analysis/liveness.py +++ b/vyper/venom/analysis/liveness.py @@ -12,21 +12,21 @@ class LivenessAnalysis(IRAnalysis): """ def analyze(self): - self.analyses_cache.request_analysis(CFGAnalysis) + cfg = self.analyses_cache.request_analysis(CFGAnalysis) self._reset_liveness() - self._worklist = deque() - self._worklist.extend(self.function.get_basic_blocks()) + worklist = deque(cfg.dfs_walk) - while len(self._worklist) > 0: + while len(worklist) > 0: changed = False - bb = self._worklist.popleft() + + bb = worklist.popleft() changed |= self._calculate_out_vars(bb) changed |= self._calculate_liveness(bb) # recompute liveness for basic blocks pointing into # this basic block if changed: - self._worklist.extend(bb.cfg_in) + worklist.extend(bb.cfg_in) def _reset_liveness(self) -> None: for bb in self.function.get_basic_blocks(): @@ -64,7 +64,7 @@ def _calculate_out_vars(self, bb: IRBasicBlock) -> bool: bb.out_vars = OrderedSet() for out_bb in bb.cfg_out: target_vars = self.input_vars_from(bb, out_bb) - bb.out_vars = bb.out_vars.union(target_vars) + bb.out_vars.update(target_vars) return out_vars != bb.out_vars # calculate the input variables into self from source diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index eb4f7e67ba..f73f847a62 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -185,15 +185,6 @@ def __init__(self, value: str, is_symbol: bool = False) -> None: self.value = value self.is_symbol = is_symbol - def __eq__(self, other): - # no need for is_symbol to participate in equality - return super().__eq__(other) - - def __hash__(self): - # __hash__ is required when __eq__ is overridden -- - # https://docs.python.org/3/reference/datamodel.html#object.__hash__ - return super().__hash__() - class IRInstruction: """ @@ -393,7 +384,6 @@ class IRBasicBlock: # stack items which this basic block produces out_vars: OrderedSet[IRVariable] - reachable: OrderedSet["IRBasicBlock"] is_reachable: bool = False def __init__(self, label: IRLabel, parent: "IRFunction") -> None: @@ -404,7 +394,6 @@ def __init__(self, label: IRLabel, parent: "IRFunction") -> None: self.cfg_in = OrderedSet() self.cfg_out = OrderedSet() self.out_vars = OrderedSet() - self.reachable = OrderedSet() self.is_reachable = False def add_cfg_in(self, bb: "IRBasicBlock") -> None: @@ -495,6 +484,32 @@ def replace_operands(self, replacements: dict) -> None: for instruction in self.instructions: instruction.replace_operands(replacements) + def fix_phi_instructions(self): + cfg_in_labels = tuple(bb.label for bb in self.cfg_in) + + needs_sort = False + for inst in self.instructions: + if inst.opcode != "phi": + continue + + labels = inst.get_label_operands() + for label in labels: + if label not in cfg_in_labels: + needs_sort = True + inst.remove_phi_operand(label) + + op_len = len(inst.operands) + if op_len == 2: + inst.opcode = "store" + inst.operands = [inst.operands[1]] + elif op_len == 0: + inst.opcode = "nop" + inst.output = None + inst.operands = [] + + if needs_sort: + self.instructions.sort(key=lambda inst: inst.opcode != "phi") + def get_assignments(self): """ Get all assignments in basic block. diff --git a/vyper/venom/function.py b/vyper/venom/function.py index 2f2d477460..0c48c9740e 100644 --- a/vyper/venom/function.py +++ b/vyper/venom/function.py @@ -1,8 +1,7 @@ from typing import Iterator, Optional from vyper.codegen.ir_node import IRnode -from vyper.utils import OrderedSet -from vyper.venom.basicblock import CFG_ALTERING_INSTRUCTIONS, IRBasicBlock, IRLabel, IRVariable +from vyper.venom.basicblock import IRBasicBlock, IRLabel, IRVariable class IRFunction: @@ -89,60 +88,31 @@ def get_last_variable(self) -> str: return f"%{self.last_variable}" def remove_unreachable_blocks(self) -> int: - self._compute_reachability() + # Remove unreachable basic blocks + # pre: requires CFG analysis! + # NOTE: should this be a pass? - removed = [] + removed = set() - # Remove unreachable basic blocks for bb in self.get_basic_blocks(): if not bb.is_reachable: - removed.append(bb) + removed.add(bb) for bb in removed: self.remove_basic_block(bb) # Remove phi instructions that reference removed basic blocks - for bb in removed: - for out_bb in bb.cfg_out: - out_bb.remove_cfg_in(bb) - for inst in out_bb.instructions: - if inst.opcode != "phi": - continue - in_labels = inst.get_label_operands() - if bb.label in in_labels: - inst.remove_phi_operand(bb.label) - op_len = len(inst.operands) - if op_len == 2: - inst.opcode = "store" - inst.operands = [inst.operands[1]] - elif op_len == 0: - out_bb.remove_instruction(inst) - - return len(removed) - - def _compute_reachability(self) -> None: - """ - Compute reachability of basic blocks. - """ for bb in self.get_basic_blocks(): - bb.reachable = OrderedSet() - bb.is_reachable = False + for in_bb in list(bb.cfg_in): + if in_bb not in removed: + continue - self._compute_reachability_from(self.entry) + bb.remove_cfg_in(in_bb) - def _compute_reachability_from(self, bb: IRBasicBlock) -> None: - """ - Compute reachability of basic blocks from bb. - """ - if bb.is_reachable: - return - bb.is_reachable = True - for inst in bb.instructions: - if inst.opcode in CFG_ALTERING_INSTRUCTIONS: - for op in inst.get_label_operands(): - out_bb = self.get_basic_block(op.value) - bb.reachable.add(out_bb) - self._compute_reachability_from(out_bb) + # TODO: only run this if cfg_in changed + bb.fix_phi_instructions() + + return len(removed) @property def normalized(self) -> bool: diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index d85e09c9b4..2bdd0ace44 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -5,7 +5,7 @@ from vyper.exceptions import CompilerPanic, StaticAssertionException from vyper.utils import OrderedSet -from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, DominatorTreeAnalysis, IRAnalysesCache +from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, IRAnalysesCache from vyper.venom.basicblock import ( IRBasicBlock, IRInstruction, @@ -50,7 +50,6 @@ class SCCP(IRPass): """ fn: IRFunction - dom: DominatorTreeAnalysis dfg: DFGAnalysis lattice: Lattice work_list: list[WorkListItem] @@ -66,14 +65,13 @@ def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): def run_pass(self): self.fn = self.function - self.dom = self.analyses_cache.request_analysis(DominatorTreeAnalysis) self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) self._calculate_sccp(self.fn.entry) self._propagate_constants() if self.cfg_dirty: self.analyses_cache.force_analysis(CFGAnalysis) - self._fix_phi_nodes() + self.fn.remove_unreachable_blocks() self.analyses_cache.invalidate_analysis(DFGAnalysis) @@ -269,7 +267,7 @@ def _propagate_constants(self): with their actual values. It also replaces conditional jumps with unconditional jumps if the condition is a constant value. """ - for bb in self.dom.dfs_walk: + for bb in self.function.get_basic_blocks(): for inst in bb.instructions: self._replace_constants(inst) @@ -315,33 +313,6 @@ def _replace_constants(self, inst: IRInstruction): if isinstance(lat, IRLiteral): inst.operands[i] = lat - def _fix_phi_nodes(self): - # fix basic blocks whose cfg in was changed - # maybe this should really be done in _visit_phi - for bb in self.fn.get_basic_blocks(): - cfg_in_labels = OrderedSet(in_bb.label for in_bb in bb.cfg_in) - - needs_sort = False - for inst in bb.instructions: - if inst.opcode != "phi": - break - needs_sort |= self._fix_phi_inst(inst, cfg_in_labels) - - # move phi instructions to the top of the block - if needs_sort: - bb.instructions.sort(key=lambda inst: inst.opcode != "phi") - - def _fix_phi_inst(self, inst: IRInstruction, cfg_in_labels: OrderedSet): - operands = [op for label, op in inst.phi_operands if label in cfg_in_labels] - - if len(operands) != 1: - return False - - assert inst.output is not None - inst.opcode = "store" - inst.operands = operands - return True - def _meet(x: LatticeItem, y: LatticeItem) -> LatticeItem: if x == LatticeEnum.TOP: diff --git a/vyper/venom/passes/simplify_cfg.py b/vyper/venom/passes/simplify_cfg.py index acf37376e0..10535c2144 100644 --- a/vyper/venom/passes/simplify_cfg.py +++ b/vyper/venom/passes/simplify_cfg.py @@ -122,16 +122,16 @@ def run_pass(self): for _ in range(fn.num_basic_blocks): changes = self._optimize_empty_basicblocks() + self.analyses_cache.force_analysis(CFGAnalysis) changes += fn.remove_unreachable_blocks() if changes == 0: break else: raise CompilerPanic("Too many iterations removing empty basic blocks") - self.analyses_cache.force_analysis(CFGAnalysis) - for _ in range(fn.num_basic_blocks): # essentially `while True` self._collapse_chained_blocks(entry) + self.analyses_cache.force_analysis(CFGAnalysis) if fn.remove_unreachable_blocks() == 0: break else: diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 264ec35eee..68c8fc3fd7 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -10,7 +10,12 @@ optimize_assembly, ) from vyper.utils import MemoryPositions, OrderedSet -from vyper.venom.analysis import IRAnalysesCache, LivenessAnalysis, VarEquivalenceAnalysis +from vyper.venom.analysis import ( + CFGAnalysis, + IRAnalysesCache, + LivenessAnalysis, + VarEquivalenceAnalysis, +) from vyper.venom.basicblock import ( IRBasicBlock, IRInstruction, @@ -152,6 +157,7 @@ def generate_evm(self, no_optimize: bool = False) -> list[str]: NormalizationPass(ac, fn).run_pass() self.liveness_analysis = ac.request_analysis(LivenessAnalysis) self.equivalence = ac.request_analysis(VarEquivalenceAnalysis) + ac.request_analysis(CFGAnalysis) assert fn.normalized, "Non-normalized CFG!" @@ -308,7 +314,7 @@ def _generate_evm_for_basicblock_r( ref.extend(asm) - for bb in basicblock.reachable: + for bb in basicblock.cfg_out: self._generate_evm_for_basicblock_r(ref, bb, stack.copy()) # pop values from stack at entry to bb From c32b9b4c6f0d8b8cdb103d3017ff540faf56a305 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Tue, 12 Nov 2024 09:44:34 +0200 Subject: [PATCH 17/70] feat[venom]: new `DFTPass` algorithm (#4255) this commit upgrades the DFT algorithm to allow for more instruction movement and performs "multidimensional" fencing, which allows instructions to be reordered across volatile instructions if there is no effect barrier. since barriers do not truly live in the data dependency graph, it introduces a heuristic which chooses which barrier to recurse into first. it also removes the use of order ids and sorting, which improves performance. --------- Co-authored-by: Charles Cooper Co-authored-by: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> --- vyper/ir/compile_ir.py | 3 + vyper/venom/analysis/analysis.py | 2 +- vyper/venom/analysis/dfg.py | 8 +- vyper/venom/analysis/liveness.py | 2 +- vyper/venom/basicblock.py | 65 ++++++++++-- vyper/venom/effects.py | 6 ++ vyper/venom/passes/dft.py | 167 +++++++++++++++++++++---------- vyper/venom/venom_to_assembly.py | 8 +- 8 files changed, 195 insertions(+), 66 deletions(-) diff --git a/vyper/ir/compile_ir.py b/vyper/ir/compile_ir.py index 2cc951b188..e87cf1b310 100644 --- a/vyper/ir/compile_ir.py +++ b/vyper/ir/compile_ir.py @@ -1033,6 +1033,9 @@ def _stack_peephole_opts(assembly): if assembly[i] == "SWAP1" and assembly[i + 1].lower() in COMMUTATIVE_OPS: changed = True del assembly[i] + if assembly[i] == "DUP1" and assembly[i + 1] == "SWAP1": + changed = True + del assembly[i + 1] i += 1 return changed diff --git a/vyper/venom/analysis/analysis.py b/vyper/venom/analysis/analysis.py index f154993925..7bff6ba555 100644 --- a/vyper/venom/analysis/analysis.py +++ b/vyper/venom/analysis/analysis.py @@ -50,9 +50,9 @@ def request_analysis(self, analysis_cls: Type[IRAnalysis], *args, **kwargs): if analysis_cls in self.analyses_cache: return self.analyses_cache[analysis_cls] analysis = analysis_cls(self, self.function) + self.analyses_cache[analysis_cls] = analysis analysis.analyze(*args, **kwargs) - self.analyses_cache[analysis_cls] = analysis return analysis def invalidate_analysis(self, analysis_cls: Type[IRAnalysis]): diff --git a/vyper/venom/analysis/dfg.py b/vyper/venom/analysis/dfg.py index f49b2ac6ac..a2e050094d 100644 --- a/vyper/venom/analysis/dfg.py +++ b/vyper/venom/analysis/dfg.py @@ -3,7 +3,7 @@ from vyper.utils import OrderedSet from vyper.venom.analysis.analysis import IRAnalysesCache, IRAnalysis from vyper.venom.analysis.liveness import LivenessAnalysis -from vyper.venom.basicblock import IRInstruction, IRVariable +from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRVariable from vyper.venom.function import IRFunction @@ -20,6 +20,12 @@ def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): def get_uses(self, op: IRVariable) -> OrderedSet[IRInstruction]: return self._dfg_inputs.get(op, OrderedSet()) + def get_uses_in_bb(self, op: IRVariable, bb: IRBasicBlock): + """ + Get uses of a given variable in a specific basic block. + """ + return [inst for inst in self.get_uses(op) if inst.parent == bb] + # the instruction which produces this variable. def get_producing_instruction(self, op: IRVariable) -> Optional[IRInstruction]: return self._dfg_outputs.get(op) diff --git a/vyper/venom/analysis/liveness.py b/vyper/venom/analysis/liveness.py index 2ee28b9530..0ccda3de2c 100644 --- a/vyper/venom/analysis/liveness.py +++ b/vyper/venom/analysis/liveness.py @@ -60,7 +60,7 @@ def _calculate_out_vars(self, bb: IRBasicBlock) -> bool: Compute out_vars of basic block. Returns True if out_vars changed """ - out_vars = bb.out_vars + out_vars = bb.out_vars.copy() bb.out_vars = OrderedSet() for out_bb in bb.cfg_out: target_vars = self.input_vars_from(bb, out_bb) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index f73f847a62..c0abcefcb0 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -202,7 +202,6 @@ class IRInstruction: # set of live variables at this instruction liveness: OrderedSet[IRVariable] parent: "IRBasicBlock" - fence_id: int annotation: Optional[str] ast_source: Optional[IRnode] error_msg: Optional[str] @@ -219,7 +218,6 @@ def __init__( self.operands = list(operands) # in case we get an iterator self.output = output self.liveness = OrderedSet() - self.fence_id = -1 self.annotation = None self.ast_source = None self.error_msg = None @@ -236,6 +234,22 @@ def is_commutative(self) -> bool: def is_bb_terminator(self) -> bool: return self.opcode in BB_TERMINATORS + @property + def is_phi(self) -> bool: + return self.opcode == "phi" + + @property + def is_param(self) -> bool: + return self.opcode == "param" + + @property + def is_pseudo(self) -> bool: + """ + Check if instruction is pseudo, i.e. not an actual instruction but + a construct for intermediate representation like phi and param. + """ + return self.is_phi or self.is_param + def get_read_effects(self): return effects.reads.get(self.opcode, effects.EMPTY) @@ -321,6 +335,20 @@ def get_ast_source(self) -> Optional[IRnode]: return inst.ast_source return self.parent.parent.ast_source + def str_short(self) -> str: + s = "" + if self.output: + s += f"{self.output} = " + opcode = f"{self.opcode} " if self.opcode != "store" else "" + s += opcode + operands = self.operands + if opcode not in ["jmp", "jnz", "invoke"]: + operands = list(reversed(operands)) + s += ", ".join( + [(f"label %{op}" if isinstance(op, IRLabel) else str(op)) for op in operands] + ) + return s + def __repr__(self) -> str: s = "" if self.output: @@ -337,10 +365,7 @@ def __repr__(self) -> str: if self.annotation: s += f" <{self.annotation}>" - if self.liveness: - return f"{s: <30} # {self.liveness}" - - return s + return f"{s: <30}" def _ir_operand_from_value(val: Any) -> IROperand: @@ -477,6 +502,34 @@ def remove_instruction(self, instruction: IRInstruction) -> None: def clear_instructions(self) -> None: self.instructions = [] + @property + def phi_instructions(self) -> Iterator[IRInstruction]: + for inst in self.instructions: + if inst.opcode == "phi": + yield inst + else: + return + + @property + def non_phi_instructions(self) -> Iterator[IRInstruction]: + return (inst for inst in self.instructions if inst.opcode != "phi") + + @property + def param_instructions(self) -> Iterator[IRInstruction]: + for inst in self.instructions: + if inst.opcode == "param": + yield inst + else: + return + + @property + def pseudo_instructions(self) -> Iterator[IRInstruction]: + return (inst for inst in self.instructions if inst.is_pseudo) + + @property + def body_instructions(self) -> Iterator[IRInstruction]: + return (inst for inst in self.instructions[:-1] if not inst.is_pseudo) + def replace_operands(self, replacements: dict) -> None: """ Update operands with replacements. diff --git a/vyper/venom/effects.py b/vyper/venom/effects.py index 20cc0e4b02..a668ff5439 100644 --- a/vyper/venom/effects.py +++ b/vyper/venom/effects.py @@ -12,6 +12,11 @@ class Effects(Flag): BALANCE = auto() EXTCODE = auto() + def __iter__(self): + # python3.10 doesn't have an iter implementation. we can + # remove this once we drop python3.10 support. + return (m for m in self.__class__.__members__.values() if m in self) + EMPTY = Effects(0) ALL = ~EMPTY @@ -68,6 +73,7 @@ class Effects(Flag): "revert": MEMORY, "return": MEMORY, "sha3": MEMORY, + "sha3_64": MEMORY, "msize": MSIZE, } diff --git a/vyper/venom/passes/dft.py b/vyper/venom/passes/dft.py index 85f27867a7..2bf82810b6 100644 --- a/vyper/venom/passes/dft.py +++ b/vyper/venom/passes/dft.py @@ -1,81 +1,138 @@ +from collections import defaultdict + +import vyper.venom.effects as effects from vyper.utils import OrderedSet -from vyper.venom.analysis import DFGAnalysis -from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRVariable +from vyper.venom.analysis import DFGAnalysis, IRAnalysesCache, LivenessAnalysis +from vyper.venom.basicblock import IRBasicBlock, IRInstruction from vyper.venom.function import IRFunction from vyper.venom.passes.base_pass import IRPass class DFTPass(IRPass): function: IRFunction - inst_order: dict[IRInstruction, int] - inst_order_num: int + inst_offspring: dict[IRInstruction, OrderedSet[IRInstruction]] + visited_instructions: OrderedSet[IRInstruction] + ida: dict[IRInstruction, OrderedSet[IRInstruction]] + + def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): + super().__init__(analyses_cache, function) + self.inst_offspring = {} + + def run_pass(self) -> None: + self.inst_offspring = {} + self.visited_instructions: OrderedSet[IRInstruction] = OrderedSet() + + self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) + basic_blocks = list(self.function.get_basic_blocks()) + + self.function.clear_basic_blocks() + for bb in basic_blocks: + self._process_basic_block(bb) + + self.analyses_cache.invalidate_analysis(LivenessAnalysis) + + def _process_basic_block(self, bb: IRBasicBlock) -> None: + self.function.append_basic_block(bb) + + self._calculate_dependency_graphs(bb) + self.instructions = list(bb.pseudo_instructions) + non_phi_instructions = list(bb.non_phi_instructions) + + self.visited_instructions = OrderedSet() + for inst in non_phi_instructions: + self._calculate_instruction_offspring(inst) + + # Compute entry points in the graph of instruction dependencies + entry_instructions: OrderedSet[IRInstruction] = OrderedSet(non_phi_instructions) + for inst in non_phi_instructions: + to_remove = self.ida.get(inst, OrderedSet()) + if len(to_remove) > 0: + entry_instructions.dropmany(to_remove) + + entry_instructions_list = list(entry_instructions) - def _process_instruction_r(self, bb: IRBasicBlock, inst: IRInstruction, offset: int = 0): - for op in inst.get_outputs(): - assert isinstance(op, IRVariable), f"expected variable, got {op}" - uses = self.dfg.get_uses(op) + # Move the terminator instruction to the end of the list + self._move_terminator_to_end(entry_instructions_list) - for uses_this in uses: - if uses_this.parent != inst.parent or uses_this.fence_id != inst.fence_id: - # don't reorder across basic block or fence boundaries - continue + self.visited_instructions = OrderedSet() + for inst in entry_instructions_list: + self._process_instruction_r(self.instructions, inst) - # if the instruction is a terminator, we need to place - # it at the end of the basic block - # along with all the instructions that "lead" to it - self._process_instruction_r(bb, uses_this, offset) + bb.instructions = self.instructions + assert bb.is_terminated, f"Basic block should be terminated {bb}" + def _move_terminator_to_end(self, instructions: list[IRInstruction]) -> None: + terminator = next((inst for inst in instructions if inst.is_bb_terminator), None) + if terminator is None: + raise ValueError(f"Basic block should have a terminator instruction {self.function}") + instructions.remove(terminator) + instructions.append(terminator) + + def _process_instruction_r(self, instructions: list[IRInstruction], inst: IRInstruction): if inst in self.visited_instructions: return self.visited_instructions.add(inst) - self.inst_order_num += 1 - - if inst.is_bb_terminator: - offset = len(bb.instructions) - if inst.opcode == "phi": - # phi instructions stay at the beginning of the basic block - # and no input processing is needed - # bb.instructions.append(inst) - self.inst_order[inst] = 0 + if inst.is_pseudo: return - for op in inst.get_input_variables(): - target = self.dfg.get_producing_instruction(op) - assert target is not None, f"no producing instruction for {op}" - if target.parent != inst.parent or target.fence_id != inst.fence_id: - # don't reorder across basic block or fence boundaries - continue - self._process_instruction_r(bb, target, offset) + children = list(self.ida[inst]) - self.inst_order[inst] = self.inst_order_num + offset + def key(x): + cost = inst.operands.index(x.output) if x.output in inst.operands else 0 + return cost - len(self.inst_offspring[x]) * 0.5 - def _process_basic_block(self, bb: IRBasicBlock) -> None: - self.function.append_basic_block(bb) + # heuristic: sort by size of child dependency graph + children.sort(key=key) - for inst in bb.instructions: - inst.fence_id = self.fence_id - if inst.is_volatile: - self.fence_id += 1 + for dep_inst in children: + self._process_instruction_r(instructions, dep_inst) - # We go throught the instructions and calculate the order in which they should be executed - # based on the data flow graph. This order is stored in the inst_order dictionary. - # We then sort the instructions based on this order. - self.inst_order = {} - self.inst_order_num = 0 - for inst in bb.instructions: - self._process_instruction_r(bb, inst) + instructions.append(inst) - bb.instructions.sort(key=lambda x: self.inst_order[x]) + def _calculate_dependency_graphs(self, bb: IRBasicBlock) -> None: + # ida: instruction dependency analysis + self.ida = defaultdict(OrderedSet) - def run_pass(self) -> None: - self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) + non_phis = list(bb.non_phi_instructions) - self.fence_id = 0 - self.visited_instructions: OrderedSet[IRInstruction] = OrderedSet() + # + # Compute dependency graph + # + last_write_effects: dict[effects.Effects, IRInstruction] = {} + last_read_effects: dict[effects.Effects, IRInstruction] = {} - basic_blocks = list(self.function.get_basic_blocks()) + for inst in non_phis: + for op in inst.operands: + dep = self.dfg.get_producing_instruction(op) + if dep is not None and dep.parent == bb: + self.ida[inst].add(dep) - self.function.clear_basic_blocks() - for bb in basic_blocks: - self._process_basic_block(bb) + write_effects = inst.get_write_effects() + read_effects = inst.get_read_effects() + + for write_effect in write_effects: + if write_effect in last_read_effects: + self.ida[inst].add(last_read_effects[write_effect]) + last_write_effects[write_effect] = inst + + for read_effect in read_effects: + if read_effect in last_write_effects and last_write_effects[read_effect] != inst: + self.ida[inst].add(last_write_effects[read_effect]) + last_read_effects[read_effect] = inst + + def _calculate_instruction_offspring(self, inst: IRInstruction): + if inst in self.inst_offspring: + return self.inst_offspring[inst] + + self.inst_offspring[inst] = self.ida[inst].copy() + + deps = self.ida[inst] + for dep_inst in deps: + assert inst.parent == dep_inst.parent + if dep_inst.opcode == "store": + continue + res = self._calculate_instruction_offspring(dep_inst) + self.inst_offspring[inst] |= res + + return self.inst_offspring[inst] diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 68c8fc3fd7..9b52b842ba 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -600,10 +600,14 @@ def dup(self, assembly, stack, depth): assembly.append(_evm_dup_for(depth)) def swap_op(self, assembly, stack, op): - return self.swap(assembly, stack, stack.get_depth(op)) + depth = stack.get_depth(op) + assert depth is not StackModel.NOT_IN_STACK, f"Cannot swap non-existent operand {op}" + return self.swap(assembly, stack, depth) def dup_op(self, assembly, stack, op): - self.dup(assembly, stack, stack.get_depth(op)) + depth = stack.get_depth(op) + assert depth is not StackModel.NOT_IN_STACK, f"Cannot dup non-existent operand {op}" + self.dup(assembly, stack, depth) def _evm_swap_for(depth: int) -> str: From dbf9fa08e11d8b07df99bbca26973d5665249315 Mon Sep 17 00:00:00 2001 From: sudo rm -rf --no-preserve-root / Date: Tue, 19 Nov 2024 18:41:31 +0100 Subject: [PATCH 18/70] chore[docs]: add `method_id` to `abi_encode` signature (#4355) this commit adds the missing `method_id` parameter to the function signature of `abi_encode` in the documentation --- docs/built-in-functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/built-in-functions.rst b/docs/built-in-functions.rst index a0e424adb4..7a79379d08 100644 --- a/docs/built-in-functions.rst +++ b/docs/built-in-functions.rst @@ -1023,7 +1023,7 @@ Utilities >>> ExampleContract.foo() 0xa9059cbb -.. py:function:: abi_encode(*args, ensure_tuple: bool = True) -> Bytes[] +.. py:function:: abi_encode(*args, ensure_tuple: bool = True, method_id: Bytes[4] = None) -> Bytes[] Takes a variable number of args as input, and returns the ABIv2-encoded bytestring. Used for packing arguments to raw_call, EIP712 and other cases where a consistent and efficient serialization method is needed. Once this function has seen more use we provisionally plan to put it into the ``ethereum.abi`` namespace. From 7d54f326076f4bbaffab56d8582d2e991b5a4f11 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Tue, 19 Nov 2024 18:06:50 +0000 Subject: [PATCH 19/70] fix[ux]: fix empty hints in error messages (#4351) remove empty hint message from exceptions. this regression was introduced in 03095ce3921636. the root cause of the issue, however, was that `hint=""` could be constructed in the first place. this commit fixes the `get_levenshtein_error_suggestions` helper so that it returns `None` on failure to find a suggestion rather than the empty string `""`. --------- Co-authored-by: Charles Cooper --- .../exceptions/test_undeclared_definition.py | 3 ++- .../syntax/exceptions/test_unknown_type.py | 15 +++++++++++++++ vyper/semantics/analysis/levenshtein_utils.py | 8 ++++---- 3 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 tests/functional/syntax/exceptions/test_unknown_type.py diff --git a/tests/functional/syntax/exceptions/test_undeclared_definition.py b/tests/functional/syntax/exceptions/test_undeclared_definition.py index f90aa4137b..5786b37b1f 100644 --- a/tests/functional/syntax/exceptions/test_undeclared_definition.py +++ b/tests/functional/syntax/exceptions/test_undeclared_definition.py @@ -66,5 +66,6 @@ def foo(): @pytest.mark.parametrize("bad_code", fail_list) def test_undeclared_def_exception(bad_code): - with pytest.raises(UndeclaredDefinition): + with pytest.raises(UndeclaredDefinition) as e: compiler.compile_code(bad_code) + assert "(hint: )" not in str(e.value) diff --git a/tests/functional/syntax/exceptions/test_unknown_type.py b/tests/functional/syntax/exceptions/test_unknown_type.py new file mode 100644 index 0000000000..cd8866d5cb --- /dev/null +++ b/tests/functional/syntax/exceptions/test_unknown_type.py @@ -0,0 +1,15 @@ +import pytest + +from vyper import compiler +from vyper.exceptions import UnknownType + + +def test_unknown_type_exception(): + code = """ +@internal +def foobar(token: IERC20): + pass + """ + with pytest.raises(UnknownType) as e: + compiler.compile_code(code) + assert "(hint: )" not in str(e.value) diff --git a/vyper/semantics/analysis/levenshtein_utils.py b/vyper/semantics/analysis/levenshtein_utils.py index fc6e497d43..ac4fe4fab3 100644 --- a/vyper/semantics/analysis/levenshtein_utils.py +++ b/vyper/semantics/analysis/levenshtein_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Callable +from typing import Any, Callable, Optional def levenshtein_norm(source: str, target: str) -> float: @@ -79,7 +79,7 @@ def get_levenshtein_error_suggestions(*args, **kwargs) -> Callable: def _get_levenshtein_error_suggestions( key: str, namespace: dict[str, Any], threshold: float -) -> str: +) -> Optional[str]: """ Generate an error message snippet for the suggested closest values in the provided namespace with the shortest normalized Levenshtein distance from the given key if that distance @@ -100,11 +100,11 @@ def _get_levenshtein_error_suggestions( """ if key is None or key == "": - return "" + return None distances = sorted([(i, levenshtein_norm(key, i)) for i in namespace], key=lambda k: k[1]) if len(distances) > 0 and distances[0][1] <= threshold: if len(distances) > 1 and distances[1][1] <= threshold: return f"Did you mean '{distances[0][0]}', or maybe '{distances[1][0]}'?" return f"Did you mean '{distances[0][0]}'?" - return "" + return None From 9697bae7dd95078bbcc59cb10fc1a85ca486b93f Mon Sep 17 00:00:00 2001 From: tserg <8017125+tserg@users.noreply.github.com> Date: Wed, 20 Nov 2024 02:21:15 +0800 Subject: [PATCH 20/70] refactor[ux]: refactor preparser (#4293) this PR refactors the pre-parsing routine to use a new `PreParser` object. this will make it easier in the future to keep track of state during pre-parsing. --------- Co-authored-by: Charles Cooper --- tests/functional/grammar/test_grammar.py | 7 +- .../ast/test_annotate_and_optimize_ast.py | 11 +- tests/unit/ast/test_pre_parser.py | 14 ++- vyper/ast/parse.py | 39 +++--- vyper/ast/pre_parser.py | 112 +++++++----------- 5 files changed, 84 insertions(+), 99 deletions(-) diff --git a/tests/functional/grammar/test_grammar.py b/tests/functional/grammar/test_grammar.py index 0ff8c23477..871ba4547f 100644 --- a/tests/functional/grammar/test_grammar.py +++ b/tests/functional/grammar/test_grammar.py @@ -9,7 +9,7 @@ from vyper.ast import Module, parse_to_ast from vyper.ast.grammar import parse_vyper_source, vyper_grammar -from vyper.ast.pre_parser import pre_parse +from vyper.ast.pre_parser import PreParser def test_basic_grammar(): @@ -102,6 +102,7 @@ def has_no_docstrings(c): max_examples=500, suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much] ) def test_grammar_bruteforce(code): - pre_parse_result = pre_parse(code + "\n") - tree = parse_to_ast(pre_parse_result.reformatted_code) + pre_parser = PreParser() + pre_parser.parse(code + "\n") + tree = parse_to_ast(pre_parser.reformatted_code) assert isinstance(tree, Module) diff --git a/tests/unit/ast/test_annotate_and_optimize_ast.py b/tests/unit/ast/test_annotate_and_optimize_ast.py index 39ea899bd9..afba043113 100644 --- a/tests/unit/ast/test_annotate_and_optimize_ast.py +++ b/tests/unit/ast/test_annotate_and_optimize_ast.py @@ -1,6 +1,6 @@ import ast as python_ast -from vyper.ast.parse import annotate_python_ast, pre_parse +from vyper.ast.parse import PreParser, annotate_python_ast class AssertionVisitor(python_ast.NodeVisitor): @@ -28,12 +28,13 @@ def foo() -> int128: def get_contract_info(source_code): - pre_parse_result = pre_parse(source_code) - py_ast = python_ast.parse(pre_parse_result.reformatted_code) + pre_parser = PreParser() + pre_parser.parse(source_code) + py_ast = python_ast.parse(pre_parser.reformatted_code) - annotate_python_ast(py_ast, pre_parse_result.reformatted_code, pre_parse_result) + annotate_python_ast(py_ast, pre_parser.reformatted_code, pre_parser) - return py_ast, pre_parse_result.reformatted_code + return py_ast, pre_parser.reformatted_code def test_it_annotates_ast_with_source_code(): diff --git a/tests/unit/ast/test_pre_parser.py b/tests/unit/ast/test_pre_parser.py index 5d3f30481c..73712aadb8 100644 --- a/tests/unit/ast/test_pre_parser.py +++ b/tests/unit/ast/test_pre_parser.py @@ -1,7 +1,7 @@ import pytest from vyper import compile_code -from vyper.ast.pre_parser import pre_parse, validate_version_pragma +from vyper.ast.pre_parser import PreParser, validate_version_pragma from vyper.compiler.phases import CompilerData from vyper.compiler.settings import OptimizationLevel, Settings from vyper.exceptions import StructureException, VersionException @@ -174,9 +174,10 @@ def test_prerelease_invalid_version_pragma(file_version, mock_version): @pytest.mark.parametrize("code, pre_parse_settings, compiler_data_settings", pragma_examples) def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_version): mock_version("0.3.10") - pre_parse_result = pre_parse(code) + pre_parser = PreParser() + pre_parser.parse(code) - assert pre_parse_result.settings == pre_parse_settings + assert pre_parser.settings == pre_parse_settings compiler_data = CompilerData(code) @@ -203,8 +204,9 @@ def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_ve @pytest.mark.parametrize("code", pragma_venom) def test_parse_venom_pragma(code): - pre_parse_result = pre_parse(code) - assert pre_parse_result.settings.experimental_codegen is True + pre_parser = PreParser() + pre_parser.parse(code) + assert pre_parser.settings.experimental_codegen is True compiler_data = CompilerData(code) assert compiler_data.settings.experimental_codegen is True @@ -252,7 +254,7 @@ def test_parse_venom_pragma(code): @pytest.mark.parametrize("code", invalid_pragmas) def test_invalid_pragma(code): with pytest.raises(StructureException): - pre_parse(code) + PreParser().parse(code) def test_version_exception_in_import(make_input_bundle): diff --git a/vyper/ast/parse.py b/vyper/ast/parse.py index 1e88241186..5d62072b9e 100644 --- a/vyper/ast/parse.py +++ b/vyper/ast/parse.py @@ -6,7 +6,7 @@ import asttokens from vyper.ast import nodes as vy_ast -from vyper.ast.pre_parser import PreParseResult, pre_parse +from vyper.ast.pre_parser import PreParser from vyper.compiler.settings import Settings from vyper.exceptions import CompilerPanic, ParserException, SyntaxException from vyper.utils import sha256sum, vyper_warn @@ -54,9 +54,10 @@ def parse_to_ast_with_settings( """ if "\x00" in vyper_source: raise ParserException("No null bytes (\\x00) allowed in the source code.") - pre_parse_result = pre_parse(vyper_source) + pre_parser = PreParser() + pre_parser.parse(vyper_source) try: - py_ast = python_ast.parse(pre_parse_result.reformatted_code) + py_ast = python_ast.parse(pre_parser.reformatted_code) except SyntaxError as e: # TODO: Ensure 1-to-1 match of source_code:reformatted_code SyntaxErrors raise SyntaxException(str(e), vyper_source, e.lineno, e.offset) from None @@ -72,20 +73,20 @@ def parse_to_ast_with_settings( annotate_python_ast( py_ast, vyper_source, - pre_parse_result, + pre_parser, source_id=source_id, module_path=module_path, resolved_path=resolved_path, ) # postcondition: consumed all the for loop annotations - assert len(pre_parse_result.for_loop_annotations) == 0 + assert len(pre_parser.for_loop_annotations) == 0 # Convert to Vyper AST. module = vy_ast.get_node(py_ast) assert isinstance(module, vy_ast.Module) # mypy hint - return pre_parse_result.settings, module + return pre_parser.settings, module def ast_to_dict(ast_struct: Union[vy_ast.VyperNode, List]) -> Union[Dict, List]: @@ -116,7 +117,7 @@ def dict_to_ast(ast_struct: Union[Dict, List]) -> Union[vy_ast.VyperNode, List]: def annotate_python_ast( parsed_ast: python_ast.AST, vyper_source: str, - pre_parse_result: PreParseResult, + pre_parser: PreParser, source_id: int = 0, module_path: Optional[str] = None, resolved_path: Optional[str] = None, @@ -130,8 +131,8 @@ def annotate_python_ast( The AST to be annotated and optimized. vyper_source: str The original vyper source code - pre_parse_result: PreParseResult - Outputs from pre-parsing. + pre_parser: PreParser + PreParser object. Returns ------- @@ -142,7 +143,7 @@ def annotate_python_ast( tokens.mark_tokens(parsed_ast) visitor = AnnotatingVisitor( vyper_source, - pre_parse_result, + pre_parser, tokens, source_id, module_path=module_path, @@ -155,12 +156,12 @@ def annotate_python_ast( class AnnotatingVisitor(python_ast.NodeTransformer): _source_code: str - _pre_parse_result: PreParseResult + _pre_parser: PreParser def __init__( self, source_code: str, - pre_parse_result: PreParseResult, + pre_parser: PreParser, tokens: asttokens.ASTTokens, source_id: int, module_path: Optional[str] = None, @@ -171,7 +172,7 @@ def __init__( self._module_path = module_path self._resolved_path = resolved_path self._source_code = source_code - self._pre_parse_result = pre_parse_result + self._pre_parser = pre_parser self.counter: int = 0 @@ -265,7 +266,7 @@ def visit_ClassDef(self, node): """ self.generic_visit(node) - node.ast_type = self._pre_parse_result.modification_offsets[(node.lineno, node.col_offset)] + node.ast_type = self._pre_parser.modification_offsets[(node.lineno, node.col_offset)] return node def visit_For(self, node): @@ -274,7 +275,7 @@ def visit_For(self, node): the pre-parser """ key = (node.lineno, node.col_offset) - annotation_tokens = self._pre_parse_result.for_loop_annotations.pop(key) + annotation_tokens = self._pre_parser.for_loop_annotations.pop(key) if not annotation_tokens: # a common case for people migrating to 0.4.0, provide a more @@ -342,14 +343,14 @@ def visit_Expr(self, node): # CMC 2024-03-03 consider unremoving this from the enclosing Expr node = node.value key = (node.lineno, node.col_offset) - node.ast_type = self._pre_parse_result.modification_offsets[key] + node.ast_type = self._pre_parser.modification_offsets[key] return node def visit_Await(self, node): start_pos = node.lineno, node.col_offset # grab these before generic_visit modifies them self.generic_visit(node) - node.ast_type = self._pre_parse_result.modification_offsets[start_pos] + node.ast_type = self._pre_parser.modification_offsets[start_pos] return node def visit_Call(self, node): @@ -394,10 +395,10 @@ def visit_Constant(self, node): node.ast_type = "NameConstant" elif isinstance(node.value, str): key = (node.lineno, node.col_offset) - if key in self._pre_parse_result.native_hex_literal_locations: + if key in self._pre_parser.hex_string_locations: if len(node.value) % 2 != 0: raise SyntaxException( - "Native hex string must have an even number of characters", + "Hex string must have an even number of characters", self._source_code, node.lineno, node.col_offset, diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index 5d2abcf645..dbeb6181f9 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -158,67 +158,52 @@ def consume(self, token, result): CUSTOM_EXPRESSION_TYPES = {"extcall": "ExtCall", "staticcall": "StaticCall"} -class PreParseResult: +class PreParser: # Compilation settings based on the directives in the source code settings: Settings # A mapping of class names to their original class types. modification_offsets: dict[tuple[int, int], str] # A mapping of line/column offsets of `For` nodes to the annotation of the for loop target for_loop_annotations: dict[tuple[int, int], list[TokenInfo]] - # A list of line/column offsets of native hex literals - native_hex_literal_locations: list[tuple[int, int]] + # A list of line/column offsets of hex string literals + hex_string_locations: list[tuple[int, int]] # Reformatted python source string. reformatted_code: str - def __init__( - self, - settings, - modification_offsets, - for_loop_annotations, - native_hex_literal_locations, - reformatted_code, - ): - self.settings = settings - self.modification_offsets = modification_offsets - self.for_loop_annotations = for_loop_annotations - self.native_hex_literal_locations = native_hex_literal_locations - self.reformatted_code = reformatted_code - - -def pre_parse(code: str) -> PreParseResult: - """ - Re-formats a vyper source string into a python source string and performs - some validation. More specifically, - - * Translates "interface", "struct", "flag", and "event" keywords into python "class" keyword - * Validates "@version" pragma against current compiler version - * Prevents direct use of python "class" keyword - * Prevents use of python semi-colon statement separator - * Extracts type annotation of for loop iterators into a separate dictionary - - Also returns a mapping of detected interface and struct names to their - respective vyper class types ("interface" or "struct"), and a mapping of line numbers - of for loops to the type annotation of their iterators. - - Parameters - ---------- - code : str - The vyper source code to be re-formatted. - - Returns - ------- - PreParseResult - Outputs for transforming the python AST to vyper AST - """ - result: list[TokenInfo] = [] - modification_offsets: dict[tuple[int, int], str] = {} - settings = Settings() - for_parser = ForParser(code) - native_hex_parser = HexStringParser() + def parse(self, code: str): + """ + Re-formats a vyper source string into a python source string and performs + some validation. More specifically, + + * Translates "interface", "struct", "flag", and "event" keywords into python "class" keyword + * Validates "@version" pragma against current compiler version + * Prevents direct use of python "class" keyword + * Prevents use of python semi-colon statement separator + * Extracts type annotation of for loop iterators into a separate dictionary + + Stores a mapping of detected interface and struct names to their + respective vyper class types ("interface" or "struct"), and a mapping of line numbers + of for loops to the type annotation of their iterators. + + Parameters + ---------- + code : str + The vyper source code to be re-formatted. + """ + try: + self._parse(code) + except TokenError as e: + raise SyntaxException(e.args[0], code, e.args[1][0], e.args[1][1]) from e + + def _parse(self, code: str): + result: list[TokenInfo] = [] + modification_offsets: dict[tuple[int, int], str] = {} + settings = Settings() + for_parser = ForParser(code) + hex_string_parser = HexStringParser() + + _col_adjustments: dict[int, int] = defaultdict(lambda: 0) - _col_adjustments: dict[int, int] = defaultdict(lambda: 0) - - try: code_bytes = code.encode("utf-8") token_list = list(tokenize(io.BytesIO(code_bytes).readline)) @@ -301,7 +286,7 @@ def pre_parse(code: str) -> PreParseResult: # a bit cursed technique to get untokenize to put # the new tokens in the right place so that modification_offsets # will work correctly. - # (recommend comparing the result of pre_parse with the + # (recommend comparing the result of parse with the # source code side by side to visualize the whitespace) new_keyword = "await" vyper_type = CUSTOM_EXPRESSION_TYPES[string] @@ -322,20 +307,15 @@ def pre_parse(code: str) -> PreParseResult: if (typ, string) == (OP, ";"): raise SyntaxException("Semi-colon statements not allowed", code, start[0], start[1]) - if not for_parser.consume(token) and not native_hex_parser.consume(token, result): + if not for_parser.consume(token) and not hex_string_parser.consume(token, result): result.extend(toks) - except TokenError as e: - raise SyntaxException(e.args[0], code, e.args[1][0], e.args[1][1]) from e - - for_loop_annotations = {} - for k, v in for_parser.annotations.items(): - for_loop_annotations[k] = v.copy() + for_loop_annotations = {} + for k, v in for_parser.annotations.items(): + for_loop_annotations[k] = v.copy() - return PreParseResult( - settings, - modification_offsets, - for_loop_annotations, - native_hex_parser.locations, - untokenize(result).decode("utf-8"), - ) + self.settings = settings + self.modification_offsets = modification_offsets + self.for_loop_annotations = for_loop_annotations + self.hex_string_locations = hex_string_parser.locations + self.reformatted_code = untokenize(result).decode("utf-8") From 0c23b214018ac3a09bdc2df31843a29467ae3695 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 20 Nov 2024 16:27:34 +0200 Subject: [PATCH 21/70] refactor[venom]: optimize lattice evaluation (#4368) this commit improves the `_eval()` method in the SCCP pass by trying to return early when possible and avoid unnecessary set operations. it also performs some minor code clean up and dead code removal. --- vyper/venom/passes/sccp/sccp.py | 48 +++++++++++++++++---------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index 2bdd0ace44..2be84ce502 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -227,35 +227,37 @@ def _eval(self, inst) -> LatticeItem: instruction to the SSA work list if the knowledge about the variable changed. """ - opcode = inst.opcode - ops = [] + def finalize(ret): + # Update the lattice if the value changed + old_val = self.lattice.get(inst.output, LatticeEnum.TOP) + if old_val != ret: + self.lattice[inst.output] = ret + self._add_ssa_work_items(inst) + return ret + + opcode = inst.opcode + ops: list[IROperand] = [] for op in inst.operands: - if isinstance(op, IRVariable): - ops.append(self.lattice[op]) - elif isinstance(op, IRLabel): - return LatticeEnum.BOTTOM + # Evaluate the operand according to the lattice + if isinstance(op, IRLabel): + return finalize(LatticeEnum.BOTTOM) + elif isinstance(op, IRVariable): + eval_result = self.lattice[op] else: - ops.append(op) + eval_result = op - ret = None - if LatticeEnum.BOTTOM in ops: - ret = LatticeEnum.BOTTOM - else: - if opcode in ARITHMETIC_OPS: - fn = ARITHMETIC_OPS[opcode] - ret = IRLiteral(fn(ops)) # type: ignore - elif len(ops) > 0: - ret = ops[0] # type: ignore - else: - raise CompilerPanic("Bad constant evaluation") + # If any operand is BOTTOM, the whole operation is BOTTOM + # and we can stop the evaluation early + if eval_result is LatticeEnum.BOTTOM: + return finalize(LatticeEnum.BOTTOM) - old_val = self.lattice.get(inst.output, LatticeEnum.TOP) - if old_val != ret: - self.lattice[inst.output] = ret # type: ignore - self._add_ssa_work_items(inst) + assert isinstance(eval_result, IROperand) + ops.append(eval_result) - return ret # type: ignore + # If we haven't found BOTTOM yet, evaluate the operation + fn = ARITHMETIC_OPS[opcode] + return finalize(IRLiteral(fn(ops))) def _add_ssa_work_items(self, inst: IRInstruction): for target_inst in self.dfg.get_uses(inst.output): # type: ignore From 215de1da6c53fcc97d00f38621d13b622d4a224f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 20 Nov 2024 22:29:18 +0700 Subject: [PATCH 22/70] feat[venom]: make dft pass commutative aware (#4358) this commit performs some code cleanup for `DFTPass`. it separates the effects graph from the data dependency graph, and applies the heuristic for barriers to commutative and comparator instructions as well. --- vyper/venom/basicblock.py | 25 ++++++++++++ vyper/venom/passes/dft.py | 86 ++++++++++++++++++--------------------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index c0abcefcb0..968ce42bdf 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -84,6 +84,8 @@ COMMUTATIVE_INSTRUCTIONS = frozenset(["add", "mul", "smul", "or", "xor", "and", "eq"]) +COMPARATOR_INSTRUCTIONS = ("gt", "lt", "sgt", "slt") + if TYPE_CHECKING: from vyper.venom.function import IRFunction @@ -230,6 +232,14 @@ def is_volatile(self) -> bool: def is_commutative(self) -> bool: return self.opcode in COMMUTATIVE_INSTRUCTIONS + @property + def is_comparator(self) -> bool: + return self.opcode in COMPARATOR_INSTRUCTIONS + + @property + def flippable(self) -> bool: + return self.is_commutative or self.is_comparator + @property def is_bb_terminator(self) -> bool: return self.opcode in BB_TERMINATORS @@ -282,6 +292,21 @@ def get_outputs(self) -> list[IROperand]: """ return [self.output] if self.output else [] + def flip(self): + """ + Flip operands for commutative or comparator opcodes + """ + assert self.flippable + self.operands.reverse() + + if self.is_commutative: + return + + if self.opcode in ("gt", "sgt"): + self.opcode = self.opcode.replace("g", "l") + else: + self.opcode = self.opcode.replace("l", "g") + def replace_operands(self, replacements: dict) -> None: """ Update operands with replacements. diff --git a/vyper/venom/passes/dft.py b/vyper/venom/passes/dft.py index 2bf82810b6..a8d68ad676 100644 --- a/vyper/venom/passes/dft.py +++ b/vyper/venom/passes/dft.py @@ -2,7 +2,7 @@ import vyper.venom.effects as effects from vyper.utils import OrderedSet -from vyper.venom.analysis import DFGAnalysis, IRAnalysesCache, LivenessAnalysis +from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis from vyper.venom.basicblock import IRBasicBlock, IRInstruction from vyper.venom.function import IRFunction from vyper.venom.passes.base_pass import IRPass @@ -10,50 +10,41 @@ class DFTPass(IRPass): function: IRFunction - inst_offspring: dict[IRInstruction, OrderedSet[IRInstruction]] + data_offspring: dict[IRInstruction, OrderedSet[IRInstruction]] visited_instructions: OrderedSet[IRInstruction] - ida: dict[IRInstruction, OrderedSet[IRInstruction]] - - def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): - super().__init__(analyses_cache, function) - self.inst_offspring = {} + # "data dependency analysis" + dda: dict[IRInstruction, OrderedSet[IRInstruction]] + # "effect dependency analysis" + eda: dict[IRInstruction, OrderedSet[IRInstruction]] def run_pass(self) -> None: - self.inst_offspring = {} + self.data_offspring = {} self.visited_instructions: OrderedSet[IRInstruction] = OrderedSet() self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) - basic_blocks = list(self.function.get_basic_blocks()) - self.function.clear_basic_blocks() - for bb in basic_blocks: + for bb in self.function.get_basic_blocks(): self._process_basic_block(bb) self.analyses_cache.invalidate_analysis(LivenessAnalysis) def _process_basic_block(self, bb: IRBasicBlock) -> None: - self.function.append_basic_block(bb) - self._calculate_dependency_graphs(bb) self.instructions = list(bb.pseudo_instructions) non_phi_instructions = list(bb.non_phi_instructions) self.visited_instructions = OrderedSet() - for inst in non_phi_instructions: - self._calculate_instruction_offspring(inst) + for inst in bb.instructions: + self._calculate_data_offspring(inst) # Compute entry points in the graph of instruction dependencies entry_instructions: OrderedSet[IRInstruction] = OrderedSet(non_phi_instructions) for inst in non_phi_instructions: - to_remove = self.ida.get(inst, OrderedSet()) - if len(to_remove) > 0: - entry_instructions.dropmany(to_remove) + to_remove = self.dda.get(inst, OrderedSet()) | self.eda.get(inst, OrderedSet()) + entry_instructions.dropmany(to_remove) entry_instructions_list = list(entry_instructions) - # Move the terminator instruction to the end of the list - self._move_terminator_to_end(entry_instructions_list) - self.visited_instructions = OrderedSet() for inst in entry_instructions_list: self._process_instruction_r(self.instructions, inst) @@ -61,13 +52,6 @@ def _process_basic_block(self, bb: IRBasicBlock) -> None: bb.instructions = self.instructions assert bb.is_terminated, f"Basic block should be terminated {bb}" - def _move_terminator_to_end(self, instructions: list[IRInstruction]) -> None: - terminator = next((inst for inst in instructions if inst.is_bb_terminator), None) - if terminator is None: - raise ValueError(f"Basic block should have a terminator instruction {self.function}") - instructions.remove(terminator) - instructions.append(terminator) - def _process_instruction_r(self, instructions: list[IRInstruction], inst: IRInstruction): if inst in self.visited_instructions: return @@ -76,14 +60,23 @@ def _process_instruction_r(self, instructions: list[IRInstruction], inst: IRInst if inst.is_pseudo: return - children = list(self.ida[inst]) + children = list(self.dda[inst] | self.eda[inst]) - def key(x): - cost = inst.operands.index(x.output) if x.output in inst.operands else 0 - return cost - len(self.inst_offspring[x]) * 0.5 + def cost(x: IRInstruction) -> int | float: + if x in self.eda[inst] or inst.flippable: + ret = -1 * int(len(self.data_offspring[x]) > 0) + else: + assert x in self.dda[inst] # sanity check + assert x.output is not None # help mypy + ret = inst.operands.index(x.output) + return ret # heuristic: sort by size of child dependency graph - children.sort(key=key) + orig_children = children.copy() + children.sort(key=cost) + + if inst.flippable and (orig_children != children): + inst.flip() for dep_inst in children: self._process_instruction_r(instructions, dep_inst) @@ -92,7 +85,8 @@ def key(x): def _calculate_dependency_graphs(self, bb: IRBasicBlock) -> None: # ida: instruction dependency analysis - self.ida = defaultdict(OrderedSet) + self.dda = defaultdict(OrderedSet) + self.eda = defaultdict(OrderedSet) non_phis = list(bb.non_phi_instructions) @@ -106,33 +100,31 @@ def _calculate_dependency_graphs(self, bb: IRBasicBlock) -> None: for op in inst.operands: dep = self.dfg.get_producing_instruction(op) if dep is not None and dep.parent == bb: - self.ida[inst].add(dep) + self.dda[inst].add(dep) write_effects = inst.get_write_effects() read_effects = inst.get_read_effects() for write_effect in write_effects: if write_effect in last_read_effects: - self.ida[inst].add(last_read_effects[write_effect]) + self.eda[inst].add(last_read_effects[write_effect]) last_write_effects[write_effect] = inst for read_effect in read_effects: if read_effect in last_write_effects and last_write_effects[read_effect] != inst: - self.ida[inst].add(last_write_effects[read_effect]) + self.eda[inst].add(last_write_effects[read_effect]) last_read_effects[read_effect] = inst - def _calculate_instruction_offspring(self, inst: IRInstruction): - if inst in self.inst_offspring: - return self.inst_offspring[inst] + def _calculate_data_offspring(self, inst: IRInstruction): + if inst in self.data_offspring: + return self.data_offspring[inst] - self.inst_offspring[inst] = self.ida[inst].copy() + self.data_offspring[inst] = self.dda[inst].copy() - deps = self.ida[inst] + deps = self.dda[inst] for dep_inst in deps: assert inst.parent == dep_inst.parent - if dep_inst.opcode == "store": - continue - res = self._calculate_instruction_offspring(dep_inst) - self.inst_offspring[inst] |= res + res = self._calculate_data_offspring(dep_inst) + self.data_offspring[inst] |= res - return self.inst_offspring[inst] + return self.data_offspring[inst] From e0fc53a10e69abd55ad563f98477bcf9568d76b0 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Wed, 20 Nov 2024 20:42:34 +0000 Subject: [PATCH 23/70] chore[docs]: mention the `--venom` flag in venom docs (#4353) add some notes to the relevant docs that `venom` is an alias for `experimental-codegen` --- docs/compiling-a-contract.rst | 7 ++++++- docs/structure-of-a-contract.rst | 10 ++++++++++ vyper/cli/vyper_compile.py | 6 +++++- vyper/venom/README.md | 2 +- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/compiling-a-contract.rst b/docs/compiling-a-contract.rst index c2cd3ed22c..7132cff58d 100644 --- a/docs/compiling-a-contract.rst +++ b/docs/compiling-a-contract.rst @@ -31,7 +31,7 @@ Include the ``-f`` flag to specify which output formats to return. Use ``vyper - .. code:: shell - $ vyper -f abi,abi_python,bytecode,bytecode_runtime,blueprint_bytecode,interface,external_interface,ast,annotated_ast,integrity,ir,ir_json,ir_runtime,asm,opcodes,opcodes_runtime,source_map,source_map_runtime,archive,solc_json,method_identifiers,userdoc,devdoc,metadata,combined_json,layout yourFileName.vy + $ vyper -f abi,abi_python,bb,bb_runtime,bytecode,bytecode_runtime,blueprint_bytecode,cfg,cfg_runtime,interface,external_interface,ast,annotated_ast,integrity,ir,ir_json,ir_runtime,asm,opcodes,opcodes_runtime,source_map,source_map_runtime,archive,solc_json,method_identifiers,userdoc,devdoc,metadata,combined_json,layout yourFileName.vy .. note:: The ``opcodes`` and ``opcodes_runtime`` output of the compiler has been returning incorrect opcodes since ``0.2.0`` due to a lack of 0 padding (patched via `PR 3735 `_). If you rely on these functions for debugging, please use the latest patched versions. @@ -134,6 +134,11 @@ In codesize optimized mode, the compiler will try hard to minimize codesize by * out-lining code, and * using more loops for data copies. +Enabling Experimental Code Generation +=========================== + +When compiling, you can use the CLI flag ``--experimental-codegen`` or its alias ``--venom`` to activate the new `Venom IR `_. +Venom IR is inspired by LLVM IR and enables new advanced analysis and optimizations. .. _evm-version: diff --git a/docs/structure-of-a-contract.rst b/docs/structure-of-a-contract.rst index fc817cf4b6..7e599d677b 100644 --- a/docs/structure-of-a-contract.rst +++ b/docs/structure-of-a-contract.rst @@ -54,6 +54,16 @@ EVM Version The EVM version can be set with the ``evm-version`` pragma, which is documented in :ref:`evm-version`. +Experimental Code Generation +----------------- +The new experimental code generation feature can be activated using the following directive: + +.. code-block:: vyper + + #pragma experimental-codegen + +Alternatively, you can use the alias ``"venom"`` instead of ``"experimental-codegen"`` to enable this feature. + Imports ======= diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index fde35f781e..046cac2c0b 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -34,6 +34,8 @@ layout - Storage layout of a Vyper contract ast - AST (not yet annotated) in JSON format annotated_ast - Annotated AST in JSON format +cfg - Control flow graph of deployable bytecode +cfg_runtime - Control flow graph of runtime bytecode interface - Vyper interface of a contract external_interface - External interface of a contract, used for outside contract calls opcodes - List of opcodes as a string @@ -41,6 +43,8 @@ ir - Intermediate representation in list format ir_json - Intermediate representation in JSON format ir_runtime - Intermediate representation of runtime bytecode in list format +bb - Basic blocks of Venom IR for deployable bytecode +bb_runtime - Basic blocks of Venom IR for runtime bytecode asm - Output the EVM assembly of the deployable bytecode integrity - Output the integrity hash of the source code archive - Output the build as an archive file @@ -177,7 +181,7 @@ def _parse_args(argv): parser.add_argument( "--experimental-codegen", "--venom", - help="The compiler use the new IR codegen. This is an experimental feature.", + help="The compiler uses the new IR codegen. This is an experimental feature.", action="store_true", dest="experimental_codegen", ) diff --git a/vyper/venom/README.md b/vyper/venom/README.md index 4e4f5ca3d1..6f3b318c9b 100644 --- a/vyper/venom/README.md +++ b/vyper/venom/README.md @@ -193,7 +193,7 @@ An operand can be a label, a variable, or a literal. By convention, variables have a `%-` prefix, e.g. `%1` is a valid variable. However, the prefix is not required. ## Instructions -To enable Venom IR in Vyper, use the `--experimental-codegen` flag. To view the Venom IR output, use `-f bb_runtime` for the runtime code, or `-f bb` to see the deploy code. To get a dot file (for use e.g. with `xdot -`), use `-f cfg` or `-f cfg_runtime`. +To enable Venom IR in Vyper, use the `--experimental-codegen` CLI flag or its alias `--venom`, or the corresponding pragma statements (e.g. `#pragma experimental-codegen`). To view the Venom IR output, use `-f bb_runtime` for the runtime code, or `-f bb` to see the deploy code. To get a dot file (for use e.g. with `xdot -`), use `-f cfg` or `-f cfg_runtime`. Assembly can be inspected with `-f asm`, whereas an opcode view of the final bytecode can be seen with `-f opcodes` or `-f opcodes_runtime`, respectively. From f38b61a27a975c80600d54981386e3cf8697edb6 Mon Sep 17 00:00:00 2001 From: cyberthirst Date: Thu, 21 Nov 2024 16:54:55 +0000 Subject: [PATCH 24/70] refactor[test]: add some sanity checks to `abi_decode` tests (#4096) QOL improvements for abi_decode tests: - added sanity checks to abi decode tests to ensure that we're never failing on calldatasize - also split the creation of payload into two parts when calling target contract with low-level msg_call to increase readability --------- Co-authored-by: Charles Cooper --- .../builtins/codegen/test_abi_decode.py | 377 ++++++++++-------- 1 file changed, 216 insertions(+), 161 deletions(-) diff --git a/tests/functional/builtins/codegen/test_abi_decode.py b/tests/functional/builtins/codegen/test_abi_decode.py index 9ae869c9cc..475118c7e3 100644 --- a/tests/functional/builtins/codegen/test_abi_decode.py +++ b/tests/functional/builtins/codegen/test_abi_decode.py @@ -8,6 +8,8 @@ TEST_ADDR = "0x" + b"".join(chr(i).encode("utf-8") for i in range(20)).hex() +BUFFER_OVERHEAD = 4 + 2 * 32 + def test_abi_decode_complex(get_contract): contract = """ @@ -474,8 +476,10 @@ def test_abi_decode_length_mismatch(get_contract, assert_compile_failed, bad_cod assert_compile_failed(lambda: get_contract(bad_code), exception) -def _abi_payload_from_tuple(payload: tuple[int | bytes, ...]) -> bytes: - return b"".join(p.to_bytes(32, "big") if isinstance(p, int) else p for p in payload) +def _abi_payload_from_tuple(payload: tuple[int | bytes, ...], max_sz: int) -> bytes: + ret = b"".join(p.to_bytes(32, "big") if isinstance(p, int) else p for p in payload) + assert len(ret) <= max_sz + return ret def _replicate(value: int, count: int) -> tuple[int, ...]: @@ -486,11 +490,12 @@ def test_abi_decode_arithmetic_overflow(env, tx_failed, get_contract): # test based on GHSA-9p8r-4xp4-gw5w: # https://github.com/vyperlang/vyper/security/advisories/GHSA-9p8r-4xp4-gw5w#advisory-comment-91841 # buf + head causes arithmetic overflow - code = """ + buffer_size = 32 * 3 + code = f""" @external -def f(x: Bytes[32 * 3]): +def f(x: Bytes[{buffer_size}]): a: Bytes[32] = b"foo" - y: Bytes[32 * 3] = x + y: Bytes[{buffer_size}] = x decoded_y1: Bytes[32] = _abi_decode(y, Bytes[32]) a = b"bar" @@ -500,39 +505,47 @@ def f(x: Bytes[32 * 3]): """ c = get_contract(code) - data = method_id("f(bytes)") - payload = ( - 0x20, # tuple head - 0x60, # parent array length - # parent payload - this word will be considered as the head of the abi-encoded inner array - # and it will be added to base ptr leading to an arithmetic overflow - 2**256 - 0x60, - ) - data += _abi_payload_from_tuple(payload) + tuple_head_ofst = 0x20 + parent_array_len = 0x60 + msg_call_overhead = (method_id("f(bytes)"), tuple_head_ofst, parent_array_len) + + data = _abi_payload_from_tuple(msg_call_overhead, BUFFER_OVERHEAD) + + # parent payload - this word will be considered as the head of the + # abi-encoded inner array and it will be added to base ptr leading to an + # arithmetic overflow + buffer_payload = (2**256 - 0x60,) + + data += _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): env.message_call(c.address, data=data) -def test_abi_decode_nonstrict_head(env, tx_failed, get_contract): +def test_abi_decode_nonstrict_head(env, get_contract): # data isn't strictly encoded - head is 0x21 instead of 0x20 # but the head + length is still within runtime bounds of the parent buffer - code = """ + buffer_size = 32 * 5 + code = f""" @external -def f(x: Bytes[32 * 5]): - y: Bytes[32 * 5] = x +def f(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x a: Bytes[32] = b"a" decoded_y1: DynArray[uint256, 3] = _abi_decode(y, DynArray[uint256, 3]) + assert len(decoded_y1) == 1 and decoded_y1[0] == 0 a = b"aaaa" decoded_y1 = _abi_decode(y, DynArray[uint256, 3]) + assert len(decoded_y1) == 1 and decoded_y1[0] == 0 """ c = get_contract(code) - data = method_id("f(bytes)") + tuple_head_ofst = 0x20 + parent_array_len = 0xA0 + msg_call_overhead = (method_id("f(bytes)"), tuple_head_ofst, parent_array_len) - payload = ( - 0x20, # tuple head - 0xA0, # parent array length + data = _abi_payload_from_tuple(msg_call_overhead, BUFFER_OVERHEAD) + + buffer_payload = ( # head should be 0x20 but is 0x21 thus the data isn't strictly encoded 0x21, # we don't want to revert on invalid length, so set this to 0 @@ -543,27 +556,30 @@ def f(x: Bytes[32 * 5]): *_replicate(0x03, 2), ) - data += _abi_payload_from_tuple(payload) + data += _abi_payload_from_tuple(buffer_payload, buffer_size) env.message_call(c.address, data=data) def test_abi_decode_child_head_points_to_parent(tx_failed, get_contract): # data isn't strictly encoded and the head for the inner array - # skipts the corresponding payload and points to other valid section of the parent buffer - code = """ + # skips the corresponding payload and points to other valid section of the + # parent buffer + buffer_size = 14 * 32 + code = f""" @external -def run(x: Bytes[14 * 32]): - y: Bytes[14 * 32] = x +def run(x: Bytes[{buffer_size}]) -> DynArray[DynArray[DynArray[uint256, 2], 1], 2]: + y: Bytes[{buffer_size}] = x decoded_y1: DynArray[DynArray[DynArray[uint256, 2], 1], 2] = _abi_decode( y, DynArray[DynArray[DynArray[uint256, 2], 1], 2] ) + return decoded_y1 """ c = get_contract(code) # encode [[[1, 1]], [[2, 2]]] and modify the head for [1, 1] # to actually point to [2, 2] - payload = ( + buffer_payload = ( 0x20, # top-level array head 0x02, # top-level array length 0x40, # head of DAr[DAr[DAr, uint256]]][0] @@ -582,30 +598,33 @@ def run(x: Bytes[14 * 32]): 0x02, # DAr[DAr[DAr, uint256]]][1][0][1] ) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) - c.run(data) + res = c.run(data) + assert res == [[[2, 2]], [[2, 2]]] def test_abi_decode_nonstrict_head_oob(tx_failed, get_contract): # data isn't strictly encoded and (non_strict_head + len(DynArray[..][2])) > parent_static_sz # thus decoding the data pointed to by the head would cause an OOB read # non_strict_head + length == parent + parent_static_sz + 1 - code = """ + buffer_size = 2 * 32 + 3 * 32 + 3 * 32 * 4 + code = f""" @external -def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): - y: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4] = x +def run(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x decoded_y1: DynArray[Bytes[32 * 3], 3] = _abi_decode(y, DynArray[Bytes[32 * 3], 3]) """ c = get_contract(code) - payload = ( + buffer_payload = ( 0x20, # DynArray head 0x03, # DynArray length - # non_strict_head - if the length pointed to by this head is 0x60 (which is valid - # length for the Bytes[32*3] buffer), the decoding function would decode - # 1 byte over the end of the buffer - # we define the non_strict_head as: skip the remaining heads, 1st and 2nd tail + # non_strict_head - if the length pointed to by this head is 0x60 + # (which is valid length for the Bytes[32*3] buffer), the decoding + # function would decode 1 byte over the end of the buffer + # we define the non_strict_head as: + # skip the remaining heads, 1st and 2nd tail # to the third tail + 1B 0x20 * 8 + 0x20 * 3 + 0x01, # inner array0 head 0x20 * 4 + 0x20 * 3, # inner array1 head @@ -622,7 +641,7 @@ def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): *_replicate(0x03, 2), ) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): c.run(data) @@ -631,10 +650,11 @@ def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): def test_abi_decode_nonstrict_head_oob2(tx_failed, get_contract): # same principle as in Test_abi_decode_nonstrict_head_oob # but adapted for dynarrays - code = """ + buffer_size = 2 * 32 + 3 * 32 + 3 * 32 * 4 + code = f""" @external -def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): - y: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4] = x +def run(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x decoded_y1: DynArray[DynArray[uint256, 3], 3] = _abi_decode( y, DynArray[DynArray[uint256, 3], 3] @@ -642,7 +662,7 @@ def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): """ c = get_contract(code) - payload = ( + buffer_payload = ( 0x20, # DynArray head 0x03, # DynArray length (0x20 * 8 + 0x20 * 3 + 0x01), # inner array0 head @@ -658,7 +678,7 @@ def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): *_replicate(0x01, 2), # DynArray[..][2] data ) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): c.run(data) @@ -666,33 +686,36 @@ def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): def test_abi_decode_head_pointing_outside_buffer(tx_failed, get_contract): # the head points completely outside the buffer - code = """ + buffer_size = 3 * 32 + code = f""" @external -def run(x: Bytes[3 * 32]): - y: Bytes[3 * 32] = x +def run(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x decoded_y1: Bytes[32] = _abi_decode(y, Bytes[32]) """ c = get_contract(code) - payload = (0x80, 0x20, 0x01) - data = _abi_payload_from_tuple(payload) + buffer_payload = (0x80, 0x20, 0x01) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): c.run(data) def test_abi_decode_bytearray_clamp(tx_failed, get_contract): - # data has valid encoding, but the length of DynArray[Bytes[96], 3][0] is set to 0x61 + # data has valid encoding, but the length of DynArray[Bytes[96], 3][0] is + # set to 0x61 # and thus the decoding should fail on bytestring clamp - code = """ + buffer_size = 2 * 32 + 3 * 32 + 3 * 32 * 4 + code = f""" @external -def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): - y: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4] = x +def run(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x decoded_y1: DynArray[Bytes[32 * 3], 3] = _abi_decode(y, DynArray[Bytes[32 * 3], 3]) """ c = get_contract(code) - payload = ( + buffer_payload = ( 0x20, # DynArray head 0x03, # DynArray length 0x20 * 3, # inner array0 head @@ -707,32 +730,38 @@ def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): *_replicate(0x01, 3), # DynArray[Bytes[96], 3][2] data ) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): c.run(data) def test_abi_decode_runtimesz_oob(tx_failed, get_contract, env): - # provide enough data, but set the runtime size to be smaller than the actual size - # so after y: [..] = x, y will have the incorrect size set and only part of the - # original data will be copied. This will cause oob read outside the - # runtime sz (but still within static size of the buffer) - code = """ + # provide enough data, but set the runtime size to be smaller than the + # actual size so after y: [..] = x, y will have the incorrect size set and + # only part of the original data will be copied. This will cause oob read + # outside the runtime sz (but still within static size of the buffer) + buffer_size = 2 * 32 + 3 * 32 + 3 * 32 * 4 + code = f""" @external -def f(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): - y: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4] = x +def f(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x decoded_y1: DynArray[Bytes[32 * 3], 3] = _abi_decode(y, DynArray[Bytes[32 * 3], 3]) """ c = get_contract(code) - data = method_id("f(bytes)") - - payload = ( + msg_call_overhead = ( + method_id("f(bytes)"), 0x20, # tuple head # the correct size is 0x220 (2*32+3*32+4*3*32) - # therefore we will decode after the end of runtime size (but still within the buffer) + # therefore we will decode after the end of runtime size (but still + # within the buffer) 0x01E4, # top-level bytes array length + ) + + data = _abi_payload_from_tuple(msg_call_overhead, BUFFER_OVERHEAD) + + buffer_payload = ( 0x20, # DynArray head 0x03, # DynArray length 0x20 * 3, # inner array0 head @@ -746,7 +775,7 @@ def f(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): *_replicate(0x01, 3), # DynArray[Bytes[96], 3][2] data ) - data += _abi_payload_from_tuple(payload) + data += _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): env.message_call(c.address, data=data) @@ -755,10 +784,11 @@ def f(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): def test_abi_decode_runtimesz_oob2(tx_failed, get_contract, env): # same principle as in test_abi_decode_runtimesz_oob # but adapted for dynarrays - code = """ + buffer_size = 2 * 32 + 3 * 32 + 3 * 32 * 4 + code = f""" @external -def f(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): - y: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4] = x +def f(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x decoded_y1: DynArray[DynArray[uint256, 3], 3] = _abi_decode( y, DynArray[DynArray[uint256, 3], 3] @@ -766,11 +796,15 @@ def f(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): """ c = get_contract(code) - data = method_id("f(bytes)") - - payload = ( + msg_call_overhead = ( + method_id("f(bytes)"), 0x20, # tuple head 0x01E4, # top-level bytes array length + ) + + data = _abi_payload_from_tuple(msg_call_overhead, BUFFER_OVERHEAD) + + buffer_payload = ( 0x20, # DynArray head 0x03, # DynArray length 0x20 * 3, # inner array0 head @@ -784,7 +818,7 @@ def f(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): *_replicate(0x01, 3), # DynArray[..][2] data ) - data += _abi_payload_from_tuple(payload) + data += _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): env.message_call(c.address, data=data) @@ -796,11 +830,13 @@ def test_abi_decode_head_roundtrip(tx_failed, get_contract, env): # which are in turn in the y2 buffer # NOTE: the test is memory allocator dependent - we assume that y1 and y2 # have the 800 & 960 addresses respectively - code = """ + buffer_size1 = 4 * 32 + buffer_size2 = 2 * 32 + 3 * 32 + 3 * 32 * 4 + code = f""" @external -def run(x1: Bytes[4 * 32], x2: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): - y1: Bytes[4*32] = x1 # addr: 800 - y2: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4] = x2 # addr: 960 +def run(x1: Bytes[{buffer_size1}], x2: Bytes[{buffer_size2}]): + y1: Bytes[{buffer_size1}] = x1 # addr: 800 + y2: Bytes[{buffer_size2}] = x2 # addr: 960 decoded_y1: DynArray[DynArray[uint256, 3], 3] = _abi_decode( y2, DynArray[DynArray[uint256, 3], 3] @@ -808,7 +844,7 @@ def run(x1: Bytes[4 * 32], x2: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): """ c = get_contract(code) - payload = ( + buffer_payload = ( 0x03, # DynArray length # distance to y2 from y1 is 160 160 + 0x20 + 0x20 * 3, # points to DynArray[..][0] length @@ -816,9 +852,9 @@ def run(x1: Bytes[4 * 32], x2: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): 160 + 0x20 + 0x20 * 8 + 0x20 * 3, # points to DynArray[..][2] length ) - data1 = _abi_payload_from_tuple(payload) + data1 = _abi_payload_from_tuple(buffer_payload, buffer_size1) - payload = ( + buffer_payload = ( # (960 + (2**256 - 160)) % 2**256 == 800, ie will roundtrip to y1 2**256 - 160, # points to y1 0x03, # DynArray length (not used) @@ -833,7 +869,7 @@ def run(x1: Bytes[4 * 32], x2: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): *_replicate(0x03, 3), # DynArray[..][2] data ) - data2 = _abi_payload_from_tuple(payload) + data2 = _abi_payload_from_tuple(buffer_payload, buffer_size2) with tx_failed(): c.run(data1, data2) @@ -841,22 +877,23 @@ def run(x1: Bytes[4 * 32], x2: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): def test_abi_decode_merge_head_and_length(get_contract): # compress head and length into 33B - code = """ + buffer_size = 32 * 2 + 8 * 32 + code = f""" @external -def run(x: Bytes[32 * 2 + 8 * 32]) -> uint256: - y: Bytes[32 * 2 + 8 * 32] = x +def run(x: Bytes[{buffer_size}]) -> Bytes[{buffer_size}]: + y: Bytes[{buffer_size}] = x decoded_y1: Bytes[256] = _abi_decode(y, Bytes[256]) - return len(decoded_y1) + return decoded_y1 """ c = get_contract(code) - payload = (0x01, (0x00).to_bytes(1, "big"), *_replicate(0x00, 8)) + buffer_payload = (0x01, (0x00).to_bytes(1, "big"), *_replicate(0x00, 8)) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) - length = c.run(data) + res = c.run(data) - assert length == 256 + assert res == bytes(256) def test_abi_decode_extcall_invalid_head(tx_failed, get_contract): @@ -880,8 +917,8 @@ def foo(): def test_abi_decode_extcall_oob(tx_failed, get_contract): # the head returned from the extcall is 1 byte bigger than expected - # thus we'll take the last 31 0-bytes from tuple[1] and the 1st byte from tuple[2] - # and consider this the length - thus the length is 2**5 + # thus we'll take the last 31 0-bytes from tuple[1] and the 1st byte from + # tuple[2] and consider this the length - thus the length is 2**5 # and thus we'll read 1B over the buffer end (33 + 32 + 32) code = """ @external @@ -902,7 +939,8 @@ def foo(): def test_abi_decode_extcall_runtimesz_oob(tx_failed, get_contract): # the runtime size (33) is bigger than the actual payload (32 bytes) - # thus we'll read 1B over the runtime size - but still within the static size of the buffer + # thus we'll read 1 byte over the runtime size - but still within the + # static size of the buffer code = """ @external def bar() -> (uint256, uint256, uint256): @@ -932,11 +970,13 @@ def bar() -> (uint256, uint256, uint256, uint256): def bar() -> Bytes[32]: nonpayable @external -def foo(): - x:Bytes[32] = extcall A(self).bar() +def foo() -> Bytes[32]: + return extcall A(self).bar() """ c = get_contract(code) - c.foo() + res = c.foo() + + assert res == (36).to_bytes(32, "big") def test_abi_decode_extcall_truncate_returndata2(tx_failed, get_contract): @@ -1053,12 +1093,14 @@ def bar() -> (uint256, uint256): def bar() -> DynArray[Bytes[32], 2]: nonpayable @external -def run(): - x: DynArray[Bytes[32], 2] = extcall A(self).bar() +def run() -> DynArray[Bytes[32], 2]: + return extcall A(self).bar() """ c = get_contract(code) - c.run() + res = c.run() + + assert res == [] def test_abi_decode_extcall_complex_empty_dynarray(get_contract): @@ -1079,13 +1121,14 @@ def bar() -> (uint256, uint256, uint256, uint256, uint256, uint256): def bar() -> DynArray[Point, 2]: nonpayable @external -def run(): - x: DynArray[Point, 2] = extcall A(self).bar() - assert len(x) == 1 and len(x[0].y) == 0 +def run() -> DynArray[Point, 2]: + return extcall A(self).bar() """ c = get_contract(code) - c.run() + res = c.run() + + assert res == [(1, [], 0)] def test_abi_decode_extcall_complex_empty_dynarray2(tx_failed, get_contract): @@ -1124,21 +1167,21 @@ def bar() -> (uint256, uint256): def bar() -> DynArray[Bytes[32], 2]: nonpayable @external -def run() -> uint256: - x: DynArray[Bytes[32], 2] = extcall A(self).bar() - return len(x) +def run() -> DynArray[Bytes[32], 2]: + return extcall A(self).bar() """ c = get_contract(code) - length = c.run() + res = c.run() - assert length == 0 + assert res == [] def test_abi_decode_top_level_head_oob(tx_failed, get_contract): - code = """ + buffer_size = 256 + code = f""" @external -def run(x: Bytes[256], y: uint256): +def run(x: Bytes[{buffer_size}], y: uint256): player_lost: bool = empty(bool) if y == 1: @@ -1150,9 +1193,9 @@ def run(x: Bytes[256], y: uint256): c = get_contract(code) # head points over the buffer end - payload = (0x0100, *_replicate(0x00, 7)) + bufffer_payload = (0x0100, *_replicate(0x00, 7)) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(bufffer_payload, buffer_size) with tx_failed(): c.run(data, 1) @@ -1162,23 +1205,24 @@ def run(x: Bytes[256], y: uint256): def test_abi_decode_dynarray_complex_insufficient_data(env, tx_failed, get_contract): - code = """ + buffer_size = 32 * 8 + code = f""" struct Point: x: uint256 y: uint256 @external -def run(x: Bytes[32 * 8]): - y: Bytes[32 * 8] = x +def run(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x decoded_y1: DynArray[Point, 3] = _abi_decode(y, DynArray[Point, 3]) """ c = get_contract(code) # runtime buffer has insufficient size - we decode 3 points, but provide only # 3 * 32B of payload - payload = (0x20, 0x03, *_replicate(0x03, 3)) + buffer_payload = (0x20, 0x03, *_replicate(0x03, 3)) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): c.run(data) @@ -1187,7 +1231,8 @@ def run(x: Bytes[32 * 8]): def test_abi_decode_dynarray_complex2(env, tx_failed, get_contract): # point head to the 1st 0x01 word (ie the length) # but size of the point is 3 * 32B, thus we'd decode 2B over the buffer end - code = """ + buffer_size = 32 * 8 + code = f""" struct Point: x: uint256 y: uint256 @@ -1195,19 +1240,19 @@ def test_abi_decode_dynarray_complex2(env, tx_failed, get_contract): @external -def run(x: Bytes[32 * 8]): +def run(x: Bytes[{buffer_size}]): y: Bytes[32 * 11] = x decoded_y1: DynArray[Point, 2] = _abi_decode(y, DynArray[Point, 2]) """ c = get_contract(code) - payload = ( + buffer_payload = ( 0xC0, # points to the 1st 0x01 word (ie the length) *_replicate(0x03, 5), *_replicate(0x01, 2), ) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): c.run(data) @@ -1216,7 +1261,8 @@ def run(x: Bytes[32 * 8]): def test_abi_decode_complex_empty_dynarray(env, tx_failed, get_contract): # point head to the last word of the payload # this will be the length, but because it's set to 0, the decoding should succeed - code = """ + buffer_size = 32 * 16 + code = f""" struct Point: x: uint256 y: DynArray[uint256, 2] @@ -1224,14 +1270,13 @@ def test_abi_decode_complex_empty_dynarray(env, tx_failed, get_contract): @external -def run(x: Bytes[32 * 16]): - y: Bytes[32 * 16] = x - decoded_y1: DynArray[Point, 2] = _abi_decode(y, DynArray[Point, 2]) - assert len(decoded_y1) == 1 and len(decoded_y1[0].y) == 0 +def run(x: Bytes[{buffer_size}]) -> DynArray[Point, 2]: + y: Bytes[{buffer_size}] = x + return _abi_decode(y, DynArray[Point, 2]) """ c = get_contract(code) - payload = ( + buffer_payload = ( 0x20, 0x01, 0x20, @@ -1243,14 +1288,17 @@ def run(x: Bytes[32 * 16]): 0x00, # length is 0, so decoding should succeed ) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) + + res = c.run(data) - c.run(data) + assert res == [(1, [], 4)] def test_abi_decode_complex_arithmetic_overflow(tx_failed, get_contract): # inner head roundtrips due to arithmetic overflow - code = """ + buffer_size = 32 * 16 + code = f""" struct Point: x: uint256 y: DynArray[uint256, 2] @@ -1258,13 +1306,13 @@ def test_abi_decode_complex_arithmetic_overflow(tx_failed, get_contract): @external -def run(x: Bytes[32 * 16]): - y: Bytes[32 * 16] = x +def run(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x decoded_y1: DynArray[Point, 2] = _abi_decode(y, DynArray[Point, 2]) """ c = get_contract(code) - payload = ( + buffer_payload = ( 0x20, 0x01, 0x20, @@ -1276,39 +1324,43 @@ def run(x: Bytes[32 * 16]): 0x00, ) - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): c.run(data) def test_abi_decode_empty_toplevel_dynarray(get_contract): - code = """ + buffer_size = 2 * 32 + 3 * 32 + 3 * 32 * 4 + code = f""" @external -def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): - y: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4] = x +def run(x: Bytes[{buffer_size}]) -> DynArray[DynArray[uint256, 3], 3]: + y: Bytes[{buffer_size}] = x assert len(y) == 2 * 32 decoded_y1: DynArray[DynArray[uint256, 3], 3] = _abi_decode( y, DynArray[DynArray[uint256, 3], 3] ) - assert len(decoded_y1) == 0 + return decoded_y1 """ c = get_contract(code) - payload = (0x20, 0x00) # DynArray head, DynArray length + buffer_payload = (0x20, 0x00) # DynArray head, DynArray length + + data = _abi_payload_from_tuple(buffer_payload, buffer_size) - data = _abi_payload_from_tuple(payload) + res = c.run(data) - c.run(data) + assert res == [] def test_abi_decode_invalid_toplevel_dynarray_head(tx_failed, get_contract): # head points 1B over the bounds of the runtime buffer - code = """ + buffer_size = 2 * 32 + 3 * 32 + 3 * 32 * 4 + code = f""" @external -def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): - y: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4] = x +def run(x: Bytes[{buffer_size}]): + y: Bytes[{buffer_size}] = x decoded_y1: DynArray[DynArray[uint256, 3], 3] = _abi_decode( y, DynArray[DynArray[uint256, 3], 3] @@ -1317,33 +1369,34 @@ def run(x: Bytes[2 * 32 + 3 * 32 + 3 * 32 * 4]): c = get_contract(code) # head points 1B over the bounds of the runtime buffer - payload = (0x21, 0x00) # DynArray head, DynArray length + buffer_payload = (0x21, 0x00) # DynArray head, DynArray length - data = _abi_payload_from_tuple(payload) + data = _abi_payload_from_tuple(buffer_payload, buffer_size) with tx_failed(): c.run(data) def test_nested_invalid_dynarray_head(get_contract, tx_failed): - code = """ + buffer_size = 320 + code = f""" @nonpayable @external -def foo(x:Bytes[320]): +def foo(x:Bytes[{buffer_size}]): if True: a: Bytes[320-32] = b'' # make the word following the buffer x_mem dirty to make a potential # OOB revert fake_head: uint256 = 32 - x_mem: Bytes[320] = x + x_mem: Bytes[{buffer_size}] = x y: DynArray[DynArray[uint256, 2], 2] = _abi_decode(x_mem,DynArray[DynArray[uint256, 2], 2]) @nonpayable @external -def bar(x:Bytes[320]): - x_mem: Bytes[320] = x +def bar(x:Bytes[{buffer_size}]): + x_mem: Bytes[{buffer_size}] = x y:DynArray[DynArray[uint256, 2], 2] = _abi_decode(x_mem,DynArray[DynArray[uint256, 2], 2]) """ @@ -1355,7 +1408,7 @@ def bar(x:Bytes[320]): # 0x0, # head2 ) - encoded = _abi_payload_from_tuple(encoded + inner) + encoded = _abi_payload_from_tuple(encoded + inner, buffer_size) with tx_failed(): c.foo(encoded) # revert with tx_failed(): @@ -1363,22 +1416,23 @@ def bar(x:Bytes[320]): def test_static_outer_type_invalid_heads(get_contract, tx_failed): - code = """ + buffer_size = 320 + code = f""" @nonpayable @external -def foo(x:Bytes[320]): - x_mem: Bytes[320] = x +def foo(x:Bytes[{buffer_size}]): + x_mem: Bytes[{buffer_size}] = x y:DynArray[uint256, 2][2] = _abi_decode(x_mem,DynArray[uint256, 2][2]) @nonpayable @external -def bar(x:Bytes[320]): +def bar(x:Bytes[{buffer_size}]): if True: a: Bytes[160] = b'' # write stuff here to make the call revert in case decode do # an out of bound access: fake_head: uint256 = 32 - x_mem: Bytes[320] = x + x_mem: Bytes[{buffer_size}] = x y:DynArray[uint256, 2][2] = _abi_decode(x_mem,DynArray[uint256, 2][2]) """ c = get_contract(code) @@ -1389,7 +1443,7 @@ def bar(x:Bytes[320]): # 0x00, # head of the second dynarray ) - encoded = _abi_payload_from_tuple(encoded + inner) + encoded = _abi_payload_from_tuple(encoded + inner, buffer_size) with tx_failed(): c.foo(encoded) @@ -1402,9 +1456,10 @@ def test_abi_decode_max_size(get_contract, tx_failed): # of abi encoding the type. this can happen when the payload is # "sparse" and has garbage bytes in between the static and dynamic # sections - code = """ + buffer_size = 1000 + code = f""" @external -def foo(a:Bytes[1000]): +def foo(a:Bytes[{buffer_size}]): v: DynArray[uint256, 1] = _abi_decode(a,DynArray[uint256, 1]) """ c = get_contract(code) @@ -1420,7 +1475,7 @@ def foo(a:Bytes[1000]): ) with tx_failed(): - c.foo(_abi_payload_from_tuple(payload)) + c.foo(_abi_payload_from_tuple(payload, buffer_size)) # returndatasize check for uint256 From bd876b114bc34643a7d210b319f69642ce80f018 Mon Sep 17 00:00:00 2001 From: tserg <8017125+tserg@users.noreply.github.com> Date: Sun, 24 Nov 2024 04:34:57 +0800 Subject: [PATCH 25/70] fix[lang]: use folded node for typechecking (#4365) This commit addresses several issues in the frontend where valid code fails to compile because typechecking was performed on non-literal AST nodes, specifically in `slice()` and `raw_log()` builtins. This is fixed by using the folded node for typechecking instead. Additionally, folding is applied for the argument to `convert()`, which results in the typechecker being able to reject more invalid programs. --- .../functional/codegen/features/test_logging.py | 17 +++++++++++++++++ .../test_invalid_literal_exception.py | 8 ++++++++ .../exceptions/test_type_mismatch_exception.py | 8 ++++++++ tests/functional/syntax/test_slice.py | 16 ++++++++++++++++ vyper/builtins/_convert.py | 2 +- vyper/builtins/functions.py | 7 ++++--- vyper/semantics/analysis/local.py | 4 ++-- 7 files changed, 56 insertions(+), 6 deletions(-) diff --git a/tests/functional/codegen/features/test_logging.py b/tests/functional/codegen/features/test_logging.py index 2bb646e6ef..87d848fae5 100644 --- a/tests/functional/codegen/features/test_logging.py +++ b/tests/functional/codegen/features/test_logging.py @@ -1254,6 +1254,23 @@ def foo(): assert log.topics == [event_id, topic1, topic2, topic3] +valid_list = [ + # test constant folding inside raw_log + """ +topic: constant(bytes32) = 0x1212121212121210212801291212121212121210121212121212121212121212 + +@external +def foo(): + raw_log([[topic]][0], b'') + """ +] + + +@pytest.mark.parametrize("code", valid_list) +def test_raw_log_pass(code): + assert compile_code(code) is not None + + fail_list = [ ( """ diff --git a/tests/functional/syntax/exceptions/test_invalid_literal_exception.py b/tests/functional/syntax/exceptions/test_invalid_literal_exception.py index a0cf10ad02..f3fd73fbfc 100644 --- a/tests/functional/syntax/exceptions/test_invalid_literal_exception.py +++ b/tests/functional/syntax/exceptions/test_invalid_literal_exception.py @@ -36,6 +36,14 @@ def foo(): def foo(): a: bytes32 = keccak256("ѓtest") """, + # test constant folding inside of `convert()` + """ +BAR: constant(uint16) = 256 + +@external +def foo(): + a: uint8 = convert(BAR, uint8) + """, ] diff --git a/tests/functional/syntax/exceptions/test_type_mismatch_exception.py b/tests/functional/syntax/exceptions/test_type_mismatch_exception.py index 76c5c481f0..63e0eb6d11 100644 --- a/tests/functional/syntax/exceptions/test_type_mismatch_exception.py +++ b/tests/functional/syntax/exceptions/test_type_mismatch_exception.py @@ -47,6 +47,14 @@ def foo(): """ a: constant(address) = 0x3cd751e6b0078be393132286c442345e5dc49699 """, + # test constant folding inside `convert()` + """ +BAR: constant(Bytes[5]) = b"vyper" + +@external +def foo(): + a: Bytes[4] = convert(BAR, Bytes[4]) + """, ] diff --git a/tests/functional/syntax/test_slice.py b/tests/functional/syntax/test_slice.py index 6bb666527e..6a091c9da3 100644 --- a/tests/functional/syntax/test_slice.py +++ b/tests/functional/syntax/test_slice.py @@ -53,6 +53,22 @@ def foo(inp: Bytes[10]) -> Bytes[4]: def foo() -> Bytes[10]: return slice(b"badmintonzzz", 1, 10) """, + # test constant folding for `slice()` `length` argument + """ +@external +def foo(): + x: Bytes[32] = slice(msg.data, 0, 31 + 1) + """, + """ +@external +def foo(a: address): + x: Bytes[32] = slice(a.code, 0, 31 + 1) + """, + """ +@external +def foo(inp: Bytes[5], start: uint256) -> Bytes[3]: + return slice(inp, 0, 1 + 1) + """, ] diff --git a/vyper/builtins/_convert.py b/vyper/builtins/_convert.py index aa53dee429..a494e4a344 100644 --- a/vyper/builtins/_convert.py +++ b/vyper/builtins/_convert.py @@ -463,7 +463,7 @@ def to_flag(expr, arg, out_typ): def convert(expr, context): assert len(expr.args) == 2, "bad typecheck: convert" - arg_ast = expr.args[0] + arg_ast = expr.args[0].reduced() arg = Expr(arg_ast, context).ir_node original_arg = arg diff --git a/vyper/builtins/functions.py b/vyper/builtins/functions.py index 674efda7ce..9ed74b8cfe 100644 --- a/vyper/builtins/functions.py +++ b/vyper/builtins/functions.py @@ -305,7 +305,7 @@ def fetch_call_return(self, node): arg = node.args[0] start_expr = node.args[1] - length_expr = node.args[2] + length_expr = node.args[2].reduced() # CMC 2022-03-22 NOTE slight code duplication with semantics/analysis/local is_adhoc_slice = arg.get("attr") == "code" or ( @@ -1257,7 +1257,8 @@ def fetch_call_return(self, node): def infer_arg_types(self, node, expected_return_typ=None): self._validate_arg_types(node) - if not isinstance(node.args[0], vy_ast.List) or len(node.args[0].elements) > 4: + arg = node.args[0].reduced() + if not isinstance(arg, vy_ast.List) or len(arg.elements) > 4: raise InvalidType("Expecting a list of 0-4 topics as first argument", node.args[0]) # return a concrete type for `data` @@ -1269,7 +1270,7 @@ def infer_arg_types(self, node, expected_return_typ=None): def build_IR(self, expr, args, kwargs, context): context.check_is_not_constant(f"use {self._id}", expr) - topics_length = len(expr.args[0].elements) + topics_length = len(expr.args[0].reduced().elements) topics = args[0].args topics = [unwrap_location(topic) for topic in topics] diff --git a/vyper/semantics/analysis/local.py b/vyper/semantics/analysis/local.py index 809c6532c6..461326d72d 100644 --- a/vyper/semantics/analysis/local.py +++ b/vyper/semantics/analysis/local.py @@ -136,7 +136,7 @@ def _validate_address_code(node: vy_ast.Attribute, value_type: VyperType) -> Non parent = node.get_ancestor() if isinstance(parent, vy_ast.Call): ok_func = isinstance(parent.func, vy_ast.Name) and parent.func.id == "slice" - ok_args = len(parent.args) == 3 and isinstance(parent.args[2], vy_ast.Int) + ok_args = len(parent.args) == 3 and isinstance(parent.args[2].reduced(), vy_ast.Int) if ok_func and ok_args: return @@ -154,7 +154,7 @@ def _validate_msg_data_attribute(node: vy_ast.Attribute) -> None: "msg.data is only allowed inside of the slice, len or raw_call functions", node ) if parent.get("func.id") == "slice": - ok_args = len(parent.args) == 3 and isinstance(parent.args[2], vy_ast.Int) + ok_args = len(parent.args) == 3 and isinstance(parent.args[2].reduced(), vy_ast.Int) if not ok_args: raise StructureException( "slice(msg.data) must use a compile-time constant for length argument", parent From 8f433f8de9ec3ead39e1691c45e2821fe8e3922b Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 23 Nov 2024 21:39:41 +0100 Subject: [PATCH 26/70] refactor[tool]: refactor `compile_from_zip()` (#4366) refactor `compile_from_zip()`, and also a generalized `outputs_from_compiler_data()` so the user can pass a `CompilerData` instead of `FileInput` + a bunch of settings. --- vyper/cli/compile_archive.py | 12 ++++++++---- vyper/compiler/__init__.py | 20 ++++++++++++-------- vyper/compiler/phases.py | 4 ++++ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/vyper/cli/compile_archive.py b/vyper/cli/compile_archive.py index 1b52343c1c..c6d07de9f1 100644 --- a/vyper/cli/compile_archive.py +++ b/vyper/cli/compile_archive.py @@ -8,8 +8,9 @@ import zipfile from pathlib import PurePath -from vyper.compiler import compile_from_file_input +from vyper.compiler import outputs_from_compiler_data from vyper.compiler.input_bundle import FileInput, ZipInputBundle +from vyper.compiler.phases import CompilerData from vyper.compiler.settings import Settings, merge_settings from vyper.exceptions import BadArchive @@ -19,6 +20,11 @@ class NotZipInput(Exception): def compile_from_zip(file_name, output_formats, settings, no_bytecode_metadata): + compiler_data = compiler_data_from_zip(file_name, settings, no_bytecode_metadata) + return outputs_from_compiler_data(compiler_data, output_formats) + + +def compiler_data_from_zip(file_name, settings, no_bytecode_metadata): with open(file_name, "rb") as f: bcontents = f.read() @@ -59,11 +65,9 @@ def compile_from_zip(file_name, output_formats, settings, no_bytecode_metadata): settings, archive_settings, lhs_source="command line", rhs_source="archive settings" ) - # TODO: validate integrity sum (probably in CompilerData) - return compile_from_file_input( + return CompilerData( file, input_bundle=input_bundle, - output_formats=output_formats, integrity_sum=integrity, settings=settings, no_bytecode_metadata=no_bytecode_metadata, diff --git a/vyper/compiler/__init__.py b/vyper/compiler/__init__.py index 0345c24931..d885599cec 100644 --- a/vyper/compiler/__init__.py +++ b/vyper/compiler/__init__.py @@ -99,13 +99,6 @@ def compile_from_file_input( """ settings = settings or get_global_settings() or Settings() - if output_formats is None: - output_formats = ("bytecode",) - - # make IR output the same between runs - # TODO: move this to CompilerData.__init__() - codegen.reset_names() - compiler_data = CompilerData( file_input, input_bundle, @@ -116,6 +109,17 @@ def compile_from_file_input( no_bytecode_metadata=no_bytecode_metadata, ) + return outputs_from_compiler_data(compiler_data, output_formats, exc_handler) + + +def outputs_from_compiler_data( + compiler_data: CompilerData, + output_formats: Optional[OutputFormats] = None, + exc_handler: Optional[Callable] = None, +): + if output_formats is None: + output_formats = ("bytecode",) + ret = {} with anchor_settings(compiler_data.settings): for output_format in output_formats: @@ -126,7 +130,7 @@ def compile_from_file_input( ret[output_format] = formatter(compiler_data) except Exception as exc: if exc_handler is not None: - exc_handler(str(file_input.path), exc) + exc_handler(str(compiler_data.file_input.path), exc) else: raise exc diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index d9b6b13b48..503281a867 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -4,6 +4,7 @@ from pathlib import Path, PurePath from typing import Any, Optional +import vyper.codegen.core as codegen from vyper import ast as vy_ast from vyper.ast import natspec from vyper.codegen import module @@ -304,6 +305,9 @@ def generate_ir_nodes(global_ctx: ModuleT, settings: Settings) -> tuple[IRnode, IR to generate deployment bytecode IR to generate runtime bytecode """ + # make IR output the same between runs + codegen.reset_names() + with anchor_settings(settings): ir_nodes, ir_runtime = module.generate_ir_for_module(global_ctx) if settings.optimize != OptimizationLevel.NONE: From f249c9364a07044135e368bf846a0da1477d62e3 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 25 Nov 2024 18:08:45 +0100 Subject: [PATCH 27/70] feat[lang]: add `module.__at__()` to cast to interface (#4090) add `module.__at__`, a new `MemberFunctionT`, which allows the user to cast addresses to a module's interface. additionally, fix a bug where interfaces defined inline could not be exported. this is simultaneously fixed as a related bug because previously, interfaces could come up in export analysis as `InterfaceT` or `TYPE_T` depending on their provenance. this commit fixes the bug by making them `TYPE_T` in both imported and inlined provenance. this also allows `module.__interface__` to be used in export position by adding it to `ModuleT`'s members. note this has an unwanted side effect of allowing `module.__interface__` in call position; in other words, `module.__interface__(
)` has the same behavior as `module.__at__(
)` when use as an expression. this can be addressed in a later refactor. refactor: - wrap interfaces in `TYPE_T` - streamline an `isinstance(t, (VyperType, TYPE_T))` check. TYPE_T` now inherits from `VyperType`, so it doesn't need to be listed separately --------- Co-authored-by: cyberthirst --- docs/using-modules.rst | 15 +++ .../codegen/modules/test_exports.py | 23 ++++ .../codegen/modules/test_interface_imports.py | 36 +++++- tests/functional/codegen/test_interfaces.py | 89 +++++++++++++++ .../syntax/modules/test_deploy_visibility.py | 34 +++++- .../functional/syntax/modules/test_exports.py | 106 ++++++++++++++++++ tests/functional/syntax/test_interfaces.py | 50 +++++++++ vyper/codegen/expr.py | 24 ++-- vyper/compiler/output.py | 3 + vyper/semantics/analysis/base.py | 1 + vyper/semantics/analysis/module.py | 24 +++- vyper/semantics/analysis/utils.py | 2 +- vyper/semantics/types/__init__.py | 4 +- vyper/semantics/types/base.py | 10 +- vyper/semantics/types/function.py | 2 +- vyper/semantics/types/module.py | 36 ++++-- 16 files changed, 431 insertions(+), 28 deletions(-) diff --git a/docs/using-modules.rst b/docs/using-modules.rst index 7d63eb6617..4400a8dfa8 100644 --- a/docs/using-modules.rst +++ b/docs/using-modules.rst @@ -62,6 +62,21 @@ The ``_times_two()`` helper function in the above module can be immediately used The other functions cannot be used yet, because they touch the ``ownable`` module's state. There are two ways to declare a module so that its state can be used. +Using a module as an interface +============================== + +A module can be used as an interface with the ``__at__`` syntax. + +.. code-block:: vyper + + import ownable + + an_ownable: ownable.__interface__ + + def call_ownable(addr: address): + self.an_ownable = ownable.__at__(addr) + self.an_ownable.transfer_ownership(...) + Initializing a module ===================== diff --git a/tests/functional/codegen/modules/test_exports.py b/tests/functional/codegen/modules/test_exports.py index 93f4fe6c2f..3cc21d61a9 100644 --- a/tests/functional/codegen/modules/test_exports.py +++ b/tests/functional/codegen/modules/test_exports.py @@ -440,3 +440,26 @@ def __init__(): # call `c.__default__()` env.message_call(c.address) assert c.counter() == 6 + + +def test_inline_interface_export(make_input_bundle, get_contract): + lib1 = """ +interface IAsset: + def asset() -> address: view + +implements: IAsset + +@external +@view +def asset() -> address: + return self + """ + main = """ +import lib1 + +exports: lib1.IAsset + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + c = get_contract(main, input_bundle=input_bundle) + + assert c.asset() == c.address diff --git a/tests/functional/codegen/modules/test_interface_imports.py b/tests/functional/codegen/modules/test_interface_imports.py index 3f0f8cb010..af9f9b5e68 100644 --- a/tests/functional/codegen/modules/test_interface_imports.py +++ b/tests/functional/codegen/modules/test_interface_imports.py @@ -1,3 +1,6 @@ +import pytest + + def test_import_interface_types(make_input_bundle, get_contract): ifaces = """ interface IFoo: @@ -50,9 +53,10 @@ def foo() -> bool: # check that this typechecks both directions a: lib1.IERC20 = IERC20(msg.sender) b: lib2.IERC20 = IERC20(msg.sender) + c: IERC20 = lib1.IERC20(msg.sender) # allowed in call position # return the equality so we can sanity check it - return a == b + return a == b and b == c """ input_bundle = make_input_bundle({"lib1.vy": lib1, "lib2.vy": lib2}) c = get_contract(main, input_bundle=input_bundle) @@ -60,6 +64,36 @@ def foo() -> bool: assert c.foo() is True +@pytest.mark.parametrize("interface_syntax", ["__at__", "__interface__"]) +def test_intrinsic_interface(get_contract, make_input_bundle, interface_syntax): + lib = """ +@external +@view +def foo() -> uint256: + # detect self call + if msg.sender == self: + return 4 + else: + return 5 + """ + + main = f""" +import lib + +exports: lib.__interface__ + +@external +@view +def bar() -> uint256: + return staticcall lib.{interface_syntax}(self).foo() + """ + input_bundle = make_input_bundle({"lib.vy": lib}) + c = get_contract(main, input_bundle=input_bundle) + + assert c.foo() == 5 + assert c.bar() == 4 + + def test_import_interface_flags(make_input_bundle, get_contract): ifaces = """ flag Foo: diff --git a/tests/functional/codegen/test_interfaces.py b/tests/functional/codegen/test_interfaces.py index 8887bf07cb..31475a3bc0 100644 --- a/tests/functional/codegen/test_interfaces.py +++ b/tests/functional/codegen/test_interfaces.py @@ -774,3 +774,92 @@ def foo(s: MyStruct) -> MyStruct: assert "b: uint256" in out assert "struct Voter:" in out assert "voted: bool" in out + + +def test_intrinsic_interface_instantiation(make_input_bundle, get_contract): + lib1 = """ +@external +@view +def foo(): + pass + """ + main = """ +import lib1 + +i: lib1.__interface__ + +@external +def bar() -> lib1.__interface__: + self.i = lib1.__at__(self) + return self.i + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + c = get_contract(main, input_bundle=input_bundle) + + assert c.bar() == c.address + + +def test_intrinsic_interface_converts(make_input_bundle, get_contract): + lib1 = """ +@external +@view +def foo(): + pass + """ + main = """ +import lib1 + +@external +def bar() -> lib1.__interface__: + return lib1.__at__(self) + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + c = get_contract(main, input_bundle=input_bundle) + + assert c.bar() == c.address + + +def test_intrinsic_interface_kws(env, make_input_bundle, get_contract): + value = 10**5 + lib1 = f""" +@external +@payable +def foo(a: address): + send(a, {value}) + """ + main = f""" +import lib1 + +exports: lib1.__interface__ + +@external +def bar(a: address): + extcall lib1.__at__(self).foo(a, value={value}) + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + c = get_contract(main, input_bundle=input_bundle) + env.set_balance(c.address, value) + original_balance = env.get_balance(env.deployer) + c.bar(env.deployer) + assert env.get_balance(env.deployer) == original_balance + value + + +def test_intrinsic_interface_defaults(env, make_input_bundle, get_contract): + lib1 = """ +@external +@payable +def foo(i: uint256=1) -> uint256: + return i + """ + main = """ +import lib1 + +exports: lib1.__interface__ + +@external +def bar() -> uint256: + return extcall lib1.__at__(self).foo() + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + c = get_contract(main, input_bundle=input_bundle) + assert c.bar() == 1 diff --git a/tests/functional/syntax/modules/test_deploy_visibility.py b/tests/functional/syntax/modules/test_deploy_visibility.py index f51bf9575b..c908d4adae 100644 --- a/tests/functional/syntax/modules/test_deploy_visibility.py +++ b/tests/functional/syntax/modules/test_deploy_visibility.py @@ -1,7 +1,7 @@ import pytest from vyper.compiler import compile_code -from vyper.exceptions import CallViolation +from vyper.exceptions import CallViolation, UnknownAttribute def test_call_deploy_from_external(make_input_bundle): @@ -25,3 +25,35 @@ def foo(): compile_code(main, input_bundle=input_bundle) assert e.value.message == "Cannot call an @deploy function from an @external function!" + + +@pytest.mark.parametrize("interface_syntax", ["__interface__", "__at__"]) +def test_module_interface_init(make_input_bundle, tmp_path, interface_syntax): + lib1 = """ +#lib1.vy +k: uint256 + +@external +def bar(): + pass + +@deploy +def __init__(): + self.k = 10 + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + + code = f""" +import lib1 + +@deploy +def __init__(): + lib1.{interface_syntax}(self).__init__() + """ + + with pytest.raises(UnknownAttribute) as e: + compile_code(code, input_bundle=input_bundle) + + # as_posix() for windows tests + lib1_path = (tmp_path / "lib1.vy").as_posix() + assert e.value.message == f"interface {lib1_path} has no member '__init__'." diff --git a/tests/functional/syntax/modules/test_exports.py b/tests/functional/syntax/modules/test_exports.py index 7b00d29c98..4314c1bbf0 100644 --- a/tests/functional/syntax/modules/test_exports.py +++ b/tests/functional/syntax/modules/test_exports.py @@ -385,6 +385,28 @@ def do_xyz(): assert e.value._message == "requested `lib1.ifoo` but `lib1` does not implement `lib1.ifoo`!" +def test_no_export_unimplemented_inline_interface(make_input_bundle): + lib1 = """ +interface ifoo: + def do_xyz(): nonpayable + +# technically implements ifoo, but missing `implements: ifoo` + +@external +def do_xyz(): + pass + """ + main = """ +import lib1 + +exports: lib1.ifoo + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + with pytest.raises(InterfaceViolation) as e: + compile_code(main, input_bundle=input_bundle) + assert e.value._message == "requested `lib1.ifoo` but `lib1` does not implement `lib1.ifoo`!" + + def test_export_selector_conflict(make_input_bundle): ifoo = """ @external @@ -444,3 +466,87 @@ def __init__(): with pytest.raises(InterfaceViolation) as e: compile_code(main, input_bundle=input_bundle) assert e.value._message == "requested `lib1.ifoo` but `lib1` does not implement `lib1.ifoo`!" + + +def test_export_empty_interface(make_input_bundle, tmp_path): + lib1 = """ +def an_internal_function(): + pass + """ + main = """ +import lib1 + +exports: lib1.__interface__ + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + with pytest.raises(StructureException) as e: + compile_code(main, input_bundle=input_bundle) + + # as_posix() for windows + lib1_path = (tmp_path / "lib1.vy").as_posix() + assert e.value._message == f"lib1 (located at `{lib1_path}`) has no external functions!" + + +def test_invalid_export(make_input_bundle): + lib1 = """ +@external +def foo(): + pass + """ + main = """ +import lib1 +a: address + +exports: lib1.__interface__(self.a).foo + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + + with pytest.raises(StructureException) as e: + compile_code(main, input_bundle=input_bundle) + + assert e.value._message == "invalid export of a value" + assert e.value._hint == "exports should look like ." + + main = """ +interface Foo: + def foo(): nonpayable + +exports: Foo + """ + with pytest.raises(StructureException) as e: + compile_code(main) + + assert e.value._message == "invalid export" + assert e.value._hint == "exports should look like ." + + +@pytest.mark.parametrize("exports_item", ["__at__", "__at__(self)", "__at__(self).__interface__"]) +def test_invalid_at_exports(get_contract, make_input_bundle, exports_item): + lib = """ +@external +@view +def foo() -> uint256: + return 5 + """ + + main = f""" +import lib + +exports: lib.{exports_item} + +@external +@view +def bar() -> uint256: + return staticcall lib.__at__(self).foo() + """ + input_bundle = make_input_bundle({"lib.vy": lib}) + + with pytest.raises(Exception) as e: + compile_code(main, input_bundle=input_bundle) + + if exports_item == "__at__": + assert "not a function or interface" in str(e.value) + if exports_item == "__at__(self)": + assert "invalid exports" in str(e.value) + if exports_item == "__at__(self).__interface__": + assert "has no member '__interface__'" in str(e.value) diff --git a/tests/functional/syntax/test_interfaces.py b/tests/functional/syntax/test_interfaces.py index 86ea4bcfd0..baf0c73c30 100644 --- a/tests/functional/syntax/test_interfaces.py +++ b/tests/functional/syntax/test_interfaces.py @@ -571,3 +571,53 @@ def bar(): compiler.compile_code(code, input_bundle=input_bundle) assert e.value.message == "Contract does not implement all interface functions: bar(), foobar()" + + +def test_intrinsic_interfaces_different_types(make_input_bundle, get_contract): + lib1 = """ +@external +@view +def foo(): + pass + """ + lib2 = """ +@external +@view +def foo(): + pass + """ + main = """ +import lib1 +import lib2 + +@external +def bar(): + assert lib1.__at__(self) == lib2.__at__(self) + """ + input_bundle = make_input_bundle({"lib1.vy": lib1, "lib2.vy": lib2}) + + with pytest.raises(TypeMismatch): + compiler.compile_code(main, input_bundle=input_bundle) + + +@pytest.mark.xfail +def test_intrinsic_interfaces_default_function(make_input_bundle, get_contract): + lib1 = """ +@external +@payable +def __default__(): + pass + """ + main = """ +import lib1 + +@external +def bar(): + extcall lib1.__at__(self).__default__() + + """ + input_bundle = make_input_bundle({"lib1.vy": lib1}) + + # TODO make the exception more precise once fixed + with pytest.raises(Exception): # noqa: B017 + compiler.compile_code(main, input_bundle=input_bundle) diff --git a/vyper/codegen/expr.py b/vyper/codegen/expr.py index cd51966710..3a09bbe6c0 100644 --- a/vyper/codegen/expr.py +++ b/vyper/codegen/expr.py @@ -51,6 +51,7 @@ FlagT, HashMapT, InterfaceT, + ModuleT, SArrayT, StringT, StructT, @@ -680,7 +681,8 @@ def parse_Call(self): # TODO fix cyclic import from vyper.builtins._signatures import BuiltinFunctionT - func_t = self.expr.func._metadata["type"] + func = self.expr.func + func_t = func._metadata["type"] if isinstance(func_t, BuiltinFunctionT): return func_t.build_IR(self.expr, self.context) @@ -691,8 +693,14 @@ def parse_Call(self): return self.handle_struct_literal() # Interface constructor. Bar(
). - if is_type_t(func_t, InterfaceT): + if is_type_t(func_t, InterfaceT) or func.get("attr") == "__at__": assert not self.is_stmt # sanity check typechecker + + # magic: do sanity checks for module.__at__ + if func.get("attr") == "__at__": + assert isinstance(func_t, MemberFunctionT) + assert isinstance(func.value._metadata["type"], ModuleT) + (arg0,) = self.expr.args arg_ir = Expr(arg0, self.context).ir_node @@ -702,16 +710,16 @@ def parse_Call(self): return arg_ir if isinstance(func_t, MemberFunctionT): - darray = Expr(self.expr.func.value, self.context).ir_node + # TODO consider moving these to builtins or a dedicated file + darray = Expr(func.value, self.context).ir_node assert isinstance(darray.typ, DArrayT) args = [Expr(x, self.context).ir_node for x in self.expr.args] - if self.expr.func.attr == "pop": - # TODO consider moving this to builtins - darray = Expr(self.expr.func.value, self.context).ir_node + if func.attr == "pop": + darray = Expr(func.value, self.context).ir_node assert len(self.expr.args) == 0 return_item = not self.is_stmt return pop_dyn_array(darray, return_popped_item=return_item) - elif self.expr.func.attr == "append": + elif func.attr == "append": (arg,) = args check_assign( dummy_node_for_type(darray.typ.value_type), dummy_node_for_type(arg.typ) @@ -726,6 +734,8 @@ def parse_Call(self): ret.append(append_dyn_array(darray, arg)) return IRnode.from_list(ret) + raise CompilerPanic("unreachable!") # pragma: nocover + assert isinstance(func_t, ContractFunctionT) assert func_t.is_internal or func_t.is_constructor diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index f5f99a0bc3..e0eea293bc 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -268,6 +268,9 @@ def build_abi_output(compiler_data: CompilerData) -> list: _ = compiler_data.ir_runtime # ensure _ir_info is generated abi = module_t.interface.to_toplevel_abi_dict() + if module_t.init_function: + abi += module_t.init_function.to_toplevel_abi_dict() + if compiler_data.show_gas_estimates: # Add gas estimates for each function to ABI gas_estimates = build_gas_estimates(compiler_data.function_signatures) diff --git a/vyper/semantics/analysis/base.py b/vyper/semantics/analysis/base.py index e275930fa0..adfc7540a0 100644 --- a/vyper/semantics/analysis/base.py +++ b/vyper/semantics/analysis/base.py @@ -96,6 +96,7 @@ class AnalysisResult: class ModuleInfo(AnalysisResult): module_t: "ModuleT" alias: str + # import_node: vy_ast._ImportStmt # maybe could be useful ownership: ModuleOwnership = ModuleOwnership.NO_OWNERSHIP ownership_decl: Optional[vy_ast.VyperNode] = None diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index 8a2beb61e6..737f675b7c 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -40,7 +40,7 @@ ) from vyper.semantics.data_locations import DataLocation from vyper.semantics.namespace import Namespace, get_namespace, override_global_namespace -from vyper.semantics.types import EventT, FlagT, InterfaceT, StructT +from vyper.semantics.types import TYPE_T, EventT, FlagT, InterfaceT, StructT, is_type_t from vyper.semantics.types.function import ContractFunctionT from vyper.semantics.types.module import ModuleT from vyper.semantics.types.utils import type_from_annotation @@ -499,9 +499,19 @@ def visit_ExportsDecl(self, node): raise StructureException("not a public variable!", decl, item) funcs = [decl._expanded_getter._metadata["func_type"]] elif isinstance(info.typ, ContractFunctionT): + # e.g. lib1.__interface__(self._addr).foo + if not isinstance(get_expr_info(item.value).typ, (ModuleT, TYPE_T)): + raise StructureException( + "invalid export of a value", + item.value, + hint="exports should look like .", + ) + # regular function funcs = [info.typ] - elif isinstance(info.typ, InterfaceT): + elif is_type_t(info.typ, InterfaceT): + interface_t = info.typ.typedef + if not isinstance(item, vy_ast.Attribute): raise StructureException( "invalid export", @@ -512,7 +522,7 @@ def visit_ExportsDecl(self, node): if module_info is None: raise StructureException("not a valid module!", item.value) - if info.typ not in module_info.typ.implemented_interfaces: + if interface_t not in module_info.typ.implemented_interfaces: iface_str = item.node_source_code module_str = item.value.node_source_code msg = f"requested `{iface_str}` but `{module_str}`" @@ -523,9 +533,15 @@ def visit_ExportsDecl(self, node): # find the specific implementation of the function in the module funcs = [ module_exposed_fns[fn.name] - for fn in info.typ.functions.values() + for fn in interface_t.functions.values() if fn.is_external ] + + if len(funcs) == 0: + path = module_info.module_node.path + msg = f"{module_info.alias} (located at `{path}`) has no external functions!" + raise StructureException(msg, item) + else: raise StructureException( f"not a function or interface: `{info.typ}`", info.typ.decl_node, item diff --git a/vyper/semantics/analysis/utils.py b/vyper/semantics/analysis/utils.py index 9734087fc3..a31ce7acc1 100644 --- a/vyper/semantics/analysis/utils.py +++ b/vyper/semantics/analysis/utils.py @@ -199,7 +199,7 @@ def _raise_invalid_reference(name, node): try: s = t.get_member(name, node) - if isinstance(s, (VyperType, TYPE_T)): + if isinstance(s, VyperType): # ex. foo.bar(). bar() is a ContractFunctionT return [s] diff --git a/vyper/semantics/types/__init__.py b/vyper/semantics/types/__init__.py index 59a20dd99f..b881f52b2b 100644 --- a/vyper/semantics/types/__init__.py +++ b/vyper/semantics/types/__init__.py @@ -1,8 +1,8 @@ from . import primitives, subscriptable, user from .base import TYPE_T, VOID_TYPE, KwargSettings, VyperType, is_type_t, map_void from .bytestrings import BytesT, StringT, _BytestringT -from .function import MemberFunctionT -from .module import InterfaceT +from .function import ContractFunctionT, MemberFunctionT +from .module import InterfaceT, ModuleT from .primitives import AddressT, BoolT, BytesM_T, DecimalT, IntegerT, SelfT from .subscriptable import DArrayT, HashMapT, SArrayT, TupleT from .user import EventT, FlagT, StructT diff --git a/vyper/semantics/types/base.py b/vyper/semantics/types/base.py index 128ede0d5b..aca37b33a3 100644 --- a/vyper/semantics/types/base.py +++ b/vyper/semantics/types/base.py @@ -114,8 +114,13 @@ def __eq__(self, other): ) def __lt__(self, other): + # CMC 2024-10-20 what is this for? return self.abi_type.selector_name() < other.abi_type.selector_name() + def __repr__(self): + # TODO: add `pretty()` to the VyperType API? + return self._id + # return a dict suitable for serializing in the AST def to_dict(self): ret = {"name": self._id} @@ -362,10 +367,7 @@ def get_member(self, key: str, node: vy_ast.VyperNode) -> "VyperType": raise StructureException(f"{self} instance does not have members", node) hint = get_levenshtein_error_suggestions(key, self.members, 0.3) - raise UnknownAttribute(f"{self} has no member '{key}'.", node, hint=hint) - - def __repr__(self): - return self._id + raise UnknownAttribute(f"{repr(self)} has no member '{key}'.", node, hint=hint) class KwargSettings: diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index 7a56b01281..ffeb5b7299 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -874,7 +874,7 @@ def _id(self): return self.name def __repr__(self): - return f"{self.underlying_type._id} member function '{self.name}'" + return f"{self.underlying_type} member function '{self.name}'" def fetch_call_return(self, node: vy_ast.Call) -> Optional[VyperType]: validate_call_args(node, len(self.arg_types)) diff --git a/vyper/semantics/types/module.py b/vyper/semantics/types/module.py index dabeaf21b6..498757b94e 100644 --- a/vyper/semantics/types/module.py +++ b/vyper/semantics/types/module.py @@ -19,7 +19,7 @@ ) from vyper.semantics.data_locations import DataLocation from vyper.semantics.types.base import TYPE_T, VyperType, is_type_t -from vyper.semantics.types.function import ContractFunctionT +from vyper.semantics.types.function import ContractFunctionT, MemberFunctionT from vyper.semantics.types.primitives import AddressT from vyper.semantics.types.user import EventT, FlagT, StructT, _UserType from vyper.utils import OrderedSet @@ -240,9 +240,6 @@ def from_ModuleT(cls, module_t: "ModuleT") -> "InterfaceT": for fn_t in module_t.exposed_functions: funcs.append((fn_t.name, fn_t)) - if (fn_t := module_t.init_function) is not None: - funcs.append((fn_t.name, fn_t)) - event_set: OrderedSet[EventT] = OrderedSet() event_set.update([node._metadata["event_type"] for node in module_t.event_defs]) event_set.update(module_t.used_events) @@ -273,6 +270,19 @@ def from_InterfaceDef(cls, node: vy_ast.InterfaceDef) -> "InterfaceT": return cls._from_lists(node.name, node, functions) +def _module_at(module_t): + return MemberFunctionT( + # set underlying_type to a TYPE_T as a bit of a kludge, since it's + # kind of like a class method (but we don't have classmethod + # abstraction) + underlying_type=TYPE_T(module_t), + name="__at__", + arg_types=[AddressT()], + return_type=module_t.interface, + is_modifying=False, + ) + + # Datatype to store all module information. class ModuleT(VyperType): typeclass = "module" @@ -330,16 +340,28 @@ def __init__(self, module: vy_ast.Module, name: Optional[str] = None): for i in self.import_stmts: import_info = i._metadata["import_info"] - self.add_member(import_info.alias, import_info.typ) if hasattr(import_info.typ, "module_t"): - self._helper.add_member(import_info.alias, TYPE_T(import_info.typ)) + module_info = import_info.typ + # get_expr_info uses ModuleInfo + self.add_member(import_info.alias, module_info) + # type_from_annotation uses TYPE_T + self._helper.add_member(import_info.alias, TYPE_T(module_info.module_t)) + else: # interfaces + assert isinstance(import_info.typ, InterfaceT) + self.add_member(import_info.alias, TYPE_T(import_info.typ)) for name, interface_t in self.interfaces.items(): # can access interfaces in type position self._helper.add_member(name, TYPE_T(interface_t)) - self.add_member("__interface__", self.interface) + # module.__at__(addr) + self.add_member("__at__", _module_at(self)) + + # allow `module.__interface__` (in exports declarations) + self.add_member("__interface__", TYPE_T(self.interface)) + # allow `module.__interface__` (in type position) + self._helper.add_member("__interface__", TYPE_T(self.interface)) # __eq__ is very strict on ModuleT - object equality! this is because we # don't want to reason about where a module came from (i.e. input bundle, From cda634dd3a1a20db20d3565f47aa1cf37ede8b9c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 26 Nov 2024 12:38:43 +0100 Subject: [PATCH 28/70] fix[venom]: fix duplicate allocas (#4321) this commit fixes a bug in the ir_node_to_venom translator. previously, `ir_node_to_venom` tried to detect unique allocas based on heuristics. this commit removes the heuristics and fixes the issue in the frontend by passing through a unique ID for each variable in the metadata. this ID is also passed into the `alloca` and `palloca` instructions to aid with debugging. note that this results in improved code, presumably due to more allocas being able to be reified. this commit makes a minor change to the `sqrt()`, builtin, which is to use `z_var.as_ir_node()` instead of `z_var.pos`, since `.as_ir_node()` correctly tags with the alloca metadata. to be maximally conservative, we could branch, only using `z_var.as_ir_node()` if we are using the venom pipeline, but the change should be correct for the legacy pipeline as well anyways. --------- Co-authored-by: Harry Kalogirou --- vyper/builtins/functions.py | 5 ++- vyper/codegen/context.py | 17 ++++++++- vyper/codegen/core.py | 4 +++ vyper/venom/README.md | 5 +-- vyper/venom/__init__.py | 3 ++ vyper/venom/ir_node_to_venom.py | 53 +++++++++++++---------------- vyper/venom/passes/__init__.py | 1 + vyper/venom/passes/float_allocas.py | 36 ++++++++++++++++++++ vyper/venom/passes/sccp/sccp.py | 2 +- 9 files changed, 89 insertions(+), 37 deletions(-) create mode 100644 vyper/venom/passes/float_allocas.py diff --git a/vyper/builtins/functions.py b/vyper/builtins/functions.py index 9ed74b8cfe..0cfcb636d7 100644 --- a/vyper/builtins/functions.py +++ b/vyper/builtins/functions.py @@ -2167,10 +2167,9 @@ def build_IR(self, expr, args, kwargs, context): variables_2=variables_2, memory_allocator=context.memory_allocator, ) + z_ir = new_ctx.vars["z"].as_ir_node() ret = IRnode.from_list( - ["seq", placeholder_copy, sqrt_ir, new_ctx.vars["z"].pos], # load x variable - typ=DecimalT(), - location=MEMORY, + ["seq", placeholder_copy, sqrt_ir, z_ir], typ=DecimalT(), location=MEMORY ) return b1.resolve(ret) diff --git a/vyper/codegen/context.py b/vyper/codegen/context.py index f49914ac78..7995b7b9f5 100644 --- a/vyper/codegen/context.py +++ b/vyper/codegen/context.py @@ -15,6 +15,17 @@ class Constancy(enum.Enum): Constant = 1 +_alloca_id = 0 + + +def _generate_alloca_id(): + # note: this gets reset between compiler runs by codegen.core.reset_names + global _alloca_id + + _alloca_id += 1 + return _alloca_id + + @dataclass(frozen=True) class Alloca: name: str @@ -22,6 +33,8 @@ class Alloca: typ: VyperType size: int + _id: int + def __post_init__(self): assert self.typ.memory_bytes_required == self.size @@ -233,7 +246,9 @@ def _new_variable( pos = f"$palloca_{ofst}_{size}" else: pos = f"$alloca_{ofst}_{size}" - alloca = Alloca(name=name, offset=ofst, typ=typ, size=size) + + alloca_id = _generate_alloca_id() + alloca = Alloca(name=name, offset=ofst, typ=typ, size=size, _id=alloca_id) var = VariableRecord( name=name, diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 2bd4f81f50..0ad7fa79c6 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -1,3 +1,4 @@ +import vyper.codegen.context as ctx from vyper.codegen.ir_node import Encoding, IRnode from vyper.compiler.settings import _opt_codesize, _opt_gas, _opt_none from vyper.evm.address_space import ( @@ -855,6 +856,9 @@ def reset_names(): global _label _label = 0 + # could be refactored + ctx._alloca_id = 0 + # returns True if t is ABI encoded and is a type that needs any kind of # validation diff --git a/vyper/venom/README.md b/vyper/venom/README.md index 6f3b318c9b..ea6eabebaa 100644 --- a/vyper/venom/README.md +++ b/vyper/venom/README.md @@ -209,15 +209,16 @@ Assembly can be inspected with `-f asm`, whereas an opcode view of the final byt - Effectively translates to `JUMP`, and marks the call site as a valid return destination (for callee to jump back to) by `JUMPDEST`. - `alloca` - ``` - out = alloca size, offset + out = alloca size, offset, id ``` - Allocates memory of a given `size` at a given `offset` in memory. + - The `id` argument is there to help debugging translation into venom - The output is the offset value itself. - Because the SSA form does not allow changing values of registers, handling mutable variables can be tricky. The `alloca` instruction is meant to simplify that. - `palloca` - ``` - out = palloca size, offset + out = palloca size, offset, id ``` - Like the `alloca` instruction but only used for parameters of internal functions which are passed by memory. - `iload` diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index bf3115b4dd..593a9556a9 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -14,6 +14,7 @@ AlgebraicOptimizationPass, BranchOptimizationPass, DFTPass, + FloatAllocas, MakeSSA, Mem2Var, RemoveUnusedVariablesPass, @@ -47,6 +48,8 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: ac = IRAnalysesCache(fn) + FloatAllocas(ac, fn).run_pass() + SimplifyCFGPass(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() Mem2Var(ac, fn).run_pass() diff --git a/vyper/venom/ir_node_to_venom.py b/vyper/venom/ir_node_to_venom.py index 02a9f4d1f7..782309d841 100644 --- a/vyper/venom/ir_node_to_venom.py +++ b/vyper/venom/ir_node_to_venom.py @@ -107,18 +107,16 @@ NOOP_INSTRUCTIONS = frozenset(["pass", "cleanup_repeat", "var_list", "unique_symbol"]) SymbolTable = dict[str, Optional[IROperand]] -_global_symbols: SymbolTable = None # type: ignore +_alloca_table: SymbolTable = None # type: ignore MAIN_ENTRY_LABEL_NAME = "__main_entry" -_external_functions: dict[int, SymbolTable] = None # type: ignore # convert IRnode directly to venom def ir_node_to_venom(ir: IRnode) -> IRContext: _ = ir.unique_symbols # run unique symbols check - global _global_symbols, _external_functions - _global_symbols = {} - _external_functions = {} + global _alloca_table + _alloca_table = {} ctx = IRContext() fn = ctx.create_function(MAIN_ENTRY_LABEL_NAME) @@ -233,7 +231,7 @@ def pop_source(*args, **kwargs): def _convert_ir_bb(fn, ir, symbols): assert isinstance(ir, IRnode), ir # TODO: refactor these to not be globals - global _break_target, _continue_target, _global_symbols, _external_functions + global _break_target, _continue_target, _alloca_table # keep a map from external functions to all possible entry points @@ -269,8 +267,8 @@ def _convert_ir_bb(fn, ir, symbols): if is_internal or len(re.findall(r"external.*__init__\(.*_deploy", current_func)) > 0: # Internal definition var_list = ir.args[0].args[1] + assert var_list.value == "var_list" does_return_data = IRnode.from_list(["return_buffer"]) in var_list.args - _global_symbols = {} symbols = {} new_fn = _handle_internal_func(fn, ir, does_return_data, symbols) for ir_node in ir.args[1:]: @@ -298,8 +296,6 @@ def _convert_ir_bb(fn, ir, symbols): cont_ret = _convert_ir_bb(fn, cond, symbols) cond_block = fn.get_basic_block() - saved_global_symbols = _global_symbols.copy() - then_block = IRBasicBlock(ctx.get_next_label("then"), fn) else_block = IRBasicBlock(ctx.get_next_label("else"), fn) @@ -314,7 +310,6 @@ def _convert_ir_bb(fn, ir, symbols): # convert "else" cond_symbols = symbols.copy() - _global_symbols = saved_global_symbols.copy() fn.append_basic_block(else_block) else_ret_val = None if len(ir.args) == 3: @@ -343,8 +338,6 @@ def _convert_ir_bb(fn, ir, symbols): if not then_block_finish.is_terminated: then_block_finish.append_instruction("jmp", exit_bb.label) - _global_symbols = saved_global_symbols - return if_ret elif ir.value == "with": @@ -385,13 +378,6 @@ def _convert_ir_bb(fn, ir, symbols): data = _convert_ir_bb(fn, c, symbols) ctx.append_data("db", [data]) # type: ignore elif ir.value == "label": - function_id_pattern = r"external (\d+)" - function_name = ir.args[0].value - m = re.match(function_id_pattern, function_name) - if m is not None: - function_id = m.group(1) - _global_symbols = _external_functions.setdefault(function_id, {}) - label = IRLabel(ir.args[0].value, True) bb = fn.get_basic_block() if not bb.is_terminated: @@ -463,13 +449,11 @@ def _convert_ir_bb(fn, ir, symbols): elif ir.value == "repeat": def emit_body_blocks(): - global _break_target, _continue_target, _global_symbols + global _break_target, _continue_target old_targets = _break_target, _continue_target _break_target, _continue_target = exit_block, incr_block - saved_global_symbols = _global_symbols.copy() _convert_ir_bb(fn, body, symbols.copy()) _break_target, _continue_target = old_targets - _global_symbols = saved_global_symbols sym = ir.args[0] start, end, _ = _convert_ir_bb_list(fn, ir.args[1:4], symbols) @@ -540,16 +524,25 @@ def emit_body_blocks(): elif isinstance(ir.value, str) and ir.value.upper() in get_opcodes(): _convert_ir_opcode(fn, ir, symbols) elif isinstance(ir.value, str): - if ir.value.startswith("$alloca") and ir.value not in _global_symbols: + if ir.value.startswith("$alloca"): alloca = ir.passthrough_metadata["alloca"] - ptr = fn.get_basic_block().append_instruction("alloca", alloca.offset, alloca.size) - _global_symbols[ir.value] = ptr - elif ir.value.startswith("$palloca") and ir.value not in _global_symbols: + if alloca._id not in _alloca_table: + ptr = fn.get_basic_block().append_instruction( + "alloca", alloca.offset, alloca.size, alloca._id + ) + _alloca_table[alloca._id] = ptr + return _alloca_table[alloca._id] + + elif ir.value.startswith("$palloca"): alloca = ir.passthrough_metadata["alloca"] - ptr = fn.get_basic_block().append_instruction("palloca", alloca.offset, alloca.size) - _global_symbols[ir.value] = ptr - - return _global_symbols.get(ir.value) or symbols.get(ir.value) + if alloca._id not in _alloca_table: + ptr = fn.get_basic_block().append_instruction( + "palloca", alloca.offset, alloca.size, alloca._id + ) + _alloca_table[alloca._id] = ptr + return _alloca_table[alloca._id] + + return symbols.get(ir.value) elif ir.is_literal: return IRLiteral(ir.value) else: diff --git a/vyper/venom/passes/__init__.py b/vyper/venom/passes/__init__.py index 83098234c1..fcd2aa1f22 100644 --- a/vyper/venom/passes/__init__.py +++ b/vyper/venom/passes/__init__.py @@ -1,6 +1,7 @@ from .algebraic_optimization import AlgebraicOptimizationPass from .branch_optimization import BranchOptimizationPass from .dft import DFTPass +from .float_allocas import FloatAllocas from .make_ssa import MakeSSA from .mem2var import Mem2Var from .normalization import NormalizationPass diff --git a/vyper/venom/passes/float_allocas.py b/vyper/venom/passes/float_allocas.py new file mode 100644 index 0000000000..81fa115645 --- /dev/null +++ b/vyper/venom/passes/float_allocas.py @@ -0,0 +1,36 @@ +from vyper.venom.passes.base_pass import IRPass + + +class FloatAllocas(IRPass): + """ + This pass moves allocas to the entry basic block of a function + We could probably move them to the immediate dominator of the basic + block defining the alloca instead of the entry (which dominates all + basic blocks), but this is done for expedience. + Without this step, sccp fails, possibly because dominators are not + guaranteed to be traversed first. + """ + + def run_pass(self): + entry_bb = self.function.entry + assert entry_bb.is_terminated + tmp = entry_bb.instructions.pop() + + for bb in self.function.get_basic_blocks(): + if bb is entry_bb: + continue + + # Extract alloca instructions + non_alloca_instructions = [] + for inst in bb.instructions: + if inst.opcode in ("alloca", "palloca"): + # note: order of allocas impacts bytecode. + # TODO: investigate. + entry_bb.insert_instruction(inst) + else: + non_alloca_instructions.append(inst) + + # Replace original instructions with filtered list + bb.instructions = non_alloca_instructions + + entry_bb.instructions.append(tmp) diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index 2be84ce502..369be3e753 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -252,7 +252,7 @@ def finalize(ret): if eval_result is LatticeEnum.BOTTOM: return finalize(LatticeEnum.BOTTOM) - assert isinstance(eval_result, IROperand) + assert isinstance(eval_result, IROperand), (inst.parent.label, op, inst) ops.append(eval_result) # If we haven't found BOTTOM yet, evaluate the operation From e98e004235961613c3d769d4c652884b2a242608 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 26 Nov 2024 12:39:48 +0100 Subject: [PATCH 29/70] fix[venom]: add missing extcodesize+hash effects (#4373) per title -- effects.py was missing extcodesize and extcodehash effects. --- vyper/venom/effects.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vyper/venom/effects.py b/vyper/venom/effects.py index a668ff5439..97cffe2cb2 100644 --- a/vyper/venom/effects.py +++ b/vyper/venom/effects.py @@ -68,6 +68,8 @@ def __iter__(self): "balance": BALANCE, "selfbalance": BALANCE, "extcodecopy": EXTCODE, + "extcodesize": EXTCODE, + "extcodehash": EXTCODE, "selfdestruct": BALANCE, # may modify code, but after the transaction "log": MEMORY, "revert": MEMORY, From 794022e5e9651f762e460c24cb5a44e2b3e7d6f1 Mon Sep 17 00:00:00 2001 From: sudo rm -rf --no-preserve-root / Date: Tue, 3 Dec 2024 23:23:06 +0100 Subject: [PATCH 30/70] chore[ci]: enable Python `3.13` tests (#4386) python 3.13 was released on oct 7 2024 https://docs.python.org/3/whatsnew/3.13.html --- .github/workflows/test.yml | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d491e2530..aa2155db20 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -116,6 +116,7 @@ jobs: # modes across all python versions - one is enough - python-version: ["3.10", "310"] - python-version: ["3.12", "312"] + - python-version: ["3.13", "313"] # os-specific rules - os: windows diff --git a/setup.py b/setup.py index 5d6bd1db3a..69a08b737c 100644 --- a/setup.py +++ b/setup.py @@ -113,6 +113,7 @@ def _global_version(version): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], package_data={"vyper.ast": ["grammar.lark"]}, data_files=[("", [hash_file_rel_path])], From c8691ac5dd95623991e51205bc90a720fc513766 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 5 Dec 2024 18:12:08 -0800 Subject: [PATCH 31/70] fix[ux]: fix validation for `abi_encode()` `method_id` kwarg (#4369) code like in the test case (`abi_encode(..., method_id=b"123")`) will panic due to the assertion in `_parse_method_id()`, since the original typecheck only checked that the method_id is <=4 bytes. this adds a rule to validate that bytestring method ids are exactly 4 bytes directly in `abi_encode()`'s typecheck routine. --------- Co-authored-by: cyberthirst --- tests/functional/syntax/test_abi_encode.py | 33 +++++++++++++++++++++- vyper/builtins/functions.py | 8 +++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/functional/syntax/test_abi_encode.py b/tests/functional/syntax/test_abi_encode.py index 5e0175857d..edb441652a 100644 --- a/tests/functional/syntax/test_abi_encode.py +++ b/tests/functional/syntax/test_abi_encode.py @@ -1,7 +1,7 @@ import pytest from vyper import compiler -from vyper.exceptions import TypeMismatch +from vyper.exceptions import InvalidLiteral, TypeMismatch fail_list = [ ( @@ -41,11 +41,37 @@ def foo(x: uint256) -> Bytes[36]: ( """ @external +def foo(x: uint256) -> Bytes[36]: + return _abi_encode(x, method_id=b"abc") + """, + InvalidLiteral, # len(method_id) must be greater than 3 + ), + ( + """ +@external def foo(x: uint256) -> Bytes[36]: return _abi_encode(x, method_id=0x1234567890) """, TypeMismatch, # len(method_id) must be less than 4 ), + ( + """ +@external +def foo(x: uint256) -> Bytes[36]: + return _abi_encode(x, method_id=0x123456) + """, + TypeMismatch, # len(method_id) must be greater than 3 + ), + ( + """ +@external +def foo() -> Bytes[132]: + x: uint256 = 1 + y: Bytes[32] = b"234" + return abi_encode(x, y, method_id=b"") + """, + InvalidLiteral, # len(method_id) must be 4 + ), ] @@ -82,6 +108,11 @@ def foo(x: Bytes[1]) -> Bytes[68]: return _abi_encode(x, ensure_tuple=False, method_id=0x12345678) """, """ +@external +def foo(x: Bytes[1]) -> Bytes[68]: + return _abi_encode(x, ensure_tuple=False, method_id=b"1234") + """, + """ BAR: constant(DynArray[uint256, 5]) = [1, 2, 3, 4, 5] @external diff --git a/vyper/builtins/functions.py b/vyper/builtins/functions.py index 0cfcb636d7..62539872bc 100644 --- a/vyper/builtins/functions.py +++ b/vyper/builtins/functions.py @@ -2363,7 +2363,13 @@ def infer_kwarg_types(self, node): for kwarg in node.keywords: kwarg_name = kwarg.arg validate_expected_type(kwarg.value, self._kwargs[kwarg_name].typ) - ret[kwarg_name] = get_exact_type_from_node(kwarg.value) + + typ = get_exact_type_from_node(kwarg.value) + if kwarg_name == "method_id" and isinstance(typ, BytesT): + if typ.length != 4: + raise InvalidLiteral("method_id must be exactly 4 bytes!", kwarg.value) + + ret[kwarg_name] = typ return ret def fetch_call_return(self, node): From 12ab4919cc4618fcac4f5d24d45a0e7fdbc4a48c Mon Sep 17 00:00:00 2001 From: tserg <8017125+tserg@users.noreply.github.com> Date: Tue, 10 Dec 2024 07:06:23 +0800 Subject: [PATCH 32/70] fix[ux]: fix false positive for overflow in type checker (#4385) this commit fixes a false positive for integer overflow in the typechecker involving nested pow operations by filtering `OverflowException` in `_validate_op`. the previous code assumed that `validate_numeric_op` could throw anything besides `InvalidOperation`, but for the `Pow` binop, it can throw `OverflowException`. --------- Co-authored-by: Charles Cooper --- .../codegen/types/numbers/test_exponents.py | 14 ++++++++++++++ vyper/semantics/analysis/utils.py | 2 +- vyper/semantics/types/primitives.py | 4 ++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/functional/codegen/types/numbers/test_exponents.py b/tests/functional/codegen/types/numbers/test_exponents.py index 702cbcb1dd..28dba59edc 100644 --- a/tests/functional/codegen/types/numbers/test_exponents.py +++ b/tests/functional/codegen/types/numbers/test_exponents.py @@ -173,3 +173,17 @@ def foo(b: int128) -> int128: c.foo(max_power) with tx_failed(): c.foo(max_power + 1) + + +valid_list = [ + """ +@external +def foo() -> uint256: + return (10**18)**2 + """ +] + + +@pytest.mark.parametrize("good_code", valid_list) +def test_exponent_success(good_code): + assert compile_code(good_code) is not None diff --git a/vyper/semantics/analysis/utils.py b/vyper/semantics/analysis/utils.py index a31ce7acc1..8727f3750d 100644 --- a/vyper/semantics/analysis/utils.py +++ b/vyper/semantics/analysis/utils.py @@ -41,7 +41,7 @@ def _validate_op(node, types_list, validation_fn_name): try: _validate_fn(node) ret.append(type_) - except InvalidOperation as e: + except (InvalidOperation, OverflowException) as e: err_list.append(e) if ret: diff --git a/vyper/semantics/types/primitives.py b/vyper/semantics/types/primitives.py index 5c0362e662..dcc4fe8c8e 100644 --- a/vyper/semantics/types/primitives.py +++ b/vyper/semantics/types/primitives.py @@ -173,11 +173,11 @@ def _get_lr(): if isinstance(left, vy_ast.Int): if left.value >= 2**value_bits: raise OverflowException( - "Base is too large, calculation will always overflow", left + f"Base is too large for {self}, calculation will always overflow", left ) elif left.value < -(2**value_bits): raise OverflowException( - "Base is too small, calculation will always underflow", left + f"Base is too small for {self}, calculation will always underflow", left ) elif isinstance(right, vy_ast.Int): if right.value < 0: From c4669d164fe81281c933140b88053edc1ae2435e Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:37:52 +0100 Subject: [PATCH 33/70] fix[tool]: add missing user errors to error map (#4286) This commit fixes a bug where not all errors are present in the error_map (in the `-f source_map` output). This could happen when the `IRnode` with an error message was optimized out, since the error message was not propagated to its descendants. To resolve this, two changes were made. In `set_error_msg`, the code now ensures that existing error messages are not overwritten if they have already been set. In `IRnode.from_list`, when an `error_msg` argument is provided, it is assigned to all nodes in the list unless a node already has an existing `error_msg`. Both changes were made, since both methods are used in the codebase to set `IRnode.error_msg`. --- tests/unit/compiler/test_source_map.py | 40 ++++++++++++++++++++++++-- vyper/codegen/ir_node.py | 15 ++++++---- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/tests/unit/compiler/test_source_map.py b/tests/unit/compiler/test_source_map.py index d99b546403..ae1999a26e 100644 --- a/tests/unit/compiler/test_source_map.py +++ b/tests/unit/compiler/test_source_map.py @@ -97,8 +97,44 @@ def update_foo(): self.foo += 1 """ error_map = compile_code(code, output_formats=["source_map"])["source_map"]["error_map"] - assert "safeadd" in list(error_map.values()) - assert "fallback function" in list(error_map.values()) + assert "safeadd" in error_map.values() + assert "fallback function" in error_map.values() + + +def test_error_map_with_user_error(): + code = """ +@external +def foo(): + raise "some error" + """ + error_map = compile_code(code, output_formats=["source_map"])["source_map"]["error_map"] + assert "user revert with reason" in error_map.values() + + +def test_error_map_with_user_error2(): + code = """ +@external +def foo(i: uint256): + a: DynArray[uint256, 10] = [1] + a[i % 10] = 2 + """ + error_map = compile_code(code, output_formats=["source_map"])["source_map"]["error_map"] + assert "safemod" in error_map.values() + + +def test_error_map_not_overriding_errors(): + code = """ +@external +def foo(i: uint256): + raise self.bar(5%i) + +@pure +def bar(i: uint256) -> String[32]: + return "foo foo" + """ + error_map = compile_code(code, output_formats=["source_map"])["source_map"]["error_map"] + assert "user revert with reason" in error_map.values() + assert "safemod" in error_map.values() def test_compress_source_map(): diff --git a/vyper/codegen/ir_node.py b/vyper/codegen/ir_node.py index ff721fafcb..81ec47f10f 100644 --- a/vyper/codegen/ir_node.py +++ b/vyper/codegen/ir_node.py @@ -378,13 +378,18 @@ def is_complex_ir(self): and self.value.lower() not in do_not_cache ) - # set an error message and push down into all children. - # useful for overriding an error message generated by a helper - # function with a more specific error message. + # set an error message and push down to its children that don't have error_msg set def set_error_msg(self, error_msg: str) -> None: + if self.error_msg is not None: + raise CompilerPanic(f"{self.value} already has error message {self.error_msg}") + self._set_error_msg(error_msg) + + def _set_error_msg(self, error_msg: str) -> None: + if self.error_msg is not None: + return self.error_msg = error_msg for arg in self.args: - arg.set_error_msg(error_msg) + arg._set_error_msg(error_msg) # get the unique symbols contained in this node, which provides # sanity check invariants for the optimizer. @@ -627,7 +632,7 @@ def from_list( else: return cls( obj[0], - [cls.from_list(o, ast_source=ast_source) for o in obj[1:]], + [cls.from_list(o, ast_source=ast_source, error_msg=error_msg) for o in obj[1:]], typ, location=location, annotation=annotation, From c951dea2c461e48894d5bb55966e3856d719de71 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:50:44 +0100 Subject: [PATCH 34/70] fix[ux]: add missing filename to syntax exceptions (#4343) this commit adds a filename to `SyntaxException`s. previously, they did not include filename information because that is typically added from the `Module` AST node fields. but, at the time a `SyntaxException` is thrown, the AST is not yet available, so the normal handler does not have the filename info. this commit adds the filename info in two places where the path is known, one in natspec.py and one in parse.py. --- .../exceptions/test_vyper_exception_pos.py | 32 ++++++++++++++++++- tests/unit/ast/test_natspec.py | 16 ++++++++++ tests/unit/ast/test_pre_parser.py | 20 ++++++++++++ .../unit/cli/vyper_json/test_compile_json.py | 2 +- vyper/ast/natspec.py | 8 +++++ vyper/ast/parse.py | 23 ++++++++++++- vyper/exceptions.py | 15 +++++++-- 7 files changed, 111 insertions(+), 5 deletions(-) diff --git a/tests/functional/syntax/exceptions/test_vyper_exception_pos.py b/tests/functional/syntax/exceptions/test_vyper_exception_pos.py index 9e0767cb83..17bd4de1cd 100644 --- a/tests/functional/syntax/exceptions/test_vyper_exception_pos.py +++ b/tests/functional/syntax/exceptions/test_vyper_exception_pos.py @@ -1,6 +1,7 @@ from pytest import raises -from vyper.exceptions import VyperException +from vyper import compile_code +from vyper.exceptions import SyntaxException, VyperException def test_type_exception_pos(): @@ -29,3 +30,32 @@ def __init__(): """ assert_compile_failed(lambda: get_contract(code), VyperException) + + +def test_exception_contains_file(make_input_bundle): + code = """ +def bar()>: + """ + input_bundle = make_input_bundle({"code.vy": code}) + with raises(SyntaxException, match="contract"): + compile_code(code, input_bundle=input_bundle) + + +def test_exception_reports_correct_file(make_input_bundle, chdir_tmp_path): + code_a = "def bar()>:" + code_b = "import A" + input_bundle = make_input_bundle({"A.vy": code_a, "B.vy": code_b}) + + with raises(SyntaxException, match=r'contract "A\.vy:\d+"'): + compile_code(code_b, input_bundle=input_bundle) + + +def test_syntax_exception_reports_correct_offset(make_input_bundle): + code = """ +def foo(): + uint256 a = pass + """ + input_bundle = make_input_bundle({"code.vy": code}) + + with raises(SyntaxException, match=r"line \d+:12"): + compile_code(code, input_bundle=input_bundle) diff --git a/tests/unit/ast/test_natspec.py b/tests/unit/ast/test_natspec.py index 710b7a9312..37120d2978 100644 --- a/tests/unit/ast/test_natspec.py +++ b/tests/unit/ast/test_natspec.py @@ -436,3 +436,19 @@ def test_natspec_parsed_implicitly(): # anything beyond ast is blocked with pytest.raises(NatSpecSyntaxException): compile_code(code, output_formats=["annotated_ast_dict"]) + + +def test_natspec_exception_contains_file_path(): + code = """ +@external +def foo() -> (int128,uint256): + ''' + @return int128 + @return uint256 + @return this should fail + ''' + return 1, 2 + """ + + with pytest.raises(NatSpecSyntaxException, match=r'contract "VyperContract\.vy:\d+"'): + parse_natspec(code) diff --git a/tests/unit/ast/test_pre_parser.py b/tests/unit/ast/test_pre_parser.py index 73712aadb8..510d1e0ed2 100644 --- a/tests/unit/ast/test_pre_parser.py +++ b/tests/unit/ast/test_pre_parser.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from vyper import compile_code @@ -56,6 +58,24 @@ def test_invalid_version_pragma(file_version, mock_version): validate_version_pragma(f"{file_version}", file_version, (SRC_LINE)) +def test_invalid_version_contains_file(mock_version): + mock_version(COMPILER_VERSION) + with pytest.raises(VersionException, match=r'contract "mock\.vy:\d+"'): + compile_code("# pragma version ^0.3.10", resolved_path=Path("mock.vy")) + + +def test_imported_invalid_version_contains_correct_file( + mock_version, make_input_bundle, chdir_tmp_path +): + code_a = "# pragma version ^0.3.10" + code_b = "import A" + input_bundle = make_input_bundle({"A.vy": code_a, "B.vy": code_b}) + mock_version(COMPILER_VERSION) + + with pytest.raises(VersionException, match=r'contract "A\.vy:\d+"'): + compile_code(code_b, input_bundle=input_bundle) + + prerelease_valid_versions = [ "<0.1.1-beta.9", "<0.1.1b9", diff --git a/tests/unit/cli/vyper_json/test_compile_json.py b/tests/unit/cli/vyper_json/test_compile_json.py index 7802ee7955..8cd9be128f 100644 --- a/tests/unit/cli/vyper_json/test_compile_json.py +++ b/tests/unit/cli/vyper_json/test_compile_json.py @@ -238,7 +238,7 @@ def test_wrong_language(): def test_exc_handler_raises_syntax(input_json): input_json["sources"]["badcode.vy"] = {"content": BAD_SYNTAX_CODE} - with pytest.raises(SyntaxException): + with pytest.raises(SyntaxException, match=r'contract "badcode\.vy:\d+"'): compile_json(input_json) diff --git a/vyper/ast/natspec.py b/vyper/ast/natspec.py index 48fc9134dd..f65a361338 100644 --- a/vyper/ast/natspec.py +++ b/vyper/ast/natspec.py @@ -19,6 +19,14 @@ class NatspecOutput: def parse_natspec(annotated_vyper_module: vy_ast.Module) -> NatspecOutput: + try: + return _parse_natspec(annotated_vyper_module) + except NatSpecSyntaxException as e: + e.resolved_path = annotated_vyper_module.resolved_path + raise e + + +def _parse_natspec(annotated_vyper_module: vy_ast.Module) -> NatspecOutput: """ Parses NatSpec documentation from a contract. diff --git a/vyper/ast/parse.py b/vyper/ast/parse.py index 5d62072b9e..d975aafac4 100644 --- a/vyper/ast/parse.py +++ b/vyper/ast/parse.py @@ -23,6 +23,22 @@ def parse_to_ast_with_settings( module_path: Optional[str] = None, resolved_path: Optional[str] = None, add_fn_node: Optional[str] = None, +) -> tuple[Settings, vy_ast.Module]: + try: + return _parse_to_ast_with_settings( + vyper_source, source_id, module_path, resolved_path, add_fn_node + ) + except SyntaxException as e: + e.resolved_path = resolved_path + raise e + + +def _parse_to_ast_with_settings( + vyper_source: str, + source_id: int = 0, + module_path: Optional[str] = None, + resolved_path: Optional[str] = None, + add_fn_node: Optional[str] = None, ) -> tuple[Settings, vy_ast.Module]: """ Parses a Vyper source string and generates basic Vyper AST nodes. @@ -60,7 +76,12 @@ def parse_to_ast_with_settings( py_ast = python_ast.parse(pre_parser.reformatted_code) except SyntaxError as e: # TODO: Ensure 1-to-1 match of source_code:reformatted_code SyntaxErrors - raise SyntaxException(str(e), vyper_source, e.lineno, e.offset) from None + offset = e.offset + if offset is not None: + # SyntaxError offset is 1-based, not 0-based (see: + # https://docs.python.org/3/library/exceptions.html#SyntaxError.offset) + offset -= 1 + raise SyntaxException(str(e.msg), vyper_source, e.lineno, offset) from None # Add dummy function node to ensure local variables are treated as `AnnAssign` # instead of state variables (`VariableDecl`) diff --git a/vyper/exceptions.py b/vyper/exceptions.py index c69163b561..990dbf7953 100644 --- a/vyper/exceptions.py +++ b/vyper/exceptions.py @@ -54,6 +54,7 @@ def __init__(self, message="Error Message not found.", *items, hint=None, prev_d self.lineno = None self.col_offset = None self.annotations = None + self.resolved_path = None if len(items) == 1 and isinstance(items[0], tuple) and isinstance(items[0][0], int): # support older exceptions that don't annotate - remove this in the future! @@ -127,13 +128,18 @@ def format_annotation(self, value): module_node = node.module_node # TODO: handle cases where module is None or vy_ast.Module - if module_node.get("path") not in (None, ""): - node_msg = f'{node_msg}contract "{module_node.path}:{node.lineno}", ' + if module_node.get("resolved_path") not in (None, ""): + node_msg = self._format_contract_details( + node_msg, module_node.resolved_path, node.lineno + ) fn_node = node.get_ancestor(vy_ast.FunctionDef) if fn_node: node_msg = f'{node_msg}function "{fn_node.name}", ' + elif self.resolved_path is not None: + node_msg = self._format_contract_details(node_msg, self.resolved_path, node.lineno) + col_offset_str = "" if node.col_offset is None else str(node.col_offset) node_msg = f"{node_msg}line {node.lineno}:{col_offset_str} \n{source_annotation}\n" @@ -151,6 +157,11 @@ def _add_hint(self, msg): return msg return msg + f"\n (hint: {self.hint})" + def _format_contract_details(self, msg, path, lineno): + from vyper.utils import safe_relpath + + return f'{msg}contract "{safe_relpath(path)}:{lineno}", ' + def __str__(self): return self._add_hint(self._str_helper()) From 537313b0dd47b3c086fa46d1ef7d8282101fa128 Mon Sep 17 00:00:00 2001 From: Philippe Dumonet Date: Sat, 14 Dec 2024 19:29:57 +0100 Subject: [PATCH 35/70] feat[venom]: add venom parser (#4381) this commit adds a frontend (parser) for venom. it uses the lark library to define a grammar for venom, and constructs an `IRContext` which can be used to emit bytecode. the entry point to the venom compiler is `vyper/cli/venom_main.py`. possible improvements in the future include: - make data section optional in the grammar - make the entry block optional in the grammar (a la llvm) - add asm and opcodes output formats --------- Co-authored-by: Harry Kalogirou Co-authored-by: Charles Cooper --- setup.py | 2 + tests/functional/venom/__init__.py | 0 tests/functional/venom/parser/__init__.py | 0 tests/functional/venom/parser/test_parsing.py | 275 ++++++++++++++++++ vyper/cli/venom_main.py | 65 +++++ vyper/venom/README.md | 78 ++--- vyper/venom/__init__.py | 8 +- vyper/venom/analysis/cfg.py | 2 +- vyper/venom/basicblock.py | 2 +- vyper/venom/function.py | 2 - vyper/venom/parser.py | 178 ++++++++++++ vyper/venom/passes/sccp/sccp.py | 2 +- 12 files changed, 560 insertions(+), 54 deletions(-) create mode 100644 tests/functional/venom/__init__.py create mode 100644 tests/functional/venom/parser/__init__.py create mode 100644 tests/functional/venom/parser/test_parsing.py create mode 100755 vyper/cli/venom_main.py create mode 100644 vyper/venom/parser.py diff --git a/setup.py b/setup.py index 69a08b737c..5b1ae1b81a 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ def _global_version(version): "asttokens>=2.0.5,<3", "pycryptodome>=3.5.1,<4", "packaging>=23.1,<24", + "lark>=1.0.0,<2", "importlib-metadata", "wheel", ], @@ -105,6 +106,7 @@ def _global_version(version): "vyper=vyper.cli.vyper_compile:_parse_cli_args", "fang=vyper.cli.vyper_ir:_parse_cli_args", "vyper-json=vyper.cli.vyper_json:_parse_cli_args", + "venom=vyper.cli.venom_main:_parse_cli_args", ] }, classifiers=[ diff --git a/tests/functional/venom/__init__.py b/tests/functional/venom/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/functional/venom/parser/__init__.py b/tests/functional/venom/parser/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/functional/venom/parser/test_parsing.py b/tests/functional/venom/parser/test_parsing.py new file mode 100644 index 0000000000..c121edb692 --- /dev/null +++ b/tests/functional/venom/parser/test_parsing.py @@ -0,0 +1,275 @@ +from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRLabel, IRLiteral, IRVariable +from vyper.venom.context import IRContext +from vyper.venom.function import IRFunction +from vyper.venom.parser import parse_venom + +# TODO: Refactor tests with these helpers + + +def instructions_eq(i1: IRInstruction, i2: IRInstruction) -> bool: + return i1.output == i2.output and i1.opcode == i2.opcode and i1.operands == i2.operands + + +def assert_bb_eq(bb1: IRBasicBlock, bb2: IRBasicBlock): + assert bb1.label.value == bb2.label.value + assert len(bb1.instructions) == len(bb2.instructions) + for i1, i2 in zip(bb1.instructions, bb2.instructions): + assert instructions_eq(i1, i2), f"[{i1}] != [{i2}]" + + +def assert_fn_eq(fn1: IRFunction, fn2: IRFunction): + assert fn1.name.value == fn2.name.value + assert fn1.last_variable == fn2.last_variable + assert len(fn1._basic_block_dict) == len(fn2._basic_block_dict) + + for name1, bb1 in fn1._basic_block_dict.items(): + assert name1 in fn2._basic_block_dict + assert_bb_eq(bb1, fn2._basic_block_dict[name1]) + + # check function entry is the same + assert fn1.entry.label == fn2.entry.label + + +def assert_ctx_eq(ctx1: IRContext, ctx2: IRContext): + assert ctx1.last_label == ctx2.last_label + assert len(ctx1.functions) == len(ctx2.functions) + for label1, fn1 in ctx1.functions.items(): + assert label1 in ctx2.functions + assert_fn_eq(fn1, ctx2.functions[label1]) + + # check entry function is the same + assert next(iter(ctx1.functions.keys())) == next(iter(ctx2.functions.keys())) + + assert len(ctx1.data_segment) == len(ctx2.data_segment) + for d1, d2 in zip(ctx1.data_segment, ctx2.data_segment): + assert instructions_eq(d1, d2), f"data: [{d1}] != [{d2}]" + + +def test_single_bb(): + source = """ + function main { + main: + stop + } + + [data] + """ + + parsed_ctx = parse_venom(source) + + expected_ctx = IRContext() + expected_ctx.add_function(main_fn := IRFunction(IRLabel("main"))) + main_bb = main_fn.get_basic_block("main") + main_bb.append_instruction("stop") + + assert_ctx_eq(parsed_ctx, expected_ctx) + + +def test_multi_bb_single_fn(): + source = """ + function start { + start: + %1 = callvalue + jnz @fine, @has_callvalue, %1 + fine: + %2 = calldataload 4 + %4 = add %2, 279387 + return %2, %4 + has_callvalue: + revert 0, 0 + } + + [data] + """ + + parsed_ctx = parse_venom(source) + + expected_ctx = IRContext() + expected_ctx.add_function(start_fn := IRFunction(IRLabel("start"))) + + start_bb = start_fn.get_basic_block("start") + start_bb.append_instruction("callvalue", ret=IRVariable("1")) + start_bb.append_instruction("jnz", IRVariable("1"), IRLabel("has_callvalue"), IRLabel("fine")) + + start_fn.append_basic_block(fine_bb := IRBasicBlock(IRLabel("fine"), start_fn)) + fine_bb.append_instruction("calldataload", IRLiteral(4), ret=IRVariable("2")) + fine_bb.append_instruction("add", IRLiteral(279387), IRVariable("2"), ret=IRVariable("4")) + fine_bb.append_instruction("return", IRVariable("4"), IRVariable("2")) + + has_callvalue_bb = IRBasicBlock(IRLabel("has_callvalue"), start_fn) + start_fn.append_basic_block(has_callvalue_bb) + has_callvalue_bb.append_instruction("revert", IRLiteral(0), IRLiteral(0)) + has_callvalue_bb.append_instruction("stop") + + start_fn.last_variable = 4 + + assert_ctx_eq(parsed_ctx, expected_ctx) + + +def test_data_section(): + parsed_ctx = parse_venom( + """ + function entry { + entry: + stop + } + + [data] + dbname @selector_buckets + db @selector_bucket_0 + db @fallback + db @selector_bucket_2 + db @selector_bucket_3 + db @fallback + db @selector_bucket_5 + db @selector_bucket_6 + """ + ) + + expected_ctx = IRContext() + expected_ctx.add_function(entry_fn := IRFunction(IRLabel("entry"))) + entry_fn.get_basic_block("entry").append_instruction("stop") + + expected_ctx.data_segment = [ + IRInstruction("dbname", [IRLabel("selector_buckets")]), + IRInstruction("db", [IRLabel("selector_bucket_0")]), + IRInstruction("db", [IRLabel("fallback")]), + IRInstruction("db", [IRLabel("selector_bucket_2")]), + IRInstruction("db", [IRLabel("selector_bucket_3")]), + IRInstruction("db", [IRLabel("fallback")]), + IRInstruction("db", [IRLabel("selector_bucket_5")]), + IRInstruction("db", [IRLabel("selector_bucket_6")]), + ] + + assert_ctx_eq(parsed_ctx, expected_ctx) + + +def test_multi_function(): + parsed_ctx = parse_venom( + """ + function entry { + entry: + invoke @check_cv + jmp @wow + wow: + mstore 0, 1 + return 0, 32 + } + + function check_cv { + check_cv: + %1 = callvalue + %2 = param + jnz @no_value, @has_value, %1 + no_value: + ret %2 + has_value: + revert 0, 0 + } + + [data] + """ + ) + + expected_ctx = IRContext() + expected_ctx.add_function(entry_fn := IRFunction(IRLabel("entry"))) + + entry_bb = entry_fn.get_basic_block("entry") + entry_bb.append_instruction("invoke", IRLabel("check_cv")) + entry_bb.append_instruction("jmp", IRLabel("wow")) + + entry_fn.append_basic_block(wow_bb := IRBasicBlock(IRLabel("wow"), entry_fn)) + wow_bb.append_instruction("mstore", IRLiteral(1), IRLiteral(0)) + wow_bb.append_instruction("return", IRLiteral(32), IRLiteral(0)) + + expected_ctx.add_function(check_fn := IRFunction(IRLabel("check_cv"))) + + check_entry_bb = check_fn.get_basic_block("check_cv") + check_entry_bb.append_instruction("callvalue", ret=IRVariable("1")) + check_entry_bb.append_instruction("param", ret=IRVariable("2")) + check_entry_bb.append_instruction( + "jnz", IRVariable("1"), IRLabel("has_value"), IRLabel("no_value") + ) + check_fn.append_basic_block(no_value_bb := IRBasicBlock(IRLabel("no_value"), check_fn)) + no_value_bb.append_instruction("ret", IRVariable("2")) + + check_fn.append_basic_block(value_bb := IRBasicBlock(IRLabel("has_value"), check_fn)) + value_bb.append_instruction("revert", IRLiteral(0), IRLiteral(0)) + value_bb.append_instruction("stop") + + check_fn.last_variable = 2 + + assert_ctx_eq(parsed_ctx, expected_ctx) + + +def test_multi_function_and_data(): + parsed_ctx = parse_venom( + """ + function entry { + entry: + invoke @check_cv + jmp @wow + wow: + mstore 0, 1 + return 0, 32 + } + + function check_cv { + check_cv: + %1 = callvalue + %2 = param + jnz @no_value, @has_value, %1 + no_value: + ret %2 + has_value: + revert 0, 0 + } + + [data] + dbname @selector_buckets + db @selector_bucket_0 + db @fallback + db @selector_bucket_2 + db @selector_bucket_3 + db @selector_bucket_6 + """ + ) + + expected_ctx = IRContext() + expected_ctx.add_function(entry_fn := IRFunction(IRLabel("entry"))) + + entry_bb = entry_fn.get_basic_block("entry") + entry_bb.append_instruction("invoke", IRLabel("check_cv")) + entry_bb.append_instruction("jmp", IRLabel("wow")) + + entry_fn.append_basic_block(wow_bb := IRBasicBlock(IRLabel("wow"), entry_fn)) + wow_bb.append_instruction("mstore", IRLiteral(1), IRLiteral(0)) + wow_bb.append_instruction("return", IRLiteral(32), IRLiteral(0)) + + expected_ctx.add_function(check_fn := IRFunction(IRLabel("check_cv"))) + + check_entry_bb = check_fn.get_basic_block("check_cv") + check_entry_bb.append_instruction("callvalue", ret=IRVariable("1")) + check_entry_bb.append_instruction("param", ret=IRVariable("2")) + check_entry_bb.append_instruction( + "jnz", IRVariable("1"), IRLabel("has_value"), IRLabel("no_value") + ) + check_fn.append_basic_block(no_value_bb := IRBasicBlock(IRLabel("no_value"), check_fn)) + no_value_bb.append_instruction("ret", IRVariable("2")) + + check_fn.append_basic_block(value_bb := IRBasicBlock(IRLabel("has_value"), check_fn)) + value_bb.append_instruction("revert", IRLiteral(0), IRLiteral(0)) + value_bb.append_instruction("stop") + + check_fn.last_variable = 2 + + expected_ctx.data_segment = [ + IRInstruction("dbname", [IRLabel("selector_buckets")]), + IRInstruction("db", [IRLabel("selector_bucket_0")]), + IRInstruction("db", [IRLabel("fallback")]), + IRInstruction("db", [IRLabel("selector_bucket_2")]), + IRInstruction("db", [IRLabel("selector_bucket_3")]), + IRInstruction("db", [IRLabel("selector_bucket_6")]), + ] + + assert_ctx_eq(parsed_ctx, expected_ctx) diff --git a/vyper/cli/venom_main.py b/vyper/cli/venom_main.py new file mode 100755 index 0000000000..3114246e04 --- /dev/null +++ b/vyper/cli/venom_main.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import argparse +import sys + +import vyper +import vyper.evm.opcodes as evm +from vyper.compiler.phases import generate_bytecode +from vyper.compiler.settings import OptimizationLevel, Settings, set_global_settings +from vyper.venom import generate_assembly_experimental, run_passes_on +from vyper.venom.parser import parse_venom + +""" +Standalone entry point into venom compiler. Parses venom input and emits +bytecode. +""" + + +def _parse_cli_args(): + return _parse_args(sys.argv[1:]) + + +def _parse_args(argv: list[str]): + parser = argparse.ArgumentParser( + description="Venom EVM IR parser & compiler", formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument("input_file", help="Venom sourcefile", nargs="?") + parser.add_argument("--version", action="version", version=vyper.__long_version__) + parser.add_argument( + "--evm-version", + help=f"Select desired EVM version (default {evm.DEFAULT_EVM_VERSION})", + choices=list(evm.EVM_VERSIONS), + dest="evm_version", + ) + parser.add_argument( + "--stdin", action="store_true", help="whether to pull venom input from stdin" + ) + + args = parser.parse_args(argv) + + if args.evm_version is not None: + set_global_settings(Settings(evm_version=args.evm_version)) + + if args.stdin: + if not sys.stdin.isatty(): + venom_source = sys.stdin.read() + else: + # No input provided + print("Error: --stdin flag used but no input provided") + sys.exit(1) + else: + if args.input_file is None: + print("Error: No input file provided, either use --stdin or provide a path") + sys.exit(1) + with open(args.input_file, "r") as f: + venom_source = f.read() + + ctx = parse_venom(venom_source) + run_passes_on(ctx, OptimizationLevel.default()) + asm = generate_assembly_experimental(ctx) + bytecode = generate_bytecode(asm, compiler_metadata=None) + print(f"0x{bytecode.hex()}") + + +if __name__ == "__main__": + _parse_args(sys.argv[1:]) diff --git a/vyper/venom/README.md b/vyper/venom/README.md index ea6eabebaa..964f52b524 100644 --- a/vyper/venom/README.md +++ b/vyper/venom/README.md @@ -29,59 +29,43 @@ Venom employs two scopes: global and function level. ### Example code ```llvm -IRFunction: global - -global: - %1 = calldataload 0 - %2 = shr 224, %1 - jmp label %selector_bucket_0 - -selector_bucket_0: - %3 = xor %2, 1579456981 - %4 = iszero %3 - jnz label %1, label %2, %4 - -1: IN=[selector_bucket_0] OUT=[9] - jmp label %fallback - -2: - %5 = callvalue - %6 = calldatasize - %7 = lt %6, 164 - %8 = or %5, %7 - %9 = iszero %8 - assert %9 - stop - -fallback: - revert 0, 0 +function global { + global: + %1 = calldataload 0 + %2 = shr 224, %1 + jmp @selector_bucket_0 + + selector_bucket_0: + %3 = xor %2, 1579456981 + %4 = iszero %3 + jnz @1, @2, %4 + + 1: + jmp @fallback + + 2: + %5 = callvalue + %6 = calldatasize + %7 = lt %6, 164 + %8 = or %5, %7 + %9 = iszero %8 + assert %9 + stop + + fallback: + revert 0, 0 +} + +[data] ``` ### Grammar -Below is a (not-so-complete) grammar to describe the text format of Venom IR: +To see a definition of grammar see the [venom parser](./parser.py) -```llvm -program ::= function_declaration* - -function_declaration ::= "IRFunction:" identifier input_list? output_list? "=>" block - -input_list ::= "IN=" "[" (identifier ("," identifier)*)? "]" -output_list ::= "OUT=" "[" (identifier ("," identifier)*)? "]" - -block ::= label ":" input_list? output_list? "=>{" operation* "}" +### Compiling Venom -operation ::= "%" identifier "=" opcode operand ("," operand)* - | opcode operand ("," operand)* - -opcode ::= "calldataload" | "shr" | "shl" | "and" | "add" | "codecopy" | "mload" | "jmp" | "xor" | "iszero" | "jnz" | "label" | "lt" | "or" | "assert" | "callvalue" | "calldatasize" | "alloca" | "calldatacopy" | "invoke" | "gt" | ... - -operand ::= "%" identifier | label | integer | "label" "%" identifier -label ::= "%" identifier - -identifier ::= [a-zA-Z_][a-zA-Z0-9_]* -integer ::= [0-9]+ -``` +Vyper ships with a venom compiler which compiles venom code to bytecode directly. It can be run by running `venom`, which is installed as a standalone binary when `vyper` is installed via `pip`. ## Implementation diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index 593a9556a9..7d9404b9ef 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -72,10 +72,14 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: DFTPass(ac, fn).run_pass() +def run_passes_on(ctx: IRContext, optimize: OptimizationLevel): + for fn in ctx.functions.values(): + _run_passes(fn, optimize) + + def generate_ir(ir: IRnode, optimize: OptimizationLevel) -> IRContext: # Convert "old" IR to "new" IR ctx = ir_node_to_venom(ir) - for fn in ctx.functions.values(): - _run_passes(fn, optimize) + run_passes_on(ctx, optimize) return ctx diff --git a/vyper/venom/analysis/cfg.py b/vyper/venom/analysis/cfg.py index 700fd73f26..2f90410cd5 100644 --- a/vyper/venom/analysis/cfg.py +++ b/vyper/venom/analysis/cfg.py @@ -23,7 +23,7 @@ def analyze(self) -> None: bb.is_reachable = False for bb in fn.get_basic_blocks(): - assert bb.is_terminated + assert bb.is_terminated, f"not terminating:\n{bb}" term = bb.instructions[-1] if term.opcode in CFG_ALTERING_INSTRUCTIONS: diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index 968ce42bdf..e159a6d464 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -513,7 +513,7 @@ def insert_instruction(self, instruction: IRInstruction, index: Optional[int] = assert isinstance(instruction, IRInstruction), "instruction must be an IRInstruction" if index is None: - assert not self.is_terminated, self + assert not self.is_terminated, (self, instruction) index = len(self.instructions) instruction.parent = self instruction.ast_source = self.parent.ast_source diff --git a/vyper/venom/function.py b/vyper/venom/function.py index 0c48c9740e..2372f8ba52 100644 --- a/vyper/venom/function.py +++ b/vyper/venom/function.py @@ -12,7 +12,6 @@ class IRFunction: name: IRLabel # symbol name ctx: "IRContext" # type: ignore # noqa: F821 args: list - last_label: int last_variable: int _basic_block_dict: dict[str, IRBasicBlock] @@ -182,7 +181,6 @@ def chain_basic_blocks(self) -> None: def copy(self): new = IRFunction(self.name) new._basic_block_dict = self._basic_block_dict.copy() - new.last_label = self.last_label new.last_variable = self.last_variable return new diff --git a/vyper/venom/parser.py b/vyper/venom/parser.py new file mode 100644 index 0000000000..d2574c3b0c --- /dev/null +++ b/vyper/venom/parser.py @@ -0,0 +1,178 @@ +from lark import Lark, Transformer + +from vyper.venom.basicblock import ( + IRBasicBlock, + IRInstruction, + IRLabel, + IRLiteral, + IROperand, + IRVariable, +) +from vyper.venom.context import IRContext +from vyper.venom.function import IRFunction + +VENOM_PARSER = Lark( + """ + %import common.CNAME + %import common.DIGIT + %import common.LETTER + %import common.WS + %import common.INT + + # TODO: make data_section optional -- `function* data_section?` + start: function* data_section + + # TODO: consider making entry block implicit, e.g. + # `"{" instruction+ block* "}"` + function: "function" NAME "{" block* "}" + + data_section: "[data]" instruction* + + block: NAME ":" statement* + + statement: instruction | assignment + assignment: VAR_IDENT "=" expr + expr: instruction | CONST + instruction: OPCODE operands_list? + + operands_list: operand ("," operand)* + + operand: VAR_IDENT | CONST | LABEL + + CONST: INT + OPCODE: CNAME + VAR_IDENT: "%" INT (":" INT)? + LABEL: "@" NAME + NAME: (DIGIT|LETTER|"_")+ + + %ignore WS + """ +) + + +def _set_last_var(fn: IRFunction): + for bb in fn.get_basic_blocks(): + for inst in bb.instructions: + if inst.output is None: + continue + value = inst.output.value + assert value.startswith("%") + fn.last_variable = max(fn.last_variable, int(value[1:])) + + +def _set_last_label(ctx: IRContext): + for fn in ctx.functions.values(): + for bb in fn.get_basic_blocks(): + label = bb.label.value + label_head, *_ = label.split("_", maxsplit=1) + if label_head.isdigit(): + ctx.last_label = max(int(label_head), ctx.last_label) + + +def _ensure_terminated(bb): + # Since "revert" is not considered terminal explicitly check for it to ensure basic + # blocks are terminating + if not bb.is_terminated and any(inst.opcode == "revert" for inst in bb.instructions): + bb.append_instruction("stop") + + +class VenomTransformer(Transformer): + def start(self, children) -> IRContext: + ctx = IRContext() + funcs = children[:-1] + data_section = children[-1] + for fn_name, blocks in funcs: + fn = ctx.create_function(fn_name) + fn._basic_block_dict.clear() + + for block_name, instructions in blocks: + bb = IRBasicBlock(IRLabel(block_name), fn) + fn.append_basic_block(bb) + + for instruction in instructions: + assert isinstance(instruction, IRInstruction) # help mypy + bb.insert_instruction(instruction) + + _ensure_terminated(bb) + + _set_last_var(fn) + _set_last_label(ctx) + + ctx.data_segment = data_section + + return ctx + + def function(self, children) -> tuple[str, list[tuple[str, list[IRInstruction]]]]: + name, *blocks = children + return name, blocks + + def statement(self, children): + return children[0] + + def data_section(self, children): + return children + + def block(self, children) -> tuple[str, list[IRInstruction]]: + label, *instructions = children + return label, instructions + + def assignment(self, children) -> IRInstruction: + to, value = children + if isinstance(value, IRInstruction): + value.output = to + return value + if isinstance(value, IRLiteral): + return IRInstruction("store", [value], output=to) + raise TypeError(f"Unexpected value {value} of type {type(value)}") + + def expr(self, children): + return children[0] + + def instruction(self, children) -> IRInstruction: + if len(children) == 1: + name = children[0] + operands = [] + else: + assert len(children) == 2 + name, operands = children + + # reverse operands, venom internally represents top of stack + # as rightmost operand + return IRInstruction(name, reversed(operands)) + + def operands_list(self, children) -> list[IROperand]: + return children + + def operand(self, children) -> IROperand: + return children[0] + + def OPCODE(self, token): + return token.value + + def LABEL(self, label) -> IRLabel: + return IRLabel(label[1:]) + + def VAR_IDENT(self, var_ident) -> IRVariable: + parts = var_ident[1:].split(":", maxsplit=1) + assert 1 <= len(parts) <= 2 + varname = parts[0] + version = None + if len(parts) > 1: + version = parts[1] + return IRVariable(varname, version=version) + + def CONST(self, val) -> IRLiteral: + return IRLiteral(int(val)) + + def CNAME(self, val) -> str: + return val.value + + def NAME(self, val) -> str: + return val.value + + +def parse_venom(source: str) -> IRContext: + tree = VENOM_PARSER.parse(source) + ctx = VenomTransformer().transform(tree) + assert isinstance(ctx, IRContext) # help mypy + return ctx diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index 369be3e753..9004a357f0 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -143,7 +143,7 @@ def _handle_SSA_work_item(self, work_item: SSAWorkListItem): self._visit_expr(work_item.inst) def _lookup_from_lattice(self, op: IROperand) -> LatticeItem: - assert isinstance(op, IRVariable), "Can't get lattice for non-variable" + assert isinstance(op, IRVariable), f"Can't get lattice for non-variable ({op})" lat = self.lattice[op] assert lat is not None, f"Got undefined var {op}" return lat From 135c2d6b6c5ea053825b48b2a64312f20e0dc487 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 17 Dec 2024 09:12:11 -0500 Subject: [PATCH 36/70] refactor[venom]: add new venom test machinery (#4401) refactor venom tests with new parsing machinery. it should make tests easier to read. refactor `test_simplify_cfg.py` as an example of the new testing style additionally add some parser / text format fixes: - switch order of operands in `phi` text format - fix `jnz` parsing to match `IRInstruction.__repr__()` - add comments to venom grammar (`;` indicates a comment a la llvm) - make data section optional - fix grammar rule: expr can be any operand. --------- Co-authored-by: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> --- tests/functional/venom/parser/test_parsing.py | 50 ++----------- .../unit/compiler/venom/test_simplify_cfg.py | 75 ++++++++----------- tests/venom_utils.py | 50 +++++++++++++ vyper/venom/basicblock.py | 2 +- vyper/venom/parser.py | 38 +++++++--- 5 files changed, 114 insertions(+), 101 deletions(-) create mode 100644 tests/venom_utils.py diff --git a/tests/functional/venom/parser/test_parsing.py b/tests/functional/venom/parser/test_parsing.py index c121edb692..f1fc59cf40 100644 --- a/tests/functional/venom/parser/test_parsing.py +++ b/tests/functional/venom/parser/test_parsing.py @@ -1,49 +1,9 @@ +from tests.venom_utils import assert_ctx_eq from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRLabel, IRLiteral, IRVariable from vyper.venom.context import IRContext from vyper.venom.function import IRFunction from vyper.venom.parser import parse_venom -# TODO: Refactor tests with these helpers - - -def instructions_eq(i1: IRInstruction, i2: IRInstruction) -> bool: - return i1.output == i2.output and i1.opcode == i2.opcode and i1.operands == i2.operands - - -def assert_bb_eq(bb1: IRBasicBlock, bb2: IRBasicBlock): - assert bb1.label.value == bb2.label.value - assert len(bb1.instructions) == len(bb2.instructions) - for i1, i2 in zip(bb1.instructions, bb2.instructions): - assert instructions_eq(i1, i2), f"[{i1}] != [{i2}]" - - -def assert_fn_eq(fn1: IRFunction, fn2: IRFunction): - assert fn1.name.value == fn2.name.value - assert fn1.last_variable == fn2.last_variable - assert len(fn1._basic_block_dict) == len(fn2._basic_block_dict) - - for name1, bb1 in fn1._basic_block_dict.items(): - assert name1 in fn2._basic_block_dict - assert_bb_eq(bb1, fn2._basic_block_dict[name1]) - - # check function entry is the same - assert fn1.entry.label == fn2.entry.label - - -def assert_ctx_eq(ctx1: IRContext, ctx2: IRContext): - assert ctx1.last_label == ctx2.last_label - assert len(ctx1.functions) == len(ctx2.functions) - for label1, fn1 in ctx1.functions.items(): - assert label1 in ctx2.functions - assert_fn_eq(fn1, ctx2.functions[label1]) - - # check entry function is the same - assert next(iter(ctx1.functions.keys())) == next(iter(ctx2.functions.keys())) - - assert len(ctx1.data_segment) == len(ctx2.data_segment) - for d1, d2 in zip(ctx1.data_segment, ctx2.data_segment): - assert instructions_eq(d1, d2), f"data: [{d1}] != [{d2}]" - def test_single_bb(): source = """ @@ -70,7 +30,7 @@ def test_multi_bb_single_fn(): function start { start: %1 = callvalue - jnz @fine, @has_callvalue, %1 + jnz %1, @fine, @has_callvalue fine: %2 = calldataload 4 %4 = add %2, 279387 @@ -89,7 +49,7 @@ def test_multi_bb_single_fn(): start_bb = start_fn.get_basic_block("start") start_bb.append_instruction("callvalue", ret=IRVariable("1")) - start_bb.append_instruction("jnz", IRVariable("1"), IRLabel("has_callvalue"), IRLabel("fine")) + start_bb.append_instruction("jnz", IRVariable("1"), IRLabel("fine"), IRLabel("has_callvalue")) start_fn.append_basic_block(fine_bb := IRBasicBlock(IRLabel("fine"), start_fn)) fine_bb.append_instruction("calldataload", IRLiteral(4), ret=IRVariable("2")) @@ -160,7 +120,7 @@ def test_multi_function(): check_cv: %1 = callvalue %2 = param - jnz @no_value, @has_value, %1 + jnz %1, @has_value, @no_value no_value: ret %2 has_value: @@ -218,7 +178,7 @@ def test_multi_function_and_data(): check_cv: %1 = callvalue %2 = param - jnz @no_value, @has_value, %1 + jnz %1, @has_value, @no_value no_value: ret %2 has_value: diff --git a/tests/unit/compiler/venom/test_simplify_cfg.py b/tests/unit/compiler/venom/test_simplify_cfg.py index 3de6a77cc9..583f10efda 100644 --- a/tests/unit/compiler/venom/test_simplify_cfg.py +++ b/tests/unit/compiler/venom/test_simplify_cfg.py @@ -1,48 +1,37 @@ +from tests.venom_utils import assert_ctx_eq, parse_venom from vyper.venom.analysis import IRAnalysesCache -from vyper.venom.basicblock import IRBasicBlock, IRLabel, IRLiteral -from vyper.venom.context import IRContext from vyper.venom.passes import SCCP, SimplifyCFGPass def test_phi_reduction_after_block_pruning(): - ctx = IRContext() - fn = ctx.create_function("_global") - - bb = fn.get_basic_block() - - br1 = IRBasicBlock(IRLabel("then"), fn) - fn.append_basic_block(br1) - br2 = IRBasicBlock(IRLabel("else"), fn) - fn.append_basic_block(br2) - - join = IRBasicBlock(IRLabel("join"), fn) - fn.append_basic_block(join) - - true = IRLiteral(1) - bb.append_instruction("jnz", true, br1.label, br2.label) - - op1 = br1.append_instruction("store", 1) - op2 = br2.append_instruction("store", 2) - - br1.append_instruction("jmp", join.label) - br2.append_instruction("jmp", join.label) - - join.append_instruction("phi", br1.label, op1, br2.label, op2) - join.append_instruction("stop") - - ac = IRAnalysesCache(fn) - SCCP(ac, fn).run_pass() - SimplifyCFGPass(ac, fn).run_pass() - - bbs = list(fn.get_basic_blocks()) - - assert len(bbs) == 1 - final_bb = bbs[0] - - inst0, inst1, inst2 = final_bb.instructions - - assert inst0.opcode == "store" - assert inst0.operands == [IRLiteral(1)] - assert inst1.opcode == "store" - assert inst1.operands == [inst0.output] - assert inst2.opcode == "stop" + pre = """ + function _global { + _global: + jnz 1, @then, @else + then: + %1 = 1 + jmp @join + else: + %2 = 2 + jmp @join + join: + %3 = phi @then, %1, @else, %2 + stop + } + """ + post = """ + function _global { + _global: + %1 = 1 + %3 = %1 + stop + } + """ + ctx1 = parse_venom(pre) + for fn in ctx1.functions.values(): + ac = IRAnalysesCache(fn) + SCCP(ac, fn).run_pass() + SimplifyCFGPass(ac, fn).run_pass() + + ctx2 = parse_venom(post) + assert_ctx_eq(ctx1, ctx2) diff --git a/tests/venom_utils.py b/tests/venom_utils.py new file mode 100644 index 0000000000..d4536e8bf7 --- /dev/null +++ b/tests/venom_utils.py @@ -0,0 +1,50 @@ +from vyper.venom.basicblock import IRBasicBlock, IRInstruction +from vyper.venom.context import IRContext +from vyper.venom.function import IRFunction +from vyper.venom.parser import parse_venom + + +def parse_from_basic_block(source: str, funcname="_global"): + """ + Parse an IRContext from a basic block + """ + source = f"function {funcname} {{\n{source}\n}}" + return parse_venom(source) + + +def instructions_eq(i1: IRInstruction, i2: IRInstruction) -> bool: + return i1.output == i2.output and i1.opcode == i2.opcode and i1.operands == i2.operands + + +def assert_bb_eq(bb1: IRBasicBlock, bb2: IRBasicBlock): + assert bb1.label.value == bb2.label.value + assert len(bb1.instructions) == len(bb2.instructions) + for i1, i2 in zip(bb1.instructions, bb2.instructions): + assert instructions_eq(i1, i2), f"[{i1}] != [{i2}]" + + +def assert_fn_eq(fn1: IRFunction, fn2: IRFunction): + assert fn1.name.value == fn2.name.value + assert len(fn1._basic_block_dict) == len(fn2._basic_block_dict) + + for name1, bb1 in fn1._basic_block_dict.items(): + assert name1 in fn2._basic_block_dict + assert_bb_eq(bb1, fn2._basic_block_dict[name1]) + + # check function entry is the same + assert fn1.entry.label == fn2.entry.label + + +def assert_ctx_eq(ctx1: IRContext, ctx2: IRContext): + assert ctx1.last_label == ctx2.last_label + assert len(ctx1.functions) == len(ctx2.functions) + for label1, fn1 in ctx1.functions.items(): + assert label1 in ctx2.functions + assert_fn_eq(fn1, ctx2.functions[label1]) + + # check entry function is the same + assert next(iter(ctx1.functions.keys())) == next(iter(ctx2.functions.keys())) + + assert len(ctx1.data_segment) == len(ctx2.data_segment) + for d1, d2 in zip(ctx1.data_segment, ctx2.data_segment): + assert instructions_eq(d1, d2), f"data: [{d1}] != [{d2}]" diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index e159a6d464..cb2904f97f 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -381,7 +381,7 @@ def __repr__(self) -> str: opcode = f"{self.opcode} " if self.opcode != "store" else "" s += opcode operands = self.operands - if opcode not in ["jmp", "jnz", "invoke"]: + if opcode not in ("jmp", "jnz", "invoke"): operands = reversed(operands) # type: ignore s += ", ".join( [(f"label %{op}" if isinstance(op, IRLabel) else str(op)) for op in operands] diff --git a/vyper/venom/parser.py b/vyper/venom/parser.py index d2574c3b0c..e502c89278 100644 --- a/vyper/venom/parser.py +++ b/vyper/venom/parser.py @@ -19,8 +19,7 @@ %import common.WS %import common.INT - # TODO: make data_section optional -- `function* data_section?` - start: function* data_section + start: function* data_section? # TODO: consider making entry block implicit, e.g. # `"{" instruction+ block* "}"` @@ -32,7 +31,7 @@ statement: instruction | assignment assignment: VAR_IDENT "=" expr - expr: instruction | CONST + expr: instruction | operand instruction: OPCODE operands_list? operands_list: operand ("," operand)* @@ -45,7 +44,10 @@ LABEL: "@" NAME NAME: (DIGIT|LETTER|"_")+ + COMMENT: ";" /[^\\n]/* + %ignore WS + %ignore COMMENT """ ) @@ -72,15 +74,24 @@ def _set_last_label(ctx: IRContext): def _ensure_terminated(bb): # Since "revert" is not considered terminal explicitly check for it to ensure basic # blocks are terminating - if not bb.is_terminated and any(inst.opcode == "revert" for inst in bb.instructions): - bb.append_instruction("stop") + if not bb.is_terminated: + if any(inst.opcode == "revert" for inst in bb.instructions): + bb.append_instruction("stop") + # TODO: raise error if still not terminated. + + +class _DataSegment: + def __init__(self, instructions): + self.instructions = instructions class VenomTransformer(Transformer): def start(self, children) -> IRContext: ctx = IRContext() - funcs = children[:-1] - data_section = children[-1] + data_section = [] + if isinstance(children[-1], _DataSegment): + data_section = children.pop().instructions + funcs = children for fn_name, blocks in funcs: fn = ctx.create_function(fn_name) fn._basic_block_dict.clear() @@ -110,7 +121,7 @@ def statement(self, children): return children[0] def data_section(self, children): - return children + return _DataSegment(children) def block(self, children) -> tuple[str, list[IRInstruction]]: label, *instructions = children @@ -121,7 +132,7 @@ def assignment(self, children) -> IRInstruction: if isinstance(value, IRInstruction): value.output = to return value - if isinstance(value, IRLiteral): + if isinstance(value, (IRLiteral, IRVariable)): return IRInstruction("store", [value], output=to) raise TypeError(f"Unexpected value {value} of type {type(value)}") @@ -130,15 +141,18 @@ def expr(self, children): def instruction(self, children) -> IRInstruction: if len(children) == 1: - name = children[0] + opcode = children[0] operands = [] else: assert len(children) == 2 - name, operands = children + opcode, operands = children # reverse operands, venom internally represents top of stack # as rightmost operand - return IRInstruction(name, reversed(operands)) + if opcode not in ("jmp", "jnz", "invoke", "phi"): + # special cases: operands with labels look better un-reversed + operands.reverse() + return IRInstruction(opcode, operands) def operands_list(self, children) -> list[IROperand]: return children From ebe26a60dfa173d19713dec8ada07af908361e04 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Tue, 17 Dec 2024 17:06:44 +0200 Subject: [PATCH 37/70] feat[venom]: allow alphanumeric variables and source comments (#4403) This commit implements minor improvements to the venom parser. Specifically it allows for alphanumeric variable names, and for single line source code comments in various styles: ";" "#" and "//" --------- Co-authored-by: Charles Cooper --- vyper/venom/parser.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/vyper/venom/parser.py b/vyper/venom/parser.py index e502c89278..d219f271b3 100644 --- a/vyper/venom/parser.py +++ b/vyper/venom/parser.py @@ -19,6 +19,9 @@ %import common.WS %import common.INT + # Allow multiple comment styles + COMMENT: ";" /[^\\n]*/ | "//" /[^\\n]*/ | "#" /[^\\n]*/ + start: function* data_section? # TODO: consider making entry block implicit, e.g. @@ -40,12 +43,10 @@ CONST: INT OPCODE: CNAME - VAR_IDENT: "%" INT (":" INT)? + VAR_IDENT: "%" NAME LABEL: "@" NAME NAME: (DIGIT|LETTER|"_")+ - COMMENT: ";" /[^\\n]/* - %ignore WS %ignore COMMENT """ @@ -59,7 +60,9 @@ def _set_last_var(fn: IRFunction): continue value = inst.output.value assert value.startswith("%") - fn.last_variable = max(fn.last_variable, int(value[1:])) + varname = value[1:] + if varname.isdigit(): + fn.last_variable = max(fn.last_variable, int(varname)) def _set_last_label(ctx: IRContext): @@ -172,7 +175,7 @@ def VAR_IDENT(self, var_ident) -> IRVariable: varname = parts[0] version = None if len(parts) > 1: - version = parts[1] + version = int(parts[1]) return IRVariable(varname, version=version) def CONST(self, val) -> IRLiteral: From b0ea1b352530b461cfbbdeb8d2069c5401a1989e Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:59:28 +0100 Subject: [PATCH 38/70] fix[lang]: disallow absolute relative imports (#4268) this commit changes the behavior of search paths within submodules. previously the current module's parent was appended to the search path in the import recursion. this means that given the following directory structure, this code is legal: ``` # directory structure: # subdir/foo.vy # subdir/bar.vy ``` ``` # subdir/foo.vy import bar ``` which has the same semantics as ``` # subdir/foo.vy from . import bar ``` this represented a divergence wrt python's import system. in python, only the second example is allowed. this commit changes the semantics to match python, so that relative imports use the current directory as search path, but absolute imports can only use the top-level search path. (the directory containing the top-level compilation target is still included in the search path, but this only happens at the top of the recursion, same as in python - https://docs.python.org/3/library/sys_path_init.html). misc/refactor: - remove dead `add_search_path()` method - remove now-dead `poke_search_path()` --------- Co-authored-by: Charles Cooper --- tests/functional/codegen/test_interfaces.py | 29 ++-- tests/functional/syntax/test_import.py | 129 ++++++++++++++++++ .../unit/cli/vyper_json/test_compile_json.py | 5 +- vyper/compiler/input_bundle.py | 14 +- vyper/semantics/analysis/imports.py | 31 +++-- 5 files changed, 177 insertions(+), 31 deletions(-) create mode 100644 tests/functional/syntax/test_import.py diff --git a/tests/functional/codegen/test_interfaces.py b/tests/functional/codegen/test_interfaces.py index 31475a3bc0..e46a7d3dd4 100644 --- a/tests/functional/codegen/test_interfaces.py +++ b/tests/functional/codegen/test_interfaces.py @@ -4,7 +4,7 @@ from eth_utils import to_wei from tests.utils import decimal_to_int -from vyper.compiler import compile_code +from vyper.compiler import compile_code, compile_from_file_input from vyper.exceptions import ( ArgumentException, DuplicateImport, @@ -161,8 +161,6 @@ def bar() -> uint256: ("import Foo as Foo", "Foo.vyi"), ("from a import Foo", "a/Foo.vyi"), ("from b.a import Foo", "b/a/Foo.vyi"), - ("from .a import Foo", "./a/Foo.vyi"), - ("from ..a import Foo", "../a/Foo.vyi"), ] @@ -174,6 +172,22 @@ def test_extract_file_interface_imports(code, filename, make_input_bundle): assert compile_code(code, input_bundle=input_bundle) is not None +VALID_RELATIVE_IMPORT_CODE = [ + # import statement, import path without suffix + ("from .a import Foo", "mock.vy"), + ("from ..a import Foo", "b/mock.vy"), +] + + +# TODO CMC 2024-10-13: should probably be in syntax tests +@pytest.mark.parametrize("code,filename", VALID_RELATIVE_IMPORT_CODE) +def test_extract_file_interface_relative_imports(code, filename, make_input_bundle): + input_bundle = make_input_bundle({"a/Foo.vyi": "", filename: code}) + + file_input = input_bundle.load_file(filename) + assert compile_from_file_input(file_input, input_bundle=input_bundle) is not None + + BAD_IMPORT_CODE = [ ("import a as A\nimport a as A", DuplicateImport), ("import a as A\nimport a as a", DuplicateImport), @@ -186,12 +200,11 @@ def test_extract_file_interface_imports(code, filename, make_input_bundle): # TODO CMC 2024-10-13: should probably be in syntax tests @pytest.mark.parametrize("code,exception_type", BAD_IMPORT_CODE) -def test_extract_file_interface_imports_raises( - code, exception_type, assert_compile_failed, make_input_bundle -): - input_bundle = make_input_bundle({"a.vyi": "", "b/a.vyi": "", "c.vyi": ""}) +def test_extract_file_interface_imports_raises(code, exception_type, make_input_bundle): + input_bundle = make_input_bundle({"a.vyi": "", "b/a.vyi": "", "c.vyi": "", "mock.vy": code}) + file_input = input_bundle.load_file("mock.vy") with pytest.raises(exception_type): - compile_code(code, input_bundle=input_bundle) + compile_from_file_input(file_input, input_bundle=input_bundle) def test_external_call_to_interface(env, get_contract, make_input_bundle): diff --git a/tests/functional/syntax/test_import.py b/tests/functional/syntax/test_import.py new file mode 100644 index 0000000000..07b1a336c3 --- /dev/null +++ b/tests/functional/syntax/test_import.py @@ -0,0 +1,129 @@ +import pytest + +from vyper import compiler +from vyper.exceptions import ModuleNotFound + +CODE_TOP = """ +import subdir0.lib0 as lib0 +@external +def foo(): + lib0.foo() +""" + +CODE_LIB1 = """ +def foo(): + pass +""" + + +def test_implicitly_relative_import_crashes(make_input_bundle): + lib0 = """ +import subdir1.lib1 as lib1 +def foo(): + lib1.foo() + """ + + input_bundle = make_input_bundle( + {"top.vy": CODE_TOP, "subdir0/lib0.vy": lib0, "subdir0/subdir1/lib1.vy": CODE_LIB1} + ) + + file_input = input_bundle.load_file("top.vy") + with pytest.raises(ModuleNotFound): + compiler.compile_from_file_input(file_input, input_bundle=input_bundle) + + +def test_implicitly_relative_import_crashes_2(make_input_bundle): + lib0 = """ +from subdir1 import lib1 as lib1 +def foo(): + lib1.foo() + """ + + input_bundle = make_input_bundle( + {"top.vy": CODE_TOP, "subdir0/lib0.vy": lib0, "subdir0/subdir1/lib1.vy": CODE_LIB1} + ) + + file_input = input_bundle.load_file("top.vy") + with pytest.raises(ModuleNotFound): + compiler.compile_from_file_input(file_input, input_bundle=input_bundle) + + +def test_relative_import_searches_only_current_path(make_input_bundle): + top = """ +from subdir import b as b +@external +def foo(): + b.foo() + """ + + a = """ +def foo(): + pass + """ + + b = """ +from . import a as a +def foo(): + a.foo() + """ + + input_bundle = make_input_bundle({"top.vy": top, "a.vy": a, "subdir/b.vy": b}) + file_input = input_bundle.load_file("top.vy") + + with pytest.raises(ModuleNotFound): + compiler.compile_from_file_input(file_input, input_bundle=input_bundle) + + +def test_absolute_import_within_relative_import(make_input_bundle): + top = """ +import subdir0.subdir1.c as c +@external +def foo(): + c.foo() + """ + a = """ +import subdir0.b as b +def foo(): + b.foo() + """ + b = """ +def foo(): + pass + """ + + c = """ +from .. import a as a +def foo(): + a.foo() + """ + + input_bundle = make_input_bundle( + {"top.vy": top, "subdir0/a.vy": a, "subdir0/b.vy": b, "subdir0/subdir1/c.vy": c} + ) + compiler.compile_code(top, input_bundle=input_bundle) + + +def test_absolute_path_passes(make_input_bundle): + lib0 = """ +import subdir0.subdir1.lib1 as lib1 +def foo(): + lib1.foo() + """ + + input_bundle = make_input_bundle( + {"top.vy": CODE_TOP, "subdir0/lib0.vy": lib0, "subdir0/subdir1/lib1.vy": CODE_LIB1} + ) + compiler.compile_code(CODE_TOP, input_bundle=input_bundle) + + +def test_absolute_path_passes_2(make_input_bundle): + lib0 = """ +from .subdir1 import lib1 as lib1 +def foo(): + lib1.foo() + """ + + input_bundle = make_input_bundle( + {"top.vy": CODE_TOP, "subdir0/lib0.vy": lib0, "subdir0/subdir1/lib1.vy": CODE_LIB1} + ) + compiler.compile_code(CODE_TOP, input_bundle=input_bundle) diff --git a/tests/unit/cli/vyper_json/test_compile_json.py b/tests/unit/cli/vyper_json/test_compile_json.py index 8cd9be128f..7e281bda2e 100644 --- a/tests/unit/cli/vyper_json/test_compile_json.py +++ b/tests/unit/cli/vyper_json/test_compile_json.py @@ -294,7 +294,7 @@ def get(filename, contractname): def test_relative_import_paths(input_json): input_json["sources"]["contracts/potato/baz/baz.vy"] = {"content": "from ... import foo"} input_json["sources"]["contracts/potato/baz/potato.vy"] = {"content": "from . import baz"} - input_json["sources"]["contracts/potato/footato.vy"] = {"content": "from baz import baz"} + input_json["sources"]["contracts/potato/footato.vy"] = {"content": "from .baz import baz"} compile_from_input_dict(input_json) @@ -319,7 +319,8 @@ def test_compile_json_with_abi_top(make_input_bundle): from . import stream """ input_bundle = make_input_bundle({"stream.json": stream, "code.vy": code}) - vyper.compiler.compile_code(code, input_bundle=input_bundle) + file_input = input_bundle.load_file("code.vy") + vyper.compiler.compile_from_file_input(file_input, input_bundle=input_bundle) def test_compile_json_with_experimental_codegen(): diff --git a/vyper/compiler/input_bundle.py b/vyper/compiler/input_bundle.py index c9eeded3cf..06fee78613 100644 --- a/vyper/compiler/input_bundle.py +++ b/vyper/compiler/input_bundle.py @@ -137,9 +137,6 @@ def load_file(self, path: PathLike | str) -> CompilerInput: return res - def add_search_path(self, path: PathLike) -> None: - self.search_paths.append(path) - # temporarily add something to the search path (within the # scope of the context manager) with highest precedence. # if `path` is None, do nothing @@ -155,16 +152,15 @@ def search_path(self, path: Optional[PathLike]) -> Iterator[None]: finally: self.search_paths.pop() - # temporarily modify the top of the search path (within the - # scope of the context manager) with highest precedence to something else + # temporarily set search paths to a given list @contextlib.contextmanager - def poke_search_path(self, path: PathLike) -> Iterator[None]: - tmp = self.search_paths[-1] - self.search_paths[-1] = path + def temporary_search_paths(self, new_paths: list[PathLike]) -> Iterator[None]: + original_paths = self.search_paths + self.search_paths = new_paths try: yield finally: - self.search_paths[-1] = tmp + self.search_paths = original_paths # regular input. takes a search path(s), and `load_file()` will search all diff --git a/vyper/semantics/analysis/imports.py b/vyper/semantics/analysis/imports.py index 3268f12e94..4f8daefa97 100644 --- a/vyper/semantics/analysis/imports.py +++ b/vyper/semantics/analysis/imports.py @@ -81,6 +81,9 @@ def __init__(self, input_bundle: InputBundle, graph: _ImportGraph): self.integrity_sum = None + # should be all system paths + topmost module path + self.absolute_search_paths = input_bundle.search_paths.copy() + def resolve_imports(self, module_ast: vy_ast.Module): self._resolve_imports_r(module_ast) self.integrity_sum = self._calculate_integrity_sum_r(module_ast) @@ -151,15 +154,7 @@ def _add_import( # load an InterfaceT or ModuleInfo from an import. # raises FileNotFoundError - def _load_import(self, node: vy_ast.VyperNode, level: int, module_str: str, alias: str) -> Any: - # the directory this (currently being analyzed) module is in - ast = self.graph.current_module - self_search_path = Path(ast.resolved_path).parent - - with self.input_bundle.poke_search_path(self_search_path): - return self._load_import_helper(node, level, module_str, alias) - - def _load_import_helper( + def _load_import( self, node: vy_ast.VyperNode, level: int, module_str: str, alias: str ) -> tuple[CompilerInput, Any]: if _is_builtin(module_str): @@ -177,7 +172,7 @@ def _load_import_helper( try: path_vy = path.with_suffix(".vy") - file = self.input_bundle.load_file(path_vy) + file = self._load_file(path_vy, level) assert isinstance(file, FileInput) # mypy hint module_ast = self._ast_from_file(file) @@ -191,7 +186,7 @@ def _load_import_helper( err = e try: - file = self.input_bundle.load_file(path.with_suffix(".vyi")) + file = self._load_file(path.with_suffix(".vyi"), level) assert isinstance(file, FileInput) # mypy hint module_ast = self._ast_from_file(file) self.resolve_imports(module_ast) @@ -205,7 +200,7 @@ def _load_import_helper( pass try: - file = self.input_bundle.load_file(path.with_suffix(".json")) + file = self._load_file(path.with_suffix(".json"), level) assert isinstance(file, ABIInput) # mypy hint return file, file.abi except FileNotFoundError: @@ -219,6 +214,18 @@ def _load_import_helper( search_paths = self.input_bundle.search_paths.copy() # noqa: F841 raise ModuleNotFound(module_str, hint=hint) from err + def _load_file(self, path: PathLike, level: int) -> CompilerInput: + ast = self.graph.current_module + + search_paths: list[PathLike] # help mypy + if level != 0: # relative import + search_paths = [Path(ast.resolved_path).parent] + else: + search_paths = self.absolute_search_paths + + with self.input_bundle.temporary_search_paths(search_paths): + return self.input_bundle.load_file(path) + def _ast_from_file(self, file: FileInput) -> vy_ast.Module: # cache ast if we have seen it before. # this gives us the additional property of object equality on From f31ff46bdf3a1708116fdaccba62301063caba93 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Tue, 17 Dec 2024 22:28:38 +0100 Subject: [PATCH 39/70] fix[tool]: add missing internal functions to metadata (#4328) this commit adds all reachable internal functions to the `-f metadata` output. this was overlooked during the 0.4.0 release - there could be internal functions in submodules which didn't get added to the `-f metadata` output, since the routine to build the `-f metadata` output only looked at the top-level module. this commit also makes a change to the format of the `.function_info` keys - since function names are not guaranteed to be globally unique anymore (that is, two different modules may both contain a function named `foo`), this commit adds a disambiguating function id to each key in `.function_info`. (it uses the function_id computed during IR generation since it is guaranteed to be unique). the function name is still present in the function metadata under the `.name` key, but downstream tooling may have to adapt if they had the assumption that the keys in `.function_info` were parseable names. for illustration, here is an example of function metadata prior to and after this commit. prior to this commit: ```json { "function_info": { // valid identifier, but not necessarily unique "foo": { "name": "foo", "_ir_identifier": "internal 0 foo()", ... } } } ``` after this commit: ```json { "function_info": { // not a valid identifier, but is unique. for the function name, see // the "name" field below. "foo (0)": { "name": "foo", "_ir_identifier": "internal 0 foo()", ... } } } ``` --------- Co-authored-by: Charles Cooper --- .../cli/vyper_json/test_output_selection.py | 53 +++++++++++++++++++ vyper/compiler/output.py | 43 ++++++++++----- 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/tests/unit/cli/vyper_json/test_output_selection.py b/tests/unit/cli/vyper_json/test_output_selection.py index f7fbfe673c..e409c43af6 100644 --- a/tests/unit/cli/vyper_json/test_output_selection.py +++ b/tests/unit/cli/vyper_json/test_output_selection.py @@ -2,6 +2,7 @@ import pytest +from vyper import compiler from vyper.cli.vyper_json import TRANSLATE_MAP, get_output_formats from vyper.exceptions import JSONError @@ -76,3 +77,55 @@ def test_solc_style(): def test_metadata(): input_json = {"sources": {"foo.vy": ""}, "settings": {"outputSelection": {"*": ["metadata"]}}} assert get_output_formats(input_json) == {PurePath("foo.vy"): ["metadata"]} + + +def test_metadata_contain_all_reachable_functions(make_input_bundle, chdir_tmp_path): + code_a = """ +@internal +def foo() -> uint256: + return 43 + +@internal +def faa() -> uint256: + return 76 + """ + + code_b = """ +import A + +@internal +def foo() -> uint256: + return 43 + +@external +def bar(): + self.foo() + A.foo() + assert 1 != 12 + """ + + input_bundle = make_input_bundle({"A.vy": code_a, "B.vy": code_b}) + file_input = input_bundle.load_file("B.vy") + + res = compiler.compile_from_file_input( + file_input, input_bundle=input_bundle, output_formats=["metadata"] + ) + function_infos = res["metadata"]["function_info"] + + assert "foo (0)" in function_infos + assert "foo (1)" in function_infos + assert "bar (2)" in function_infos + # faa is unreachable, should not be in metadata or bytecode + assert not any("faa" in key for key in function_infos.keys()) + + assert function_infos["foo (0)"]["function_id"] == 0 + assert function_infos["foo (1)"]["function_id"] == 1 + assert function_infos["bar (2)"]["function_id"] == 2 + + assert function_infos["foo (0)"]["module_path"] == "B.vy" + assert function_infos["foo (1)"]["module_path"] == "A.vy" + assert function_infos["bar (2)"]["module_path"] == "B.vy" + + assert function_infos["foo (0)"]["source_id"] == input_bundle.load_file("B.vy").source_id + assert function_infos["foo (1)"]["source_id"] == input_bundle.load_file("A.vy").source_id + assert function_infos["bar (2)"]["source_id"] == input_bundle.load_file("B.vy").source_id diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index e0eea293bc..1a6b3e9c07 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -13,10 +13,10 @@ from vyper.exceptions import VyperException from vyper.ir import compile_ir from vyper.semantics.analysis.base import ModuleInfo -from vyper.semantics.types.function import FunctionVisibility, StateMutability +from vyper.semantics.types.function import ContractFunctionT, FunctionVisibility, StateMutability from vyper.semantics.types.module import InterfaceT from vyper.typing import StorageLayout -from vyper.utils import vyper_warn +from vyper.utils import safe_relpath, vyper_warn from vyper.warnings import ContractSizeLimitWarning @@ -206,17 +206,27 @@ def build_ir_runtime_dict_output(compiler_data: CompilerData) -> dict: def build_metadata_output(compiler_data: CompilerData) -> dict: - sigs = compiler_data.function_signatures - - def _var_rec_dict(variable_record): - ret = vars(variable_record).copy() - ret["typ"] = str(ret["typ"]) - if ret["data_offset"] is None: - del ret["data_offset"] - for k in ("blockscopes", "defined_at", "encoding"): - del ret[k] - ret["location"] = ret["location"].name - return ret + # need ir info to be computed + _ = compiler_data.function_signatures + module_t = compiler_data.annotated_vyper_module._metadata["type"] + sigs = dict[str, ContractFunctionT]() + + def _fn_identifier(fn_t): + fn_id = fn_t._function_id + return f"{fn_t.name} ({fn_id})" + + for fn_t in module_t.exposed_functions: + assert isinstance(fn_t.ast_def, vy_ast.FunctionDef) + for rif_t in fn_t.reachable_internal_functions: + k = _fn_identifier(rif_t) + if k in sigs: + # sanity check that keys are injective with functions + assert sigs[k] == rif_t, (k, sigs[k], rif_t) + sigs[k] = rif_t + + fn_id = _fn_identifier(fn_t) + assert fn_id not in sigs + sigs[fn_id] = fn_t def _to_dict(func_t): ret = vars(func_t).copy() @@ -238,6 +248,10 @@ def _to_dict(func_t): ret["frame_info"] = vars(func_t._ir_info.frame_info).copy() del ret["frame_info"]["frame_vars"] # frame_var.pos might be IR, cannot serialize + ret["module_path"] = safe_relpath(func_t.decl_node.module_node.resolved_path) + ret["source_id"] = func_t.decl_node.module_node.source_id + ret["function_id"] = func_t._function_id + keep_keys = { "name", "return_type", @@ -249,6 +263,9 @@ def _to_dict(func_t): "visibility", "_ir_identifier", "nonreentrant_key", + "module_path", + "source_id", + "function_id", } ret = {k: v for k, v in ret.items() if k in keep_keys} return ret From 296a071eca28673b3937be3467f063036401564a Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 18 Dec 2024 16:12:37 +0200 Subject: [PATCH 40/70] feat[venom]: cleanup variable version handling (#4404) refactor `IRVariable` so that parsing cannot result in a nonzero `.version` (it should only be modified by internal passes). --------- Co-authored-by: Charles Cooper --- vyper/venom/basicblock.py | 48 ++++++++++++++++++++++----------------- vyper/venom/parser.py | 8 +------ 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index cb2904f97f..60f4610799 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -115,13 +115,20 @@ class IROperand: """ value: Any + _hash: Optional[int] + + def __init__(self, value: Any) -> None: + self.value = value + self._hash = None @property def name(self) -> str: return self.value def __hash__(self) -> int: - return hash(self.value) + if self._hash is None: + self._hash = hash(self.value) + return self._hash def __eq__(self, other) -> bool: if not isinstance(other, type(self)): @@ -140,6 +147,7 @@ class IRLiteral(IROperand): value: int def __init__(self, value: int) -> None: + super().__init__(value) assert isinstance(value, int), "value must be an int" self.value = value @@ -149,27 +157,25 @@ class IRVariable(IROperand): IRVariable represents a variable in IR. A variable is a string that starts with a %. """ - value: str - - def __init__(self, value: str, version: Optional[str | int] = None) -> None: - assert isinstance(value, str) - assert ":" not in value, "Variable name cannot contain ':'" - if version: - assert isinstance(value, str) or isinstance(value, int), "value must be an str or int" - value = f"{value}:{version}" - if value[0] != "%": - value = f"%{value}" - self.value = value + _name: str + version: Optional[int] + + def __init__(self, name: str, version: int = 0) -> None: + super().__init__(name) + assert isinstance(name, str) + assert isinstance(version, int | None) + if not name.startswith("%"): + name = f"%{name}" + self._name = name + self.version = version + if version > 0: + self.value = f"{name}:{version}" + else: + self.value = name @property def name(self) -> str: - return self.value.split(":")[0] - - @property - def version(self) -> int: - if ":" not in self.value: - return 0 - return int(self.value.split(":")[1]) + return self._name class IRLabel(IROperand): @@ -184,7 +190,7 @@ class IRLabel(IROperand): def __init__(self, value: str, is_symbol: bool = False) -> None: assert isinstance(value, str), "value must be an str" - self.value = value + super().__init__(value) self.is_symbol = is_symbol @@ -464,7 +470,7 @@ def remove_cfg_out(self, bb: "IRBasicBlock") -> None: self.cfg_out.remove(bb) def append_instruction( - self, opcode: str, *args: Union[IROperand, int], ret: IRVariable = None + self, opcode: str, *args: Union[IROperand, int], ret: Optional[IRVariable] = None ) -> Optional[IRVariable]: """ Append an instruction to the basic block diff --git a/vyper/venom/parser.py b/vyper/venom/parser.py index d219f271b3..f8ba33e522 100644 --- a/vyper/venom/parser.py +++ b/vyper/venom/parser.py @@ -170,13 +170,7 @@ def LABEL(self, label) -> IRLabel: return IRLabel(label[1:]) def VAR_IDENT(self, var_ident) -> IRVariable: - parts = var_ident[1:].split(":", maxsplit=1) - assert 1 <= len(parts) <= 2 - varname = parts[0] - version = None - if len(parts) > 1: - version = int(parts[1]) - return IRVariable(varname, version=version) + return IRVariable(var_ident[1:]) def CONST(self, val) -> IRLiteral: return IRLiteral(int(val)) From e20c36376e8566184b63b7ed340e4587bfb3735b Mon Sep 17 00:00:00 2001 From: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:56:23 +0000 Subject: [PATCH 41/70] feat[venom]: merge memory writes (#4341) Adapt the memory merging pass from the legacy pipeline into venom pipeline with additional improvements. It merges instructions that copy data or zero memory into a single copy instruction if possible (with the exception that if the copy is exactly 32 bytes, it uses a load/mstore pair to save 1 byte / 3 gas). It also handles sequences of interleaved and out-of-order copies, even though these are not currently generated by the frontend. Examples: input1: ``` %1 = mload 0 mstore 1000, %1 %2 = mload 32 mstore 1032, %2 ``` output1: ``` mcopy 1000, 0, 64 ``` input2: ``` mstore 100, 0 mstore 132, 0 ``` output2: ``` %1 = calldatasize calldatacopy 100, %1, 64 ``` It handles copies from memory (mload/mstore/mcopy) and calldata (calldataload/calldatacopy). For zeroing memory it uses `calldatacopy` from the `calldatasize` offset. --------- Co-authored-by: Charles Cooper --- tests/unit/compiler/venom/test_memmerging.py | 1059 ++++++++++++++++++ vyper/utils.py | 10 +- vyper/venom/__init__.py | 2 + vyper/venom/basicblock.py | 19 +- vyper/venom/passes/__init__.py | 1 + vyper/venom/passes/memmerging.py | 357 ++++++ 6 files changed, 1434 insertions(+), 14 deletions(-) create mode 100644 tests/unit/compiler/venom/test_memmerging.py create mode 100644 vyper/venom/passes/memmerging.py diff --git a/tests/unit/compiler/venom/test_memmerging.py b/tests/unit/compiler/venom/test_memmerging.py new file mode 100644 index 0000000000..b7a4d33805 --- /dev/null +++ b/tests/unit/compiler/venom/test_memmerging.py @@ -0,0 +1,1059 @@ +import pytest + +from tests.venom_utils import assert_ctx_eq, parse_from_basic_block, parse_venom +from vyper.evm.opcodes import version_check +from vyper.venom.analysis import IRAnalysesCache +from vyper.venom.passes import SCCP, MemMergePass + + +def _check_pre_post(pre, post): + ctx = parse_from_basic_block(pre) + for fn in ctx.functions.values(): + ac = IRAnalysesCache(fn) + MemMergePass(ac, fn).run_pass() + assert_ctx_eq(ctx, parse_from_basic_block(post)) + + +def _check_no_change(pre): + _check_pre_post(pre, pre) + + +def test_memmerging(): + """ + Basic memory merge test + All mloads and mstores can be + transformed into mcopy + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 32 + %3 = mload 64 + mstore 1000, %1 + mstore 1032, %2 + mstore 1064, %3 + stop + """ + + post = """ + _global: + mcopy 1000, 0, 96 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_out_of_order(): + """ + interleaved mloads/mstores which can be transformed into mcopy + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 32 + %2 = mload 0 + mstore 132, %1 + %3 = mload 64 + mstore 164, %3 + mstore 100, %2 + stop + """ + + post = """ + _global: + mcopy 100, 0, 96 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_imposs(): + """ + Test case of impossible merge + Impossible because of the overlap + [0 96] + [32 128] + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 32 + %3 = mload 64 + mstore 32, %1 + + ; BARRIER - overlap between src and dst + ; (writes to source of potential mcopy) + mstore 64, %2 + + mstore 96, %3 + stop + """ + _check_no_change(pre) + + +def test_memmerging_imposs_mstore(): + """ + Test case of impossible merge + Impossible because of the mstore barrier + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 16 + mstore 1000, %1 + %3 = mload 1000 ; BARRIER - load from dst of potential mcopy + mstore 1016, %2 + mstore 2000, %3 + stop + """ + _check_no_change(pre) + + +@pytest.mark.xfail +def test_memmerging_bypass_fence(): + """ + We should be able to optimize this to an mcopy(0, 1000, 64), but + currently do not + """ + if not version_check(begin="cancun"): + raise AssertionError() # xfail + + pre = """ + function _global { + _global: + %1 = mload 0 + %2 = mload 32 + mstore %1, 1000 + %3 = mload 1000 + mstore 1032, %2 + mstore 2000, %3 + stop + } + """ + + ctx = parse_venom(pre) + + for fn in ctx.functions.values(): + ac = IRAnalysesCache(fn) + SCCP(ac, fn).run_pass() + MemMergePass(ac, fn).run_pass() + + fn = next(iter(ctx.functions.values())) + bb = fn.entry + assert any(inst.opcode == "mcopy" for inst in bb.instructions) + + +def test_memmerging_imposs_unkown_place(): + """ + Test case of impossible merge + Impossible because of the + non constant address mload and mstore barier + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = param + %2 = mload 0 + %3 = mload %1 ; BARRIER + %4 = mload 32 + %5 = mload 64 + mstore 1000, %2 + mstore 1032, %4 + mstore 10, %1 ; BARRIER + mstore 1064, %5 + stop + """ + _check_no_change(pre) + + +def test_memmerging_imposs_msize(): + """ + Test case of impossible merge + Impossible because of the msize barier + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = msize ; BARRIER + %3 = mload 32 + %4 = mload 64 + mstore 1000, %1 + mstore 1032, %3 + %5 = msize ; BARRIER + mstore 1064, %4 + return %2, %5 + """ + _check_no_change(pre) + + +def test_memmerging_partial_msize(): + """ + Only partial merge possible + because of the msize barier + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 32 + %3 = mload 64 + mstore 1000, %1 + mstore 1032, %2 + %4 = msize ; BARRIER + mstore 1064, %3 + return %4 + """ + + post = """ + _global: + %3 = mload 64 + mcopy 1000, 0, 64 + %4 = msize + mstore 1064, %3 + return %4 + """ + _check_pre_post(pre, post) + + +def test_memmerging_partial_overlap(): + """ + Two different copies from overlapping + source range + + [0 128] + [24 88] + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 32 + %3 = mload 64 + %4 = mload 96 + %5 = mload 24 + %6 = mload 56 + mstore 1064, %3 + mstore 1096, %4 + mstore 1000, %1 + mstore 1032, %2 + mstore 2024, %5 + mstore 2056, %6 + stop + """ + + post = """ + _global: + mcopy 1000, 0, 128 + mcopy 2024, 24, 64 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_partial_different_effect(): + """ + Only partial merge possible + because of the generic memory + effect barier + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 32 + %3 = mload 64 + mstore 1000, %1 + mstore 1032, %2 + dloadbytes 2000, 1000, 1000 ; BARRIER + mstore 1064, %3 + stop + """ + + post = """ + _global: + %3 = mload 64 + mcopy 1000, 0, 64 + dloadbytes 2000, 1000, 1000 + mstore 1064, %3 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerge_ok_interval_subset(): + """ + Test subintervals get subsumed by larger intervals + mstore(, mload()) + mcopy(, , 64) + => + mcopy(, , 64) + Because the first mload/mstore is contained in the mcopy + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + mstore 100, %1 + mcopy 100, 0, 33 + stop + """ + + post = """ + _global: + mcopy 100, 0, 33 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_ok_overlap(): + """ + Test for with source overlap + which is ok to do + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 24 + %3 = mload 48 + mstore 1000, %1 + mstore 1024, %2 + mstore 1048, %3 + stop + """ + + post = """ + _global: + mcopy 1000, 0, 80 + stop + """ + + _check_pre_post(pre, post) + + +def test_memmerging_mcopy(): + """ + Test that sequences of mcopy get merged (not just loads/stores) + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + mcopy 1000, 0, 32 + mcopy 1032, 32, 32 + mcopy 1064, 64, 64 + stop + """ + + post = """ + _global: + mcopy 1000, 0, 128 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_mcopy_small(): + """ + Test that sequences of mcopies get merged, and that mcopy of 32 bytes + gets transformed to mload/mstore (saves 1 byte) + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + mcopy 1000, 0, 16 + mcopy 1016, 16, 16 + stop + """ + + post = """ + _global: + %1 = mload 0 + mstore 1000, %1 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_mcopy_weird_bisect(): + """ + Check that bisect_left finds the correct merge + copy(80, 100, 2) + copy(150, 60, 1) + copy(82, 102, 3) + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + mcopy 80, 100, 2 + mcopy 150, 60, 1 + mcopy 82, 102, 3 + stop + """ + + post = """ + _global: + mcopy 150, 60, 1 + mcopy 80, 100, 5 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_mcopy_weird_bisect2(): + """ + Check that bisect_left finds the correct merge + copy(80, 50, 2) + copy(20, 100, 1) + copy(82, 52, 3) + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + mcopy 80, 50, 2 + mcopy 20, 100, 1 + mcopy 82, 52, 3 + stop + """ + + post = """ + _global: + mcopy 20, 100, 1 + mcopy 80, 50, 5 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_allowed_overlapping(): + """ + Test merge of interleaved mload/mstore/mcopy works + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 32 + mcopy 1000, 32, 128 + %2 = mload 0 + mstore 2032, %1 + mstore 2000, %2 + stop + """ + + post = """ + _global: + mcopy 1000, 32, 128 + mcopy 2000, 0, 64 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_allowed_overlapping2(): + if not version_check(begin="cancun"): + return + + pre = """ + _global: + mcopy 1000, 0, 64 + %1 = mload 1032 + mstore 2000, %1 + %2 = mload 1064 + mstore 2032, %2 + stop + """ + + post = """ + _global: + mcopy 1000, 0, 64 + mcopy 2000, 1032, 64 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_unused_mload(): + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 100 + %2 = mload 132 + mstore 64, %2 + + # does not interfere with the mload/mstore merging even though + # it cannot be removed + %3 = mload 32 + + mstore 32, %1 + return %3, %3 + """ + + post = """ + _global: + %3 = mload 32 + mcopy 32, 100, 64 + return %3, %3 + """ + + _check_pre_post(pre, post) + + +def test_memmerging_unused_mload1(): + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 100 + %2 = mload 132 + mstore 0, %1 + + # does not interfere with the mload/mstore merging even though + # it cannot be removed + %3 = mload 32 + + mstore 32, %2 + return %3, %3 + """ + + post = """ + _global: + %3 = mload 32 + mcopy 0, 100, 64 + return %3, %3 + """ + _check_pre_post(pre, post) + + +def test_memmerging_mload_read_after_write_hazard(): + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 100 + %2 = mload 132 + mstore 0, %1 + %3 = mload 32 + mstore 32, %2 + %4 = mload 64 + + ; BARRIER - the load is overriden by existing copy + mstore 1000, %3 + mstore 1032, %4 + stop + """ + + post = """ + _global: + %3 = mload 32 + mcopy 0, 100, 64 + %4 = mload 64 + mstore 1000, %3 + mstore 1032, %4 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_mcopy_read_after_write_hazard(): + if not version_check(begin="cancun"): + return + + pre = """ + _global: + mcopy 1000, 32, 64 + mcopy 2000, 1000, 64 ; BARRIER + mcopy 1064, 96, 64 + stop + """ + _check_no_change(pre) + + +def test_memmerging_write_after_write(): + """ + Check that conflicting writes (from different source locations) + produce a barrier - mstore+mstore version + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 100 + %3 = mload 32 + %4 = mload 132 + mstore 1000, %1 + mstore 1000, %2 ; BARRIER + mstore 1032, %4 + mstore 1032, %3 ; BARRIER + """ + _check_no_change(pre) + + +def test_memmerging_write_after_write_mstore_and_mcopy(): + """ + Check that conflicting writes (from different source locations) + produce a barrier - mstore+mcopy version + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 132 + mstore 1000, %1 + mcopy 1000, 100, 16 ; write barrier + mstore 1032, %2 + mcopy 1016, 116, 64 + stop + """ + _check_no_change(pre) + + +def test_memmerging_write_after_write_only_mcopy(): + """ + Check that conflicting writes (from different source locations) + produce a barrier - mcopy+mcopy version + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + mcopy 1000, 0, 16 + mcopy 1000, 100, 16 ; write barrier + mcopy 1016, 116, 64 + mcopy 1016, 16, 64 + stop + """ + + post = """ + _global: + mcopy 1000, 0, 16 + mcopy 1000, 100, 80 + mcopy 1016, 16, 64 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_not_allowed_overlapping(): + if not version_check(begin="cancun"): + return + + # NOTE: maybe optimization is possible here, to: + # mcopy 2000, 1000, 64 + # mcopy 1000, 0, 128 + pre = """ + _global: + %1 = mload 1000 + %2 = mload 1032 + mcopy 1000, 0, 128 + mstore 2000, %1 ; BARRIER - the mload and mcopy cannot be combined + mstore 2032, %2 + stop + """ + _check_no_change(pre) + + +def test_memmerging_not_allowed_overlapping2(): + if not version_check(begin="cancun"): + return + + # NOTE: maybe optimization is possible here, to: + # mcopy 2000, 1000, 64 + # mcopy 1000, 0, 128 + pre = """ + _global: + %1 = mload 1032 + mcopy 1000, 0, 64 + mstore 2000, %1 + %2 = mload 1064 + mstore 2032, %2 + stop + """ + + _check_no_change(pre) + + +def test_memmerging_existing_copy_overwrite(): + """ + Check that memmerge does not write over source of another copy + """ + if not version_check(begin="cancun"): + return + + pre = """ + _global: + mcopy 1000, 0, 64 + %1 = mload 2000 + + # barrier, write over source of existing copy + mstore 0, %1 + + mcopy 1064, 64, 64 + stop + """ + + _check_no_change(pre) + + +def test_memmerging_double_use(): + if not version_check(begin="cancun"): + return + + pre = """ + _global: + %1 = mload 0 + %2 = mload 32 + mstore 1000, %1 + mstore 1032, %2 + return %1 + """ + + post = """ + _global: + %1 = mload 0 + mcopy 1000, 0, 64 + return %1 + """ + + _check_pre_post(pre, post) + + +def test_memmerging_calldataload(): + pre = """ + _global: + %1 = calldataload 0 + mstore 32, %1 + %2 = calldataload 32 + mstore 64, %2 + stop + """ + + post = """ + _global: + calldatacopy 32, 0, 64 + stop + """ + _check_pre_post(pre, post) + + +def test_memmerging_calldataload_two_intervals_diff_offset(): + """ + Test different calldatacopy sequences are separately merged + """ + pre = """ + _global: + %1 = calldataload 0 + mstore 0, %1 + calldatacopy 32, 32, 64 + %2 = calldataload 0 + mstore 8, %2 + calldatacopy 40, 32, 64 + stop + """ + + post = """ + _global: + calldatacopy 0, 0, 96 + calldatacopy 8, 0, 96 + stop + """ + _check_pre_post(pre, post) + + +def test_memzeroing_1(): + """ + Test of basic memzeroing done with mstore only + """ + + pre = """ + _global: + mstore 32, 0 + mstore 64, 0 + mstore 96, 0 + stop + """ + + post = """ + _global: + %1 = calldatasize + calldatacopy 32, %1, 96 + stop + """ + _check_pre_post(pre, post) + + +def test_memzeroing_2(): + """ + Test of basic memzeroing done with calldatacopy only + + sequence of these instruction will + zero out the memory at destination + %1 = calldatasize + calldatacopy %1 + """ + + pre = """ + _global: + %1 = calldatasize + calldatacopy 64, %1, 128 + %2 = calldatasize + calldatacopy 192, %2, 128 + stop + """ + + post = """ + _global: + %1 = calldatasize + %2 = calldatasize + %3 = calldatasize + calldatacopy 64, %3, 256 + stop + """ + _check_pre_post(pre, post) + + +def test_memzeroing_3(): + """ + Test of basic memzeroing done with combination of + mstores and calldatacopies + """ + + pre = """ + _global: + %1 = calldatasize + calldatacopy 0, %1, 100 + mstore 100, 0 + %2 = calldatasize + calldatacopy 132, %2, 100 + mstore 232, 0 + stop + """ + + post = """ + _global: + %1 = calldatasize + %2 = calldatasize + %3 = calldatasize + calldatacopy 0, %3, 264 + stop + """ + _check_pre_post(pre, post) + + +def test_memzeroing_small_calldatacopy(): + """ + Test of converting calldatacopy of + size 32 into mstore + """ + + pre = """ + _global: + %1 = calldatasize + calldatacopy 0, %1, 32 + stop + """ + + post = """ + _global: + %1 = calldatasize + mstore 0, 0 + stop + """ + _check_pre_post(pre, post) + + +def test_memzeroing_smaller_calldatacopy(): + """ + Test merging smaller (<32) calldatacopies + into either calldatacopy or mstore + """ + + pre = """ + _global: + %1 = calldatasize + calldatacopy 0, %1, 8 + %2 = calldatasize + calldatacopy 8, %2, 16 + %3 = calldatasize + calldatacopy 100, %3, 8 + %4 = calldatasize + calldatacopy 108, %4, 16 + %5 = calldatasize + calldatacopy 124, %5, 8 + stop + """ + + post = """ + _global: + %1 = calldatasize + %2 = calldatasize + %6 = calldatasize + calldatacopy 0, %6, 24 + %3 = calldatasize + %4 = calldatasize + %5 = calldatasize + mstore 100, 0 + stop + """ + _check_pre_post(pre, post) + + +def test_memzeroing_overlap(): + """ + Test of merging overlaping zeroing intervals + + [128 160] + [136 192] + """ + + pre = """ + _global: + mstore 100, 0 + %1 = calldatasize + calldatacopy 108, %1, 56 + stop + """ + + post = """ + _global: + %1 = calldatasize + %2 = calldatasize + calldatacopy 100, %2, 64 + stop + """ + _check_pre_post(pre, post) + + +def test_memzeroing_imposs(): + """ + Test of memzeroing barriers caused + by non constant arguments + """ + + pre = """ + _global: + %1 = param ; abstract location, causes barrier + mstore 32, 0 + mstore %1, 0 + mstore 64, 0 + %2 = calldatasize + calldatacopy %1, %2, 10 + mstore 96, 0 + %3 = calldatasize + calldatacopy 10, %3, %1 + mstore 128, 0 + calldatacopy 10, %1, 10 + mstore 160, 0 + stop + """ + _check_no_change(pre) + + +def test_memzeroing_imposs_effect(): + """ + Test of memzeroing bariers caused + by different effect + """ + + pre = """ + _global: + mstore 32, 0 + dloadbytes 10, 20, 30 ; BARRIER + mstore 64, 0 + stop + """ + _check_no_change(pre) + + +def test_memzeroing_overlaping(): + """ + Test merging overlapping memzeroes (they can be merged + since both result in zeroes being written to destination) + """ + + pre = """ + _global: + mstore 32, 0 + mstore 96, 0 + mstore 32, 0 + mstore 64, 0 + stop + """ + + post = """ + _global: + %1 = calldatasize + calldatacopy 32, %1, 96 + stop + """ + _check_pre_post(pre, post) + + +def test_memzeroing_interleaved(): + """ + Test merging overlapping memzeroes (they can be merged + since both result in zeroes being written to destination) + """ + + pre = """ + _global: + mstore 32, 0 + mstore 1000, 0 + mstore 64, 0 + mstore 1032, 0 + stop + """ + + post = """ + _global: + %1 = calldatasize + calldatacopy 32, %1, 64 + %2 = calldatasize + calldatacopy 1000, %2, 64 + stop + """ + _check_pre_post(pre, post) diff --git a/vyper/utils.py b/vyper/utils.py index d635c78383..66b3551f58 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -524,15 +524,7 @@ def timeit(msg): yield end_time = time.perf_counter() total_time = end_time - start_time - print(f"{msg}: Took {total_time:.4f} seconds") - - -@contextlib.contextmanager -def timer(msg): - t0 = time.time() - yield - t1 = time.time() - print(f"{msg} took {t1 - t0}s") + print(f"{msg}: Took {total_time:.4f} seconds", file=sys.stderr) def annotate_source_code( diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index 7d9404b9ef..a531ffda48 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -17,6 +17,7 @@ FloatAllocas, MakeSSA, Mem2Var, + MemMergePass, RemoveUnusedVariablesPass, SimplifyCFGPass, StoreElimination, @@ -56,6 +57,7 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: MakeSSA(ac, fn).run_pass() SCCP(ac, fn).run_pass() StoreElimination(ac, fn).run_pass() + MemMergePass(ac, fn).run_pass() SimplifyCFGPass(ac, fn).run_pass() AlgebraicOptimizationPass(ac, fn).run_pass() # NOTE: MakeSSA is after algebraic optimization it currently produces diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index 60f4610799..9717ca1626 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -206,7 +206,7 @@ class IRInstruction: opcode: str operands: list[IROperand] - output: Optional[IROperand] + output: Optional[IRVariable] # set of live variables at this instruction liveness: OrderedSet[IRVariable] parent: "IRBasicBlock" @@ -218,7 +218,7 @@ def __init__( self, opcode: str, operands: list[IROperand] | Iterator[IROperand], - output: Optional[IROperand] = None, + output: Optional[IRVariable] = None, ): assert isinstance(opcode, str), "opcode must be an str" assert isinstance(operands, list | Iterator), "operands must be a list" @@ -452,6 +452,8 @@ def __init__(self, label: IRLabel, parent: "IRFunction") -> None: self.out_vars = OrderedSet() self.is_reachable = False + self._garbage_instructions: set[IRInstruction] = set() + def add_cfg_in(self, bb: "IRBasicBlock") -> None: self.cfg_in.add(bb) @@ -526,13 +528,20 @@ def insert_instruction(self, instruction: IRInstruction, index: Optional[int] = instruction.error_msg = self.parent.error_msg self.instructions.insert(index, instruction) + def mark_for_removal(self, instruction: IRInstruction) -> None: + self._garbage_instructions.add(instruction) + + def clear_dead_instructions(self) -> None: + if len(self._garbage_instructions) > 0: + self.instructions = [ + inst for inst in self.instructions if inst not in self._garbage_instructions + ] + self._garbage_instructions.clear() + def remove_instruction(self, instruction: IRInstruction) -> None: assert isinstance(instruction, IRInstruction), "instruction must be an IRInstruction" self.instructions.remove(instruction) - def clear_instructions(self) -> None: - self.instructions = [] - @property def phi_instructions(self) -> Iterator[IRInstruction]: for inst in self.instructions: diff --git a/vyper/venom/passes/__init__.py b/vyper/venom/passes/__init__.py index fcd2aa1f22..2c113027a5 100644 --- a/vyper/venom/passes/__init__.py +++ b/vyper/venom/passes/__init__.py @@ -4,6 +4,7 @@ from .float_allocas import FloatAllocas from .make_ssa import MakeSSA from .mem2var import Mem2Var +from .memmerging import MemMergePass from .normalization import NormalizationPass from .remove_unused_variables import RemoveUnusedVariablesPass from .sccp import SCCP diff --git a/vyper/venom/passes/memmerging.py b/vyper/venom/passes/memmerging.py new file mode 100644 index 0000000000..41216e3041 --- /dev/null +++ b/vyper/venom/passes/memmerging.py @@ -0,0 +1,357 @@ +from bisect import bisect_left +from dataclasses import dataclass + +from vyper.evm.opcodes import version_check +from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis +from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRLiteral, IRVariable +from vyper.venom.effects import Effects +from vyper.venom.passes.base_pass import IRPass + + +@dataclass +class _Interval: + start: int + length: int + + @property + def end(self): + return self.start + self.length + + +@dataclass +class _Copy: + # abstract "copy" operation which contains a list of copy instructions + # and can fuse them into a single copy operation. + dst: int + src: int + length: int + insts: list[IRInstruction] + + @classmethod + def memzero(cls, dst, length, insts): + # factory method to simplify creation of memory zeroing operations + # (which are similar to Copy operations but src is always + # `calldatasize`). choose src=dst, so that can_merge returns True + # for overlapping memzeros. + return cls(dst, dst, length, insts) + + @property + def src_end(self) -> int: + return self.src + self.length + + @property + def dst_end(self) -> int: + return self.dst + self.length + + def src_interval(self) -> _Interval: + return _Interval(self.src, self.length) + + def dst_interval(self) -> _Interval: + return _Interval(self.dst, self.length) + + def overwrites_self_src(self) -> bool: + # return true if dst overlaps src. this is important for blocking + # mcopy batching in certain cases. + return self.overwrites(self.src_interval()) + + def overwrites(self, interval: _Interval) -> bool: + # return true if dst of self overwrites the interval + a = max(self.dst, interval.start) + b = min(self.dst_end, interval.end) + return a < b + + def can_merge(self, other: "_Copy"): + # both source and destination have to be offset by same amount, + # otherwise they do not represent the same copy. e.g. + # Copy(0, 64, 16) + # Copy(11, 74, 16) + if self.src - other.src != self.dst - other.dst: + return False + + # the copies must at least touch each other + if other.dst > self.dst_end: + return False + + return True + + def merge(self, other: "_Copy"): + # merge other into self. e.g. + # Copy(0, 64, 16); Copy(16, 80, 8) => Copy(0, 64, 24) + + assert self.dst <= other.dst, "bad bisect_left" + assert self.can_merge(other) + + new_length = max(self.dst_end, other.dst_end) - self.dst + self.length = new_length + self.insts.extend(other.insts) + + def __repr__(self) -> str: + return f"({self.src}, {self.src_end}, {self.length}, {self.dst}, {self.dst_end})" + + +class MemMergePass(IRPass): + dfg: DFGAnalysis + _copies: list[_Copy] + _loads: dict[IRVariable, int] + + def run_pass(self): + self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) # type: ignore + + for bb in self.function.get_basic_blocks(): + self._handle_bb_memzero(bb) + self._handle_bb(bb, "calldataload", "calldatacopy", allow_dst_overlaps_src=True) + + if version_check(begin="cancun"): + # mcopy is available + self._handle_bb(bb, "mload", "mcopy") + + self.analyses_cache.invalidate_analysis(DFGAnalysis) + self.analyses_cache.invalidate_analysis(LivenessAnalysis) + + def _optimize_copy(self, bb: IRBasicBlock, copy_opcode: str, load_opcode: str): + for copy in self._copies: + copy.insts.sort(key=bb.instructions.index) + + if copy_opcode == "mcopy": + assert not copy.overwrites_self_src() + + pin_inst = None + inst = copy.insts[-1] + if copy.length != 32: + inst.output = None + inst.opcode = copy_opcode + inst.operands = [IRLiteral(copy.length), IRLiteral(copy.src), IRLiteral(copy.dst)] + elif inst.opcode == "mstore": + # we already have a load which is the val for this mstore; + # leave it in place. + var, _ = inst.operands + assert isinstance(var, IRVariable) # help mypy + pin_inst = self.dfg.get_producing_instruction(var) + assert pin_inst is not None # help mypy + + else: + # we are converting an mcopy into an mload+mstore (mload+mstore + # is 1 byte smaller than mcopy). + index = inst.parent.instructions.index(inst) + var = bb.parent.get_next_variable() + load = IRInstruction(load_opcode, [IRLiteral(copy.src)], output=var) + inst.parent.insert_instruction(load, index) + + inst.output = None + inst.opcode = "mstore" + inst.operands = [var, IRLiteral(copy.dst)] + + for inst in copy.insts[:-1]: + if inst.opcode == load_opcode: + if inst is pin_inst: + continue + + # if the load is used by any instructions besides the ones + # we are removing, we can't delete it. (in the future this + # may be handled by "remove unused effects" pass). + assert isinstance(inst.output, IRVariable) # help mypy + uses = self.dfg.get_uses(inst.output) + if not all(use in copy.insts for use in uses): + continue + + bb.mark_for_removal(inst) + + self._copies.clear() + self._loads.clear() + + def _write_after_write_hazard(self, new_copy: _Copy) -> bool: + for copy in self._copies: + # note, these are the same: + # - new_copy.overwrites(copy.dst_interval()) + # - copy.overwrites(new_copy.dst_interval()) + if new_copy.overwrites(copy.dst_interval()) and not ( + copy.can_merge(new_copy) or new_copy.can_merge(copy) + ): + return True + return False + + def _read_after_write_hazard(self, new_copy: _Copy) -> bool: + new_copies = self._copies + [new_copy] + + # new copy would overwrite memory that + # needs to be read to optimize copy + if any(new_copy.overwrites(copy.src_interval()) for copy in new_copies): + return True + + # existing copies would overwrite memory that the + # new copy would need + if self._overwrites(new_copy.src_interval()): + return True + + return False + + def _find_insertion_point(self, new_copy: _Copy): + return bisect_left(self._copies, new_copy.dst, key=lambda c: c.dst) + + def _add_copy(self, new_copy: _Copy): + index = self._find_insertion_point(new_copy) + self._copies.insert(index, new_copy) + + i = max(index - 1, 0) + while i < min(index + 1, len(self._copies) - 1): + if self._copies[i].can_merge(self._copies[i + 1]): + self._copies[i].merge(self._copies[i + 1]) + del self._copies[i + 1] + else: + i += 1 + + def _overwrites(self, read_interval: _Interval) -> bool: + # check if any of self._copies tramples the interval + + # could use bisect_left to optimize, but it's harder to reason about + return any(c.overwrites(read_interval) for c in self._copies) + + def _handle_bb( + self, + bb: IRBasicBlock, + load_opcode: str, + copy_opcode: str, + allow_dst_overlaps_src: bool = False, + ): + self._loads = {} + self._copies = [] + + def _barrier(): + self._optimize_copy(bb, copy_opcode, load_opcode) + + # copy in necessary because there is a possibility + # of insertion in optimizations + for inst in bb.instructions.copy(): + if inst.opcode == load_opcode: + src_op = inst.operands[0] + if not isinstance(src_op, IRLiteral): + _barrier() + continue + + read_interval = _Interval(src_op.value, 32) + + # we will read from this memory so we need to put barier + if not allow_dst_overlaps_src and self._overwrites(read_interval): + _barrier() + + assert inst.output is not None + self._loads[inst.output] = src_op.value + + elif inst.opcode == "mstore": + var, dst = inst.operands + + if not isinstance(var, IRVariable) or not isinstance(dst, IRLiteral): + _barrier() + continue + + if var not in self._loads: + _barrier() + continue + + src_ptr = self._loads[var] + load_inst = self.dfg.get_producing_instruction(var) + assert load_inst is not None # help mypy + n_copy = _Copy(dst.value, src_ptr, 32, [inst, load_inst]) + + if self._write_after_write_hazard(n_copy): + _barrier() + # no continue needed, we have not invalidated the loads dict + + # check if the new copy does not overwrites existing data + if not allow_dst_overlaps_src and self._read_after_write_hazard(n_copy): + _barrier() + # this continue is necessary because we have invalidated + # the _loads dict, so src_ptr is no longer valid. + continue + self._add_copy(n_copy) + + elif inst.opcode == copy_opcode: + if not all(isinstance(op, IRLiteral) for op in inst.operands): + _barrier() + continue + + length, src, dst = inst.operands + n_copy = _Copy(dst.value, src.value, length.value, [inst]) + + if self._write_after_write_hazard(n_copy): + _barrier() + # check if the new copy does not overwrites existing data + if not allow_dst_overlaps_src and self._read_after_write_hazard(n_copy): + _barrier() + self._add_copy(n_copy) + + elif _volatile_memory(inst): + _barrier() + + _barrier() + bb.clear_dead_instructions() + + # optimize memzeroing operations + def _optimize_memzero(self, bb: IRBasicBlock): + for copy in self._copies: + inst = copy.insts[-1] + if copy.length == 32: + inst.opcode = "mstore" + inst.operands = [IRLiteral(0), IRLiteral(copy.dst)] + else: + index = bb.instructions.index(inst) + calldatasize = bb.parent.get_next_variable() + bb.insert_instruction(IRInstruction("calldatasize", [], output=calldatasize), index) + + inst.output = None + inst.opcode = "calldatacopy" + inst.operands = [IRLiteral(copy.length), calldatasize, IRLiteral(copy.dst)] + + for inst in copy.insts[:-1]: + bb.mark_for_removal(inst) + + self._copies.clear() + self._loads.clear() + + def _handle_bb_memzero(self, bb: IRBasicBlock): + self._loads = {} + self._copies = [] + + def _barrier(): + self._optimize_memzero(bb) + + # copy in necessary because there is a possibility + # of insertion in optimizations + for inst in bb.instructions.copy(): + if inst.opcode == "mstore": + val = inst.operands[0] + dst = inst.operands[1] + is_zero_literal = isinstance(val, IRLiteral) and val.value == 0 + if not (isinstance(dst, IRLiteral) and is_zero_literal): + _barrier() + continue + n_copy = _Copy.memzero(dst.value, 32, [inst]) + assert not self._write_after_write_hazard(n_copy) + self._add_copy(n_copy) + elif inst.opcode == "calldatacopy": + length, var, dst = inst.operands + if not isinstance(var, IRVariable): + _barrier() + continue + if not isinstance(dst, IRLiteral) or not isinstance(length, IRLiteral): + _barrier() + continue + src_inst = self.dfg.get_producing_instruction(var) + assert src_inst is not None, f"bad variable {var}" + if src_inst.opcode != "calldatasize": + _barrier() + continue + n_copy = _Copy.memzero(dst.value, length.value, [inst]) + assert not self._write_after_write_hazard(n_copy) + self._add_copy(n_copy) + elif _volatile_memory(inst): + _barrier() + continue + + _barrier() + bb.clear_dead_instructions() + + +def _volatile_memory(inst): + inst_effects = inst.get_read_effects() | inst.get_write_effects() + return Effects.MEMORY in inst_effects or Effects.MSIZE in inst_effects From f6030fb2fd9e6b9c54349dd3ae23d8013c892944 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 19 Dec 2024 11:08:36 -0500 Subject: [PATCH 42/70] refactor[codegen]: add profiling utils (#4412) add two new profiling utils, `profileit()` and `cumtimeit()` which run cProfile and printout cumulative time spent in a context, respectively. the functions both return a context manager, so it can be used both in a with statement or as a function decorator, e.g.: ```python @profileit() def foo(): ... @cumtimeit("foo") def foo(): ... ``` in a with statement ``` with profileit(): foo() with cumtimeit("foo"): foo() ``` neither of these should be used except for in local dev work, so they are ignored in coverage, and a warning is printed out if they are used as a reminder to remove them before pushing. --- vyper/utils.py | 69 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/vyper/utils.py b/vyper/utils.py index 66b3551f58..9b5bfbef62 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -519,7 +519,7 @@ def indent(text: str, indent_chars: Union[str, List[str]] = " ", level: int = 1) @contextlib.contextmanager -def timeit(msg): +def timeit(msg): # pragma: nocover start_time = time.perf_counter() yield end_time = time.perf_counter() @@ -527,6 +527,73 @@ def timeit(msg): print(f"{msg}: Took {total_time:.4f} seconds", file=sys.stderr) +_CUMTIMES = None + + +def _dump_cumtime(): # pragma: nocover + global _CUMTIMES + for msg, total_time in _CUMTIMES.items(): + print(f"{msg}: Cumulative time {total_time:.4f} seconds", file=sys.stderr) + + +@contextlib.contextmanager +def cumtimeit(msg): # pragma: nocover + import atexit + from collections import defaultdict + + global _CUMTIMES + + if _CUMTIMES is None: + warnings.warn("timing code, disable me before pushing!", stacklevel=2) + _CUMTIMES = defaultdict(int) + atexit.register(_dump_cumtime) + + start_time = time.perf_counter() + yield + end_time = time.perf_counter() + total_time = end_time - start_time + _CUMTIMES[msg] += total_time + + +_PROF = None + + +def _dump_profile(): # pragma: nocover + global _PROF + + _PROF.disable() # don't profile dumping stats + _PROF.dump_stats("stats") + + from pstats import Stats + + stats = Stats("stats", stream=sys.stderr) + stats.sort_stats("time") + stats.print_stats() + + +@contextlib.contextmanager +def profileit(): # pragma: nocover + """ + Helper function for local dev use, is not intended to ever be run in + production build + """ + import atexit + from cProfile import Profile + + global _PROF + if _PROF is None: + warnings.warn("profiling code, disable me before pushing!", stacklevel=2) + _PROF = Profile() + _PROF.disable() + atexit.register(_dump_profile) + + try: + _PROF.enable() + yield + finally: + _PROF.disable() + + def annotate_source_code( source_code: str, lineno: int, From eee31e7052264698d53805188857e0e90da3736d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 19 Dec 2024 12:24:06 -0500 Subject: [PATCH 43/70] refactor[venom]: make venom repr parseable (#4402) make the `__repr__()` implementations for venom data structures (`IRContext`, `IRFunction, `IRBasicBlock`) emit strings which will round-trip through the parser. for labels generated in the frontend which are not necessarily valid identifiers (e.g. `"internal 5 foo()"`), these are represented as escaped strings. the expedient way to implement this was to simply use `json.loads` / `json.dumps`; there did not seem to be any convenient stdlib or lark function to do this. since this adds grammar complexity, the other method that was considered was to map all labels to valid identifiers in `ir_node_to_venom.py`. but this approach seems easier than cleaning up all the non-identifier labels generated by the frontend; plus, being able to have arbitrary strings in labels seems like it will come in handy during debugging some time. a couple other grammar updates/fixes: - update instruction order in the text format for `phi` and `invoke` - ensure instructions are terminated with newline (otherwise they can continue slurping tokens from the next line). - allow signed ints inside `CONST` nodes as `IRLiteral` accepts negative numbers misc/refactor: - remove a dead function (`str_short()`). - remove a dead branch in `ir_node_to_venom.py` - when optimization level is set to `CODESIZE`, the roundtrip test is set to xfail, as the data section contains bytestrings (which do not parse yet). --- tests/functional/venom/test_venom_repr.py | 36 ++++++++++++++++ tests/venom_utils.py | 1 - vyper/venom/basicblock.py | 48 ++++++++++----------- vyper/venom/context.py | 6 +-- vyper/venom/function.py | 11 +++-- vyper/venom/ir_node_to_venom.py | 5 +-- vyper/venom/parser.py | 52 +++++++++++++++++------ vyper/venom/venom_to_assembly.py | 6 +-- 8 files changed, 111 insertions(+), 54 deletions(-) create mode 100644 tests/functional/venom/test_venom_repr.py diff --git a/tests/functional/venom/test_venom_repr.py b/tests/functional/venom/test_venom_repr.py new file mode 100644 index 0000000000..caa315dbbb --- /dev/null +++ b/tests/functional/venom/test_venom_repr.py @@ -0,0 +1,36 @@ +import glob + +import pytest + +from tests.venom_utils import assert_ctx_eq, parse_venom +from vyper.compiler import compile_code +from vyper.compiler.settings import OptimizationLevel +from vyper.venom.context import IRContext + +""" +Check that venom text format round-trips through parser +""" + + +def get_example_vy_filenames(): + return glob.glob("**/*.vy", root_dir="examples/", recursive=True) + + +@pytest.mark.parametrize("vy_filename", get_example_vy_filenames()) +def test_round_trip(vy_filename, optimize, request): + if optimize == OptimizationLevel.CODESIZE: + # codesize optimization issues things like `db b"\x12\x34"` which we + # don't handle. + request.node.add_marker(pytest.mark.xfail(strict=False, reason="unimplemented in parser")) + + path = f"examples/{vy_filename}" + with open(path) as f: + vyper_source = f.read() + + out = compile_code(vyper_source, output_formats=["bb_runtime"]) + bb_runtime = out["bb_runtime"] + venom_code = IRContext.__repr__(bb_runtime) + + ctx = parse_venom(venom_code) + + assert_ctx_eq(bb_runtime, ctx) diff --git a/tests/venom_utils.py b/tests/venom_utils.py index d4536e8bf7..a67df6c275 100644 --- a/tests/venom_utils.py +++ b/tests/venom_utils.py @@ -36,7 +36,6 @@ def assert_fn_eq(fn1: IRFunction, fn2: IRFunction): def assert_ctx_eq(ctx1: IRContext, ctx2: IRContext): - assert ctx1.last_label == ctx2.last_label assert len(ctx1.functions) == len(ctx2.functions) for label1, fn1 in ctx1.functions.items(): assert label1 in ctx2.functions diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index 9717ca1626..4b8eec2263 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -1,3 +1,5 @@ +import json +import re from typing import TYPE_CHECKING, Any, Iterator, Optional, Union import vyper.venom.effects as effects @@ -105,7 +107,7 @@ def __init__(self, line_no: int, src: str) -> None: def __repr__(self) -> str: src = self.src if self.src else "" - return f"\t# line {self.line_no}: {src}".expandtabs(20) + return f"\t; line {self.line_no}: {src}".expandtabs(20) class IROperand: @@ -189,10 +191,19 @@ class IRLabel(IROperand): value: str def __init__(self, value: str, is_symbol: bool = False) -> None: - assert isinstance(value, str), "value must be an str" + assert isinstance(value, str), f"not a str: {value} ({type(value)})" + assert len(value) > 0 super().__init__(value) self.is_symbol = is_symbol + _IS_IDENTIFIER = re.compile("[0-9a-zA-Z_]*") + + def __repr__(self): + if self.__class__._IS_IDENTIFIER.fullmatch(self.value): + return self.value + + return json.dumps(self.value) # escape it + class IRInstruction: """ @@ -366,20 +377,6 @@ def get_ast_source(self) -> Optional[IRnode]: return inst.ast_source return self.parent.parent.ast_source - def str_short(self) -> str: - s = "" - if self.output: - s += f"{self.output} = " - opcode = f"{self.opcode} " if self.opcode != "store" else "" - s += opcode - operands = self.operands - if opcode not in ["jmp", "jnz", "invoke"]: - operands = list(reversed(operands)) - s += ", ".join( - [(f"label %{op}" if isinstance(op, IRLabel) else str(op)) for op in operands] - ) - return s - def __repr__(self) -> str: s = "" if self.output: @@ -387,14 +384,15 @@ def __repr__(self) -> str: opcode = f"{self.opcode} " if self.opcode != "store" else "" s += opcode operands = self.operands - if opcode not in ("jmp", "jnz", "invoke"): + if self.opcode == "invoke": + operands = [operands[0]] + list(reversed(operands[1:])) + elif self.opcode not in ("jmp", "jnz", "phi"): operands = reversed(operands) # type: ignore - s += ", ".join( - [(f"label %{op}" if isinstance(op, IRLabel) else str(op)) for op in operands] - ) + + s += ", ".join([(f"@{op}" if isinstance(op, IRLabel) else str(op)) for op in operands]) if self.annotation: - s += f" <{self.annotation}>" + s += f" ; {self.annotation}" return f"{s: <30}" @@ -659,10 +657,8 @@ def copy(self): return bb def __repr__(self) -> str: - s = ( - f"{repr(self.label)}: IN={[bb.label for bb in self.cfg_in]}" - f" OUT={[bb.label for bb in self.cfg_out]} => {self.out_vars}\n" - ) + s = f"{self.label}: ; IN={[bb.label for bb in self.cfg_in]}" + s += f" OUT={[bb.label for bb in self.cfg_out]} => {self.out_vars}\n" for instruction in self.instructions: - s += f" {str(instruction).strip()}\n" + s += f" {str(instruction).strip()}\n" return s diff --git a/vyper/venom/context.py b/vyper/venom/context.py index 0b0252d976..391da3e189 100644 --- a/vyper/venom/context.py +++ b/vyper/venom/context.py @@ -62,14 +62,14 @@ def as_graph(self) -> str: return "\n".join(s) def __repr__(self) -> str: - s = ["IRContext:"] + s = [] for fn in self.functions.values(): s.append(fn.__repr__()) s.append("\n") if len(self.data_segment) > 0: - s.append("\nData segment:") + s.append("\n[data]") for inst in self.data_segment: - s.append(f"{inst}") + s.append(f" {inst}") return "\n".join(s) diff --git a/vyper/venom/function.py b/vyper/venom/function.py index 2372f8ba52..5709f65be4 100644 --- a/vyper/venom/function.py +++ b/vyper/venom/function.py @@ -1,3 +1,4 @@ +import textwrap from typing import Iterator, Optional from vyper.codegen.ir_node import IRnode @@ -41,7 +42,7 @@ def append_basic_block(self, bb: IRBasicBlock): Append basic block to function. """ assert isinstance(bb, IRBasicBlock), bb - assert bb.label.name not in self._basic_block_dict + assert bb.label.name not in self._basic_block_dict, bb.label self._basic_block_dict[bb.label.name] = bb def remove_basic_block(self, bb: IRBasicBlock): @@ -222,7 +223,9 @@ def _make_label(bb): return "\n".join(ret) def __repr__(self) -> str: - str = f"IRFunction: {self.name}\n" + ret = f"function {self.name} {{\n" for bb in self.get_basic_blocks(): - str += f"{bb}\n" - return str.strip() + bb_str = textwrap.indent(str(bb), " ") + ret += f"{bb_str}\n" + ret = ret.strip() + "\n}" + return ret.strip() diff --git a/vyper/venom/ir_node_to_venom.py b/vyper/venom/ir_node_to_venom.py index 782309d841..5454b994b3 100644 --- a/vyper/venom/ir_node_to_venom.py +++ b/vyper/venom/ir_node_to_venom.py @@ -369,10 +369,7 @@ def _convert_ir_bb(fn, ir, symbols): label = IRLabel(ir.args[0].value) ctx.append_data("dbname", [label]) for c in ir.args[1:]: - if isinstance(c, int): - assert 0 <= c <= 255, "data with invalid size" - ctx.append_data("db", [c]) # type: ignore - elif isinstance(c.value, bytes): + if isinstance(c.value, bytes): ctx.append_data("db", [c.value]) # type: ignore elif isinstance(c, IRnode): data = _convert_ir_bb(fn, c, symbols) diff --git a/vyper/venom/parser.py b/vyper/venom/parser.py index f8ba33e522..9ab223179e 100644 --- a/vyper/venom/parser.py +++ b/vyper/venom/parser.py @@ -1,3 +1,5 @@ +import json + from lark import Lark, Transformer from vyper.venom.basicblock import ( @@ -11,13 +13,14 @@ from vyper.venom.context import IRContext from vyper.venom.function import IRFunction -VENOM_PARSER = Lark( - """ +VENOM_GRAMMAR = """ %import common.CNAME %import common.DIGIT %import common.LETTER %import common.WS %import common.INT + %import common.SIGNED_INT + %import common.ESCAPED_STRING # Allow multiple comment styles COMMENT: ";" /[^\\n]*/ | "//" /[^\\n]*/ | "#" /[^\\n]*/ @@ -26,13 +29,13 @@ # TODO: consider making entry block implicit, e.g. # `"{" instruction+ block* "}"` - function: "function" NAME "{" block* "}" + function: "function" LABEL_IDENT "{" block* "}" data_section: "[data]" instruction* - block: NAME ":" statement* + block: LABEL_IDENT ":" "\\n" statement* - statement: instruction | assignment + statement: (instruction | assignment) "\\n" assignment: VAR_IDENT "=" expr expr: instruction | operand instruction: OPCODE operands_list? @@ -41,16 +44,22 @@ operand: VAR_IDENT | CONST | LABEL - CONST: INT + CONST: SIGNED_INT OPCODE: CNAME - VAR_IDENT: "%" NAME - LABEL: "@" NAME + VAR_IDENT: "%" (DIGIT|LETTER|"_"|":")+ + + # handy for identifier to be an escaped string sometimes + # (especially for machine-generated labels) + LABEL_IDENT: (NAME | ESCAPED_STRING) + LABEL: "@" LABEL_IDENT + NAME: (DIGIT|LETTER|"_")+ %ignore WS %ignore COMMENT """ -) + +VENOM_PARSER = Lark(VENOM_GRAMMAR) def _set_last_var(fn: IRFunction): @@ -83,6 +92,15 @@ def _ensure_terminated(bb): # TODO: raise error if still not terminated. +def _unescape(s: str): + """ + Unescape the escaped string. This is the inverse of `IRLabel.__repr__()`. + """ + if s.startswith('"'): + return json.loads(s) + return s + + class _DataSegment: def __init__(self, instructions): self.instructions = instructions @@ -100,7 +118,7 @@ def start(self, children) -> IRContext: fn._basic_block_dict.clear() for block_name, instructions in blocks: - bb = IRBasicBlock(IRLabel(block_name), fn) + bb = IRBasicBlock(IRLabel(block_name, True), fn) fn.append_basic_block(bb) for instruction in instructions: @@ -152,8 +170,12 @@ def instruction(self, children) -> IRInstruction: # reverse operands, venom internally represents top of stack # as rightmost operand - if opcode not in ("jmp", "jnz", "invoke", "phi"): - # special cases: operands with labels look better un-reversed + if opcode == "invoke": + # reverse stack arguments but not label arg + # invoke + operands = [operands[0]] + list(reversed(operands[1:])) + # special cases: operands with labels look better un-reversed + elif opcode not in ("jmp", "jnz", "phi"): operands.reverse() return IRInstruction(opcode, operands) @@ -166,8 +188,12 @@ def operand(self, children) -> IROperand: def OPCODE(self, token): return token.value + def LABEL_IDENT(self, label) -> str: + return _unescape(label) + def LABEL(self, label) -> IRLabel: - return IRLabel(label[1:]) + label = _unescape(label[1:]) + return IRLabel(label, True) def VAR_IDENT(self, var_ident) -> IRVariable: return IRVariable(var_ident[1:]) diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 9b52b842ba..5cf37006a4 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -185,8 +185,8 @@ def generate_evm(self, no_optimize: bool = False) -> list[str]: data_segments: dict = dict() for inst in ctx.data_segment: if inst.opcode == "dbname": - label = inst.operands[0].value - data_segments[label] = [DataHeader(f"_sym_{label}")] + label = inst.operands[0] + data_segments[label] = [DataHeader(f"_sym_{label.value}")] elif inst.opcode == "db": data = inst.operands[0] if isinstance(data, IRLabel): @@ -293,7 +293,7 @@ def _generate_evm_for_basicblock_r( asm = [] # assembly entry point into the block - asm.append(f"_sym_{basicblock.label}") + asm.append(f"_sym_{basicblock.label.value}") asm.append("JUMPDEST") if len(basicblock.cfg_in) == 1: From 724559a343d98dcedfdd9b38e0df10832ad19885 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 20 Dec 2024 12:00:47 -0500 Subject: [PATCH 44/70] feat[venom]: propagate `dload` instruction to venom (#4410) this commit propagates `dload` instructions to venom from the frontend. this improves our ability to merge `dload/mstore` instructions. then, after the LowerDload pass, we have stripped both `dload` and `dloadbytes` instructions from venom. --- tests/unit/compiler/venom/test_memmerging.py | 38 ++++++++++-------- vyper/venom/__init__.py | 2 + vyper/venom/effects.py | 1 + vyper/venom/ir_node_to_venom.py | 19 +-------- vyper/venom/passes/__init__.py | 1 + vyper/venom/passes/lower_dload.py | 42 ++++++++++++++++++++ vyper/venom/passes/memmerging.py | 3 +- vyper/venom/venom_to_assembly.py | 3 +- 8 files changed, 73 insertions(+), 36 deletions(-) create mode 100644 vyper/venom/passes/lower_dload.py diff --git a/tests/unit/compiler/venom/test_memmerging.py b/tests/unit/compiler/venom/test_memmerging.py index b7a4d33805..d309752621 100644 --- a/tests/unit/compiler/venom/test_memmerging.py +++ b/tests/unit/compiler/venom/test_memmerging.py @@ -18,6 +18,10 @@ def _check_no_change(pre): _check_pre_post(pre, pre) +# for parametrizing tests +LOAD_COPY = [("dload", "dloadbytes"), ("calldataload", "calldatacopy")] + + def test_memmerging(): """ Basic memory merge test @@ -762,43 +766,45 @@ def test_memmerging_double_use(): _check_pre_post(pre, post) -def test_memmerging_calldataload(): - pre = """ +@pytest.mark.parametrize("load_opcode,copy_opcode", LOAD_COPY) +def test_memmerging_load(load_opcode, copy_opcode): + pre = f""" _global: - %1 = calldataload 0 + %1 = {load_opcode} 0 mstore 32, %1 - %2 = calldataload 32 + %2 = {load_opcode} 32 mstore 64, %2 stop """ - post = """ + post = f""" _global: - calldatacopy 32, 0, 64 + {copy_opcode} 32, 0, 64 stop """ _check_pre_post(pre, post) -def test_memmerging_calldataload_two_intervals_diff_offset(): +@pytest.mark.parametrize("load_opcode,copy_opcode", LOAD_COPY) +def test_memmerging_two_intervals_diff_offset(load_opcode, copy_opcode): """ - Test different calldatacopy sequences are separately merged + Test different dloadbytes/calldatacopy sequences are separately merged """ - pre = """ + pre = f""" _global: - %1 = calldataload 0 + %1 = {load_opcode} 0 mstore 0, %1 - calldatacopy 32, 32, 64 - %2 = calldataload 0 + {copy_opcode} 32, 32, 64 + %2 = {load_opcode} 0 mstore 8, %2 - calldatacopy 40, 32, 64 + {copy_opcode} 40, 32, 64 stop """ - post = """ + post = f""" _global: - calldatacopy 0, 0, 96 - calldatacopy 8, 0, 96 + {copy_opcode} 0, 0, 96 + {copy_opcode} 8, 0, 96 stop """ _check_pre_post(pre, post) diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index a531ffda48..1aac42e4fb 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -15,6 +15,7 @@ BranchOptimizationPass, DFTPass, FloatAllocas, + LowerDloadPass, MakeSSA, Mem2Var, MemMergePass, @@ -59,6 +60,7 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: StoreElimination(ac, fn).run_pass() MemMergePass(ac, fn).run_pass() SimplifyCFGPass(ac, fn).run_pass() + LowerDloadPass(ac, fn).run_pass() AlgebraicOptimizationPass(ac, fn).run_pass() # NOTE: MakeSSA is after algebraic optimization it currently produces # smaller code by adding some redundant phi nodes. This is not a diff --git a/vyper/venom/effects.py b/vyper/venom/effects.py index 97cffe2cb2..bbda481e14 100644 --- a/vyper/venom/effects.py +++ b/vyper/venom/effects.py @@ -44,6 +44,7 @@ def __iter__(self): "invoke": ALL, # could be smarter, look up the effects of the invoked function "log": LOG, "dloadbytes": MEMORY, + "dload": MEMORY, "returndatacopy": MEMORY, "calldatacopy": MEMORY, "codecopy": MEMORY, diff --git a/vyper/venom/ir_node_to_venom.py b/vyper/venom/ir_node_to_venom.py index 5454b994b3..4dcc5ee4e6 100644 --- a/vyper/venom/ir_node_to_venom.py +++ b/vyper/venom/ir_node_to_venom.py @@ -4,7 +4,6 @@ from vyper.codegen.ir_node import IRnode from vyper.evm.opcodes import get_opcodes -from vyper.utils import MemoryPositions from vyper.venom.basicblock import ( IRBasicBlock, IRInstruction, @@ -67,6 +66,8 @@ "mload", "iload", "istore", + "dload", + "dloadbytes", "sload", "sstore", "tload", @@ -400,22 +401,6 @@ def _convert_ir_bb(fn, ir, symbols): else: bb.append_instruction("jmp", label) - elif ir.value == "dload": - arg_0 = _convert_ir_bb(fn, ir.args[0], symbols) - bb = fn.get_basic_block() - src = bb.append_instruction("add", arg_0, IRLabel("code_end")) - - bb.append_instruction("dloadbytes", 32, src, MemoryPositions.FREE_VAR_SPACE) - return bb.append_instruction("mload", MemoryPositions.FREE_VAR_SPACE) - - elif ir.value == "dloadbytes": - dst, src_offset, len_ = _convert_ir_bb_list(fn, ir.args, symbols) - - bb = fn.get_basic_block() - src = bb.append_instruction("add", src_offset, IRLabel("code_end")) - bb.append_instruction("dloadbytes", len_, src, dst) - return None - elif ir.value == "mstore": # some upstream code depends on reversed order of evaluation -- # to fix upstream. diff --git a/vyper/venom/passes/__init__.py b/vyper/venom/passes/__init__.py index 2c113027a5..fe1e387c56 100644 --- a/vyper/venom/passes/__init__.py +++ b/vyper/venom/passes/__init__.py @@ -2,6 +2,7 @@ from .branch_optimization import BranchOptimizationPass from .dft import DFTPass from .float_allocas import FloatAllocas +from .lower_dload import LowerDloadPass from .make_ssa import MakeSSA from .mem2var import Mem2Var from .memmerging import MemMergePass diff --git a/vyper/venom/passes/lower_dload.py b/vyper/venom/passes/lower_dload.py new file mode 100644 index 0000000000..c863a1b7c7 --- /dev/null +++ b/vyper/venom/passes/lower_dload.py @@ -0,0 +1,42 @@ +from vyper.utils import MemoryPositions +from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis +from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRLabel, IRLiteral +from vyper.venom.passes.base_pass import IRPass + + +class LowerDloadPass(IRPass): + """ + Lower dload and dloadbytes instructions + """ + + def run_pass(self): + for bb in self.function.get_basic_blocks(): + self._handle_bb(bb) + self.analyses_cache.invalidate_analysis(LivenessAnalysis) + self.analyses_cache.invalidate_analysis(DFGAnalysis) + + def _handle_bb(self, bb: IRBasicBlock): + fn = bb.parent + for idx, inst in enumerate(bb.instructions): + if inst.opcode == "dload": + (ptr,) = inst.operands + var = fn.get_next_variable() + bb.insert_instruction( + IRInstruction("add", [ptr, IRLabel("code_end")], output=var), index=idx + ) + idx += 1 + dst = IRLiteral(MemoryPositions.FREE_VAR_SPACE) + bb.insert_instruction( + IRInstruction("codecopy", [IRLiteral(32), var, dst]), index=idx + ) + + inst.opcode = "mload" + inst.operands = [dst] + elif inst.opcode == "dloadbytes": + _, src, _ = inst.operands + code_ptr = fn.get_next_variable() + bb.insert_instruction( + IRInstruction("add", [src, IRLabel("code_end")], output=code_ptr), index=idx + ) + inst.opcode = "codecopy" + inst.operands[1] = code_ptr diff --git a/vyper/venom/passes/memmerging.py b/vyper/venom/passes/memmerging.py index 41216e3041..2e5ee46b84 100644 --- a/vyper/venom/passes/memmerging.py +++ b/vyper/venom/passes/memmerging.py @@ -100,6 +100,7 @@ def run_pass(self): for bb in self.function.get_basic_blocks(): self._handle_bb_memzero(bb) self._handle_bb(bb, "calldataload", "calldatacopy", allow_dst_overlaps_src=True) + self._handle_bb(bb, "dload", "dloadbytes", allow_dst_overlaps_src=True) if version_check(begin="cancun"): # mcopy is available @@ -117,7 +118,7 @@ def _optimize_copy(self, bb: IRBasicBlock, copy_opcode: str, load_opcode: str): pin_inst = None inst = copy.insts[-1] - if copy.length != 32: + if copy.length != 32 or load_opcode == "dload": inst.output = None inst.opcode = copy_opcode inst.operands = [IRLiteral(copy.length), IRLiteral(copy.src), IRLiteral(copy.dst)] diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 5cf37006a4..e136932f51 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -41,6 +41,7 @@ "calldatacopy", "mcopy", "calldataload", + "codecopy", "gas", "gasprice", "gaslimit", @@ -472,8 +473,6 @@ def _generate_evm_for_instruction( pass elif opcode == "dbname": pass - elif opcode in ["codecopy", "dloadbytes"]: - assembly.append("CODECOPY") elif opcode == "jnz": # jump if not zero if_nonzero_label = inst.operands[1] From 9ff9080fdea8f6ac6aaf5b8db7fc5b3a30f758de Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 20 Dec 2024 12:19:51 -0500 Subject: [PATCH 45/70] feat[venom]: remove special cases in store elimination (#4413) remove special cases in store elimination, allowing elimination of `%ret_ofst` and `%ret_size` store chains. this was previously to protect correctness in `mem2var`, but it no longer seems to be needed. running an additional store elimination before `mem2var` results in a slight bytecode size improvement, since it allows more memory locations to be promoted to stack variables. --- vyper/venom/__init__.py | 1 + vyper/venom/passes/store_elimination.py | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index 1aac42e4fb..89a4534a4c 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -54,6 +54,7 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: SimplifyCFGPass(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() + StoreElimination(ac, fn).run_pass() Mem2Var(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() SCCP(ac, fn).run_pass() diff --git a/vyper/venom/passes/store_elimination.py b/vyper/venom/passes/store_elimination.py index 559205adc8..97ab424cd6 100644 --- a/vyper/venom/passes/store_elimination.py +++ b/vyper/venom/passes/store_elimination.py @@ -16,12 +16,6 @@ def run_pass(self): for var, inst in dfg.outputs.items(): if inst.opcode != "store": continue - if not isinstance(inst.operands[0], IRVariable): - continue - if inst.operands[0].name in ["%ret_ofst", "%ret_size"]: - continue - if inst.output.name in ["%ret_ofst", "%ret_size"]: - continue self._process_store(dfg, inst, var, inst.operands[0]) self.analyses_cache.invalidate_analysis(LivenessAnalysis) From a56d79d34c2174b525d146bf098aa0c7970f76e3 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 20 Dec 2024 12:33:54 -0500 Subject: [PATCH 46/70] feat[venom]: add codesize optimization pass (#4333) add basic codesize optimization pass for venom which strengthens large literals to evm computations which are shorter --------- Co-authored-by: Hodan --- .../compiler/venom/test_literals_codesize.py | 117 ++++++++++++++++++ vyper/utils.py | 5 + vyper/venom/__init__.py | 5 + vyper/venom/passes/__init__.py | 1 + vyper/venom/passes/literals_codesize.py | 58 +++++++++ vyper/venom/passes/sccp/eval.py | 8 +- 6 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 tests/unit/compiler/venom/test_literals_codesize.py create mode 100644 vyper/venom/passes/literals_codesize.py diff --git a/tests/unit/compiler/venom/test_literals_codesize.py b/tests/unit/compiler/venom/test_literals_codesize.py new file mode 100644 index 0000000000..4de4d9de64 --- /dev/null +++ b/tests/unit/compiler/venom/test_literals_codesize.py @@ -0,0 +1,117 @@ +import pytest + +from vyper.utils import evm_not +from vyper.venom.analysis import IRAnalysesCache +from vyper.venom.basicblock import IRLiteral +from vyper.venom.context import IRContext +from vyper.venom.passes import ReduceLiteralsCodesize + + +def _calc_push_size(val: int): + s = hex(val).removeprefix("0x") + if len(s) % 2 != 0: # justify to multiple of 2 + s = "0" + s + return 1 + len(s) + + +should_invert = [2**256 - 1] + [((2**i) - 1) << (256 - i) for i in range(121, 256 + 1)] + + +@pytest.mark.parametrize("orig_value", should_invert) +def test_literal_codesize_ff_inversion(orig_value): + """ + Test that literals like 0xfffffffffffabcd get inverted to `not 0x5432` + """ + ctx = IRContext() + fn = ctx.create_function("_global") + bb = fn.get_basic_block() + + bb.append_instruction("store", IRLiteral(orig_value)) + bb.append_instruction("stop") + ac = IRAnalysesCache(fn) + ReduceLiteralsCodesize(ac, fn).run_pass() + + inst0 = bb.instructions[0] + assert inst0.opcode == "not" + op0 = inst0.operands[0] + assert evm_not(op0.value) == orig_value + # check the optimization actually improved codesize, after accounting + # for the addl NOT instruction + assert _calc_push_size(op0.value) + 1 < _calc_push_size(orig_value) + + +should_not_invert = [1, 0xFE << 248 | (2**248 - 1)] + [ + ((2**255 - 1) >> i) << i for i in range(0, 3 * 8) +] + + +@pytest.mark.parametrize("orig_value", should_not_invert) +def test_literal_codesize_no_inversion(orig_value): + """ + Check funky cases where inversion would result in bytecode increase + """ + ctx = IRContext() + fn = ctx.create_function("_global") + bb = fn.get_basic_block() + + bb.append_instruction("store", IRLiteral(orig_value)) + bb.append_instruction("stop") + ac = IRAnalysesCache(fn) + ReduceLiteralsCodesize(ac, fn).run_pass() + + assert bb.instructions[0].opcode == "store" + assert bb.instructions[0].operands[0].value == orig_value + + +should_shl = ( + [2**i for i in range(3 * 8, 255)] + + [((2**i) - 1) << (256 - i) for i in range(1, 121)] + + [((2**255 - 1) >> i) << i for i in range(3 * 8, 254)] +) + + +@pytest.mark.parametrize("orig_value", should_shl) +def test_literal_codesize_shl(orig_value): + """ + Test that literals like 0xabcd00000000 get transformed to `shl 32 0xabcd` + """ + ctx = IRContext() + fn = ctx.create_function("_global") + bb = fn.get_basic_block() + + bb.append_instruction("store", IRLiteral(orig_value)) + bb.append_instruction("stop") + ac = IRAnalysesCache(fn) + ReduceLiteralsCodesize(ac, fn).run_pass() + + assert bb.instructions[0].opcode == "shl" + op0, op1 = bb.instructions[0].operands + assert op0.value << op1.value == orig_value + + # check the optimization actually improved codesize, after accounting + # for the addl PUSH and SHL instructions + assert _calc_push_size(op0.value) + _calc_push_size(op1.value) + 1 < _calc_push_size(orig_value) + + +should_not_shl = [1 << i for i in range(0, 3 * 8)] + [ + 0x0, + (((2 ** (256 - 2)) - 1) << (2 * 8)) ^ (2**255), +] + + +@pytest.mark.parametrize("orig_value", should_not_shl) +def test_literal_codesize_no_shl(orig_value): + """ + Check funky cases where shl transformation would result in bytecode increase + """ + ctx = IRContext() + fn = ctx.create_function("_global") + bb = fn.get_basic_block() + + bb.append_instruction("store", IRLiteral(orig_value)) + bb.append_instruction("stop") + ac = IRAnalysesCache(fn) + ReduceLiteralsCodesize(ac, fn).run_pass() + + assert bb.instructions[0].opcode == "store" + assert bb.instructions[0].operands[0].value == orig_value diff --git a/vyper/utils.py b/vyper/utils.py index 9b5bfbef62..db50626713 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -391,6 +391,11 @@ def evm_twos_complement(x: int) -> int: return ((2**256 - 1) ^ x) + 1 +def evm_not(val: int) -> int: + assert 0 <= val <= SizeLimits.MAX_UINT256, "Value out of bounds" + return SizeLimits.MAX_UINT256 ^ val + + # EVM div semantics as a python function def evm_div(x, y): if y == 0: diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index 89a4534a4c..d1eb4189c3 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -19,6 +19,7 @@ MakeSSA, Mem2Var, MemMergePass, + ReduceLiteralsCodesize, RemoveUnusedVariablesPass, SimplifyCFGPass, StoreElimination, @@ -74,6 +75,10 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: RemoveUnusedVariablesPass(ac, fn).run_pass() StoreExpansionPass(ac, fn).run_pass() + + if optimize == OptimizationLevel.CODESIZE: + ReduceLiteralsCodesize(ac, fn).run_pass() + DFTPass(ac, fn).run_pass() diff --git a/vyper/venom/passes/__init__.py b/vyper/venom/passes/__init__.py index fe1e387c56..b49791ee24 100644 --- a/vyper/venom/passes/__init__.py +++ b/vyper/venom/passes/__init__.py @@ -2,6 +2,7 @@ from .branch_optimization import BranchOptimizationPass from .dft import DFTPass from .float_allocas import FloatAllocas +from .literals_codesize import ReduceLiteralsCodesize from .lower_dload import LowerDloadPass from .make_ssa import MakeSSA from .mem2var import Mem2Var diff --git a/vyper/venom/passes/literals_codesize.py b/vyper/venom/passes/literals_codesize.py new file mode 100644 index 0000000000..daf195dfd4 --- /dev/null +++ b/vyper/venom/passes/literals_codesize.py @@ -0,0 +1,58 @@ +from vyper.utils import evm_not +from vyper.venom.basicblock import IRLiteral +from vyper.venom.passes.base_pass import IRPass + +# not takes 1 byte1, so it makes sense to use it when we can save at least +# 1 byte +NOT_THRESHOLD = 1 + +# shl takes 3 bytes, so it makes sense to use it when we can save at least +# 3 bytes +SHL_THRESHOLD = 3 + + +class ReduceLiteralsCodesize(IRPass): + def run_pass(self): + for bb in self.function.get_basic_blocks(): + self._process_bb(bb) + + def _process_bb(self, bb): + for inst in bb.instructions: + if inst.opcode != "store": + continue + + (op,) = inst.operands + if not isinstance(op, IRLiteral): + continue + + val = op.value % (2**256) + + # calculate amount of bits saved by not optimization + not_benefit = ((len(hex(val)) // 2 - len(hex(evm_not(val))) // 2) - NOT_THRESHOLD) * 8 + + # calculate amount of bits saved by shl optimization + binz = bin(val)[2:] + ix = len(binz) - binz.rfind("1") + shl_benefit = ix - SHL_THRESHOLD * 8 + + if not_benefit <= 0 and shl_benefit <= 0: + # no optimization can be done here + continue + + if not_benefit >= shl_benefit: + assert not_benefit > 0 # implied by previous conditions + # transform things like 0xffff...01 to (not 0xfe) + inst.opcode = "not" + op.value = evm_not(val) + continue + else: + assert shl_benefit > 0 # implied by previous conditions + # transform things like 0x123400....000 to 0x1234 << ... + ix -= 1 + # sanity check + assert (val >> ix) << ix == val, val + assert (val >> ix) & 1 == 1, val + + inst.opcode = "shl" + inst.operands = [IRLiteral(val >> ix), IRLiteral(ix)] + continue diff --git a/vyper/venom/passes/sccp/eval.py b/vyper/venom/passes/sccp/eval.py index b5786bb304..99f0ba70d9 100644 --- a/vyper/venom/passes/sccp/eval.py +++ b/vyper/venom/passes/sccp/eval.py @@ -5,6 +5,7 @@ SizeLimits, evm_div, evm_mod, + evm_not, evm_pow, signed_to_unsigned, unsigned_to_signed, @@ -95,11 +96,6 @@ def _evm_sar(shift_len: int, value: int) -> int: return value >> shift_len -def _evm_not(value: int) -> int: - assert 0 <= value <= SizeLimits.MAX_UINT256, "Value out of bounds" - return SizeLimits.MAX_UINT256 ^ value - - ARITHMETIC_OPS: dict[str, Callable[[list[IROperand]], int]] = { "add": _wrap_binop(operator.add), "sub": _wrap_binop(operator.sub), @@ -122,7 +118,7 @@ def _evm_not(value: int) -> int: "or": _wrap_binop(operator.or_), "and": _wrap_binop(operator.and_), "xor": _wrap_binop(operator.xor), - "not": _wrap_unop(_evm_not), + "not": _wrap_unop(evm_not), "signextend": _wrap_binop(_evm_signextend), "iszero": _wrap_unop(_evm_iszero), "shr": _wrap_binop(_evm_shr), From fadd4de32be4f328b8ab17590d7fae262d0150fb Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 20 Dec 2024 14:48:51 -0500 Subject: [PATCH 47/70] feat[venom]: update text format for data section (#4414) this commit updates the parser and grammar with new syntax for the data section. it also adds cases for raw bytes (which are generated by `--optimize codesize`) in the data section, which are required to round-trip venom produced in `--optimize codesize` mode. new format: ``` data readonly { dbsection foo { db @label1 db x"abcd" } } ``` the new "readonly" modifier doesn't have disambiguating power yet, but it will make it easier to add "writable" data sections to the grammar in the future if we decide to add that. misc/refactor: - remove xfail from previously failing test cases. - refactor internal representation of data section to not use `IRInstruction` - add `_as_asm_symbol` for easier lowering of labels - tweak `IRFunction.__repr__()` to add the function name at close (helpful for debugging). - tweak `IRBasicBlock.__repr__()` to add basic block and function name at close to help with readability --------- Co-authored-by: Harry Kalogirou --- tests/functional/venom/parser/test_parsing.py | 86 +++++++++---------- tests/functional/venom/test_venom_repr.py | 6 -- tests/venom_utils.py | 7 +- vyper/venom/basicblock.py | 4 + vyper/venom/context.py | 50 +++++++++-- vyper/venom/function.py | 3 +- vyper/venom/ir_node_to_venom.py | 9 +- vyper/venom/parser.py | 47 +++++++--- vyper/venom/passes/normalization.py | 7 +- vyper/venom/venom_to_assembly.py | 49 ++++++----- 10 files changed, 164 insertions(+), 104 deletions(-) diff --git a/tests/functional/venom/parser/test_parsing.py b/tests/functional/venom/parser/test_parsing.py index f1fc59cf40..f18a51fe76 100644 --- a/tests/functional/venom/parser/test_parsing.py +++ b/tests/functional/venom/parser/test_parsing.py @@ -1,6 +1,6 @@ from tests.venom_utils import assert_ctx_eq -from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRLabel, IRLiteral, IRVariable -from vyper.venom.context import IRContext +from vyper.venom.basicblock import IRBasicBlock, IRLabel, IRLiteral, IRVariable +from vyper.venom.context import DataItem, DataSection, IRContext from vyper.venom.function import IRFunction from vyper.venom.parser import parse_venom @@ -11,8 +11,6 @@ def test_single_bb(): main: stop } - - [data] """ parsed_ctx = parse_venom(source) @@ -38,8 +36,6 @@ def test_multi_bb_single_fn(): has_callvalue: revert 0, 0 } - - [data] """ parsed_ctx = parse_venom(source) @@ -61,8 +57,6 @@ def test_multi_bb_single_fn(): has_callvalue_bb.append_instruction("revert", IRLiteral(0), IRLiteral(0)) has_callvalue_bb.append_instruction("stop") - start_fn.last_variable = 4 - assert_ctx_eq(parsed_ctx, expected_ctx) @@ -74,15 +68,16 @@ def test_data_section(): stop } - [data] - dbname @selector_buckets - db @selector_bucket_0 - db @fallback - db @selector_bucket_2 - db @selector_bucket_3 - db @fallback - db @selector_bucket_5 - db @selector_bucket_6 + data readonly { + dbsection selector_buckets: + db @selector_bucket_0 + db @fallback + db @selector_bucket_2 + db @selector_bucket_3 + db @fallback + db @selector_bucket_5 + db @selector_bucket_6 + } """ ) @@ -91,14 +86,18 @@ def test_data_section(): entry_fn.get_basic_block("entry").append_instruction("stop") expected_ctx.data_segment = [ - IRInstruction("dbname", [IRLabel("selector_buckets")]), - IRInstruction("db", [IRLabel("selector_bucket_0")]), - IRInstruction("db", [IRLabel("fallback")]), - IRInstruction("db", [IRLabel("selector_bucket_2")]), - IRInstruction("db", [IRLabel("selector_bucket_3")]), - IRInstruction("db", [IRLabel("fallback")]), - IRInstruction("db", [IRLabel("selector_bucket_5")]), - IRInstruction("db", [IRLabel("selector_bucket_6")]), + DataSection( + IRLabel("selector_buckets"), + [ + DataItem(IRLabel("selector_bucket_0")), + DataItem(IRLabel("fallback")), + DataItem(IRLabel("selector_bucket_2")), + DataItem(IRLabel("selector_bucket_3")), + DataItem(IRLabel("fallback")), + DataItem(IRLabel("selector_bucket_5")), + DataItem(IRLabel("selector_bucket_6")), + ], + ) ] assert_ctx_eq(parsed_ctx, expected_ctx) @@ -126,8 +125,6 @@ def test_multi_function(): has_value: revert 0, 0 } - - [data] """ ) @@ -157,8 +154,6 @@ def test_multi_function(): value_bb.append_instruction("revert", IRLiteral(0), IRLiteral(0)) value_bb.append_instruction("stop") - check_fn.last_variable = 2 - assert_ctx_eq(parsed_ctx, expected_ctx) @@ -185,13 +180,14 @@ def test_multi_function_and_data(): revert 0, 0 } - [data] - dbname @selector_buckets - db @selector_bucket_0 - db @fallback - db @selector_bucket_2 - db @selector_bucket_3 - db @selector_bucket_6 + data readonly { + dbsection selector_buckets: + db @selector_bucket_0 + db @fallback + db @selector_bucket_2 + db @selector_bucket_3 + db @selector_bucket_6 + } """ ) @@ -221,15 +217,17 @@ def test_multi_function_and_data(): value_bb.append_instruction("revert", IRLiteral(0), IRLiteral(0)) value_bb.append_instruction("stop") - check_fn.last_variable = 2 - expected_ctx.data_segment = [ - IRInstruction("dbname", [IRLabel("selector_buckets")]), - IRInstruction("db", [IRLabel("selector_bucket_0")]), - IRInstruction("db", [IRLabel("fallback")]), - IRInstruction("db", [IRLabel("selector_bucket_2")]), - IRInstruction("db", [IRLabel("selector_bucket_3")]), - IRInstruction("db", [IRLabel("selector_bucket_6")]), + DataSection( + IRLabel("selector_buckets"), + [ + DataItem(IRLabel("selector_bucket_0")), + DataItem(IRLabel("fallback")), + DataItem(IRLabel("selector_bucket_2")), + DataItem(IRLabel("selector_bucket_3")), + DataItem(IRLabel("selector_bucket_6")), + ], + ) ] assert_ctx_eq(parsed_ctx, expected_ctx) diff --git a/tests/functional/venom/test_venom_repr.py b/tests/functional/venom/test_venom_repr.py index caa315dbbb..c25ce381d8 100644 --- a/tests/functional/venom/test_venom_repr.py +++ b/tests/functional/venom/test_venom_repr.py @@ -4,7 +4,6 @@ from tests.venom_utils import assert_ctx_eq, parse_venom from vyper.compiler import compile_code -from vyper.compiler.settings import OptimizationLevel from vyper.venom.context import IRContext """ @@ -18,11 +17,6 @@ def get_example_vy_filenames(): @pytest.mark.parametrize("vy_filename", get_example_vy_filenames()) def test_round_trip(vy_filename, optimize, request): - if optimize == OptimizationLevel.CODESIZE: - # codesize optimization issues things like `db b"\x12\x34"` which we - # don't handle. - request.node.add_marker(pytest.mark.xfail(strict=False, reason="unimplemented in parser")) - path = f"examples/{vy_filename}" with open(path) as f: vyper_source = f.read() diff --git a/tests/venom_utils.py b/tests/venom_utils.py index a67df6c275..85298ccb87 100644 --- a/tests/venom_utils.py +++ b/tests/venom_utils.py @@ -36,14 +36,11 @@ def assert_fn_eq(fn1: IRFunction, fn2: IRFunction): def assert_ctx_eq(ctx1: IRContext, ctx2: IRContext): - assert len(ctx1.functions) == len(ctx2.functions) for label1, fn1 in ctx1.functions.items(): assert label1 in ctx2.functions assert_fn_eq(fn1, ctx2.functions[label1]) + assert len(ctx1.functions) == len(ctx2.functions) # check entry function is the same assert next(iter(ctx1.functions.keys())) == next(iter(ctx2.functions.keys())) - - assert len(ctx1.data_segment) == len(ctx2.data_segment) - for d1, d2 in zip(ctx1.data_segment, ctx2.data_segment): - assert instructions_eq(d1, d2), f"data: [{d1}] != [{d2}]" + assert ctx1.data_segment == ctx2.data_segment, ctx2.data_segment diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index 4b8eec2263..b0f0b00341 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -661,4 +661,8 @@ def __repr__(self) -> str: s += f" OUT={[bb.label for bb in self.cfg_out]} => {self.out_vars}\n" for instruction in self.instructions: s += f" {str(instruction).strip()}\n" + if len(self.instructions) > 30: + s += f" ; {self.label}\n" + if len(self.instructions) > 30 or self.parent.num_basic_blocks > 5: + s += f" ; ({self.parent.name})\n\n" return s diff --git a/vyper/venom/context.py b/vyper/venom/context.py index 391da3e189..0c5cbc379c 100644 --- a/vyper/venom/context.py +++ b/vyper/venom/context.py @@ -1,14 +1,40 @@ +import textwrap +from dataclasses import dataclass, field from typing import Optional -from vyper.venom.basicblock import IRInstruction, IRLabel, IROperand +from vyper.venom.basicblock import IRLabel from vyper.venom.function import IRFunction +@dataclass +class DataItem: + data: IRLabel | bytes # can be raw data or bytes + + def __str__(self): + if isinstance(self.data, IRLabel): + return f"@{self.data}" + else: + assert isinstance(self.data, bytes) + return f'x"{self.data.hex()}"' + + +@dataclass +class DataSection: + label: IRLabel + data_items: list[DataItem] = field(default_factory=list) + + def __str__(self): + ret = [f"dbsection {self.label.value}:"] + for item in self.data_items: + ret.append(f" db {item}") + return "\n".join(ret) + + class IRContext: functions: dict[IRLabel, IRFunction] ctor_mem_size: Optional[int] immutables_len: Optional[int] - data_segment: list[IRInstruction] + data_segment: list[DataSection] last_label: int def __init__(self) -> None: @@ -47,11 +73,16 @@ def chain_basic_blocks(self) -> None: for fn in self.functions.values(): fn.chain_basic_blocks() - def append_data(self, opcode: str, args: list[IROperand]) -> None: + def append_data_section(self, name: IRLabel) -> None: + self.data_segment.append(DataSection(name)) + + def append_data_item(self, data: IRLabel | bytes) -> None: """ - Append data + Append data to current data section """ - self.data_segment.append(IRInstruction(opcode, args)) # type: ignore + assert len(self.data_segment) > 0 + data_section = self.data_segment[-1] + data_section.data_items.append(DataItem(data)) def as_graph(self) -> str: s = ["digraph G {"] @@ -64,12 +95,13 @@ def as_graph(self) -> str: def __repr__(self) -> str: s = [] for fn in self.functions.values(): - s.append(fn.__repr__()) + s.append(IRFunction.__repr__(fn)) s.append("\n") if len(self.data_segment) > 0: - s.append("\n[data]") - for inst in self.data_segment: - s.append(f" {inst}") + s.append("data readonly {") + for data_section in self.data_segment: + s.append(textwrap.indent(DataSection.__str__(data_section), " ")) + s.append("}") return "\n".join(s) diff --git a/vyper/venom/function.py b/vyper/venom/function.py index 5709f65be4..f02da77fe3 100644 --- a/vyper/venom/function.py +++ b/vyper/venom/function.py @@ -228,4 +228,5 @@ def __repr__(self) -> str: bb_str = textwrap.indent(str(bb), " ") ret += f"{bb_str}\n" ret = ret.strip() + "\n}" - return ret.strip() + ret += f" ; close function {self.name}" + return ret diff --git a/vyper/venom/ir_node_to_venom.py b/vyper/venom/ir_node_to_venom.py index 4dcc5ee4e6..f46457b77f 100644 --- a/vyper/venom/ir_node_to_venom.py +++ b/vyper/venom/ir_node_to_venom.py @@ -367,14 +367,15 @@ def _convert_ir_bb(fn, ir, symbols): elif ir.value == "symbol": return IRLabel(ir.args[0].value, True) elif ir.value == "data": - label = IRLabel(ir.args[0].value) - ctx.append_data("dbname", [label]) + label = IRLabel(ir.args[0].value, True) + ctx.append_data_section(label) for c in ir.args[1:]: if isinstance(c.value, bytes): - ctx.append_data("db", [c.value]) # type: ignore + ctx.append_data_item(c.value) elif isinstance(c, IRnode): data = _convert_ir_bb(fn, c, symbols) - ctx.append_data("db", [data]) # type: ignore + assert isinstance(data, IRLabel) # help mypy + ctx.append_data_item(data) elif ir.value == "label": label = IRLabel(ir.args[0].value, True) bb = fn.get_basic_block() diff --git a/vyper/venom/parser.py b/vyper/venom/parser.py index 9ab223179e..5ccc29b7a4 100644 --- a/vyper/venom/parser.py +++ b/vyper/venom/parser.py @@ -10,12 +10,13 @@ IROperand, IRVariable, ) -from vyper.venom.context import IRContext +from vyper.venom.context import DataItem, DataSection, IRContext from vyper.venom.function import IRFunction VENOM_GRAMMAR = """ %import common.CNAME %import common.DIGIT + %import common.HEXDIGIT %import common.LETTER %import common.WS %import common.INT @@ -25,13 +26,15 @@ # Allow multiple comment styles COMMENT: ";" /[^\\n]*/ | "//" /[^\\n]*/ | "#" /[^\\n]*/ - start: function* data_section? + start: function* data_segment? # TODO: consider making entry block implicit, e.g. # `"{" instruction+ block* "}"` function: "function" LABEL_IDENT "{" block* "}" - data_section: "[data]" instruction* + data_segment: "data" "readonly" "{" data_section* "}" + data_section: "dbsection" LABEL_IDENT ":" data_item+ + data_item: "db" (HEXSTR | LABEL) block: LABEL_IDENT ":" "\\n" statement* @@ -53,7 +56,9 @@ LABEL_IDENT: (NAME | ESCAPED_STRING) LABEL: "@" LABEL_IDENT + DOUBLE_QUOTE: "\\"" NAME: (DIGIT|LETTER|"_")+ + HEXSTR: "x" DOUBLE_QUOTE (HEXDIGIT|"_")+ DOUBLE_QUOTE %ignore WS %ignore COMMENT @@ -101,17 +106,21 @@ def _unescape(s: str): return s -class _DataSegment: - def __init__(self, instructions): - self.instructions = instructions +class _TypedItem: + def __init__(self, children): + self.children = children + + +class _DataSegment(_TypedItem): + pass class VenomTransformer(Transformer): def start(self, children) -> IRContext: ctx = IRContext() - data_section = [] - if isinstance(children[-1], _DataSegment): - data_section = children.pop().instructions + if len(children) > 0 and isinstance(children[-1], _DataSegment): + ctx.data_segment = children.pop().children + funcs = children for fn_name, blocks in funcs: fn = ctx.create_function(fn_name) @@ -130,8 +139,6 @@ def start(self, children) -> IRContext: _set_last_var(fn) _set_last_label(ctx) - ctx.data_segment = data_section - return ctx def function(self, children) -> tuple[str, list[tuple[str, list[IRInstruction]]]]: @@ -141,9 +148,25 @@ def function(self, children) -> tuple[str, list[tuple[str, list[IRInstruction]]] def statement(self, children): return children[0] - def data_section(self, children): + def data_segment(self, children): return _DataSegment(children) + def data_section(self, children): + label = IRLabel(children[0], True) + data_items = children[1:] + assert all(isinstance(item, DataItem) for item in data_items) + return DataSection(label, data_items) + + def data_item(self, children): + item = children[0] + if isinstance(item, IRLabel): + return DataItem(item) + assert item.startswith('x"') + assert item.endswith('"') + item = item.removeprefix('x"').removesuffix('"') + item = item.replace("_", "") + return DataItem(bytes.fromhex(item)) + def block(self, children) -> tuple[str, list[IRInstruction]]: label, *instructions = children return label, instructions diff --git a/vyper/venom/passes/normalization.py b/vyper/venom/passes/normalization.py index 7ca242c74e..37ba1023c9 100644 --- a/vyper/venom/passes/normalization.py +++ b/vyper/venom/passes/normalization.py @@ -45,9 +45,10 @@ def _insert_split_basicblock(self, bb: IRBasicBlock, in_bb: IRBasicBlock) -> IRB inst.operands[i] = split_bb.label # Update the labels in the data segment - for inst in fn.ctx.data_segment: - if inst.opcode == "db" and inst.operands[0] == bb.label: - inst.operands[0] = split_bb.label + for data_section in fn.ctx.data_segment: + for item in data_section.data_items: + if item.data == bb.label: + item.data = split_bb.label return split_bb diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index e136932f51..048555a221 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -122,6 +122,11 @@ def apply_line_numbers(inst: IRInstruction, asm) -> list[str]: return ret # type: ignore +def _as_asm_symbol(label: IRLabel) -> str: + # Lower an IRLabel to an assembly symbol + return f"_sym_{label.value}" + + # TODO: "assembly" gets into the recursion due to how the original # IR was structured recursively in regards with the deploy instruction. # There, recursing into the deploy instruction was by design, and @@ -183,19 +188,19 @@ def generate_evm(self, no_optimize: bool = False) -> list[str]: asm.extend(_REVERT_POSTAMBLE) # Append data segment - data_segments: dict = dict() - for inst in ctx.data_segment: - if inst.opcode == "dbname": - label = inst.operands[0] - data_segments[label] = [DataHeader(f"_sym_{label.value}")] - elif inst.opcode == "db": - data = inst.operands[0] + for data_section in ctx.data_segment: + label = data_section.label + asm_data_section: list[Any] = [] + asm_data_section.append(DataHeader(_as_asm_symbol(label))) + for item in data_section.data_items: + data = item.data if isinstance(data, IRLabel): - data_segments[label].append(f"_sym_{data.value}") + asm_data_section.append(_as_asm_symbol(data)) else: - data_segments[label].append(data) + assert isinstance(data, bytes) + asm_data_section.append(data) - asm.extend(list(data_segments.values())) + asm.append(asm_data_section) if no_optimize is False: optimize_assembly(top_asm) @@ -260,7 +265,7 @@ def _emit_input_operands( # invoke emits the actual instruction itself so we don't need # to emit it here but we need to add it to the stack map if inst.opcode != "invoke": - assembly.append(f"_sym_{op.value}") + assembly.append(_as_asm_symbol(op)) stack.push(op) continue @@ -294,7 +299,7 @@ def _generate_evm_for_basicblock_r( asm = [] # assembly entry point into the block - asm.append(f"_sym_{basicblock.label.value}") + asm.append(_as_asm_symbol(basicblock.label)) asm.append("JUMPDEST") if len(basicblock.cfg_in) == 1: @@ -409,7 +414,9 @@ def _generate_evm_for_instruction( return apply_line_numbers(inst, assembly) if opcode == "offset": - assembly.extend(["_OFST", f"_sym_{inst.operands[1].value}", inst.operands[0].value]) + ofst, label = inst.operands + assert isinstance(label, IRLabel) # help mypy + assembly.extend(["_OFST", _as_asm_symbol(label), ofst.value]) assert isinstance(inst.output, IROperand), "Offset must have output" stack.push(inst.output) return apply_line_numbers(inst, assembly) @@ -471,24 +478,26 @@ def _generate_evm_for_instruction( pass elif opcode == "store": pass + elif opcode in ["codecopy", "dloadbytes"]: + assembly.append("CODECOPY") elif opcode == "dbname": pass elif opcode == "jnz": # jump if not zero - if_nonzero_label = inst.operands[1] - if_zero_label = inst.operands[2] - assembly.append(f"_sym_{if_nonzero_label.value}") + if_nonzero_label, if_zero_label = inst.get_label_operands() + assembly.append(_as_asm_symbol(if_nonzero_label)) assembly.append("JUMPI") # make sure the if_zero_label will be optimized out # assert if_zero_label == next(iter(inst.parent.cfg_out)).label - assembly.append(f"_sym_{if_zero_label.value}") + assembly.append(_as_asm_symbol(if_zero_label)) assembly.append("JUMP") elif opcode == "jmp": - assert isinstance(inst.operands[0], IRLabel) - assembly.append(f"_sym_{inst.operands[0].value}") + (target,) = inst.operands + assert isinstance(target, IRLabel) + assembly.append(_as_asm_symbol(target)) assembly.append("JUMP") elif opcode == "djmp": assert isinstance( @@ -503,7 +512,7 @@ def _generate_evm_for_instruction( assembly.extend( [ f"_sym_label_ret_{self.label_counter}", - f"_sym_{target.value}", + _as_asm_symbol(target), "JUMP", f"_sym_label_ret_{self.label_counter}", "JUMPDEST", From 9c98b3ed4a4fbb1a614e63f815617fc275a0d16a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 20 Dec 2024 15:07:05 -0500 Subject: [PATCH 48/70] feat[venom]: add load elimination pass (#4265) add primitive `sload`/`tload`/`mload` elimination. keeps latest (m/s/t)load or (m/s/t)store in a one-element "lattice", and weakens the (m/s/t)load to a `store` instruction if there is a hit on the same key. --------- Co-authored-by: Harry Kalogirou Co-authored-by: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> --- .../compiler/venom/test_load_elimination.py | 129 ++++++++++++++++++ vyper/venom/__init__.py | 4 + vyper/venom/passes/__init__.py | 1 + vyper/venom/passes/load_elimination.py | 50 +++++++ vyper/venom/passes/store_elimination.py | 3 + 5 files changed, 187 insertions(+) create mode 100644 tests/unit/compiler/venom/test_load_elimination.py create mode 100644 vyper/venom/passes/load_elimination.py diff --git a/tests/unit/compiler/venom/test_load_elimination.py b/tests/unit/compiler/venom/test_load_elimination.py new file mode 100644 index 0000000000..52c7baf3c9 --- /dev/null +++ b/tests/unit/compiler/venom/test_load_elimination.py @@ -0,0 +1,129 @@ +from tests.venom_utils import assert_ctx_eq, parse_from_basic_block +from vyper.venom.analysis.analysis import IRAnalysesCache +from vyper.venom.passes.load_elimination import LoadElimination + + +def _check_pre_post(pre, post): + ctx = parse_from_basic_block(pre) + + for fn in ctx.functions.values(): + ac = IRAnalysesCache(fn) + LoadElimination(ac, fn).run_pass() + + assert_ctx_eq(ctx, parse_from_basic_block(post)) + + +def _check_no_change(pre): + _check_pre_post(pre, pre) + + +def test_simple_load_elimination(): + pre = """ + main: + %ptr = 11 + %1 = mload %ptr + + %2 = mload %ptr + + stop + """ + post = """ + main: + %ptr = 11 + %1 = mload %ptr + + %2 = %1 + + stop + """ + _check_pre_post(pre, post) + + +def test_equivalent_var_elimination(): + """ + Test that the lattice can "peer through" equivalent vars + """ + pre = """ + main: + %1 = 11 + %2 = %1 + %3 = mload %1 + + %4 = mload %2 + + stop + """ + post = """ + main: + %1 = 11 + %2 = %1 + %3 = mload %1 + + %4 = %3 # %2 == %1 + + stop + """ + _check_pre_post(pre, post) + + +def test_elimination_barrier(): + """ + Check for barrier between load/load + """ + pre = """ + main: + %1 = 11 + %2 = mload %1 + %3 = %100 + # fence - writes to memory + staticcall %3, %3, %3, %3 + %4 = mload %1 + """ + _check_no_change(pre) + + +def test_store_load_elimination(): + """ + Check that lattice stores the result of mstores (even through + equivalent variables) + """ + pre = """ + main: + %val = 55 + %ptr1 = 11 + %ptr2 = %ptr1 + mstore %ptr1, %val + + %3 = mload %ptr2 + + stop + """ + post = """ + main: + %val = 55 + %ptr1 = 11 + %ptr2 = %ptr1 + mstore %ptr1, %val + + %3 = %val + + stop + """ + _check_pre_post(pre, post) + + +def test_store_load_barrier(): + """ + Check for barrier between store/load + """ + pre = """ + main: + %ptr = 11 + %val = 55 + mstore %ptr, %val + %3 = %100 ; arbitrary + # fence + staticcall %3, %3, %3, %3 + %4 = mload %ptr + """ + _check_no_change(pre) diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index d1eb4189c3..ddd9065194 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -15,6 +15,7 @@ BranchOptimizationPass, DFTPass, FloatAllocas, + LoadElimination, LowerDloadPass, MakeSSA, Mem2Var, @@ -59,9 +60,12 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: Mem2Var(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() SCCP(ac, fn).run_pass() + + LoadElimination(ac, fn).run_pass() StoreElimination(ac, fn).run_pass() MemMergePass(ac, fn).run_pass() SimplifyCFGPass(ac, fn).run_pass() + LowerDloadPass(ac, fn).run_pass() AlgebraicOptimizationPass(ac, fn).run_pass() # NOTE: MakeSSA is after algebraic optimization it currently produces diff --git a/vyper/venom/passes/__init__.py b/vyper/venom/passes/__init__.py index b49791ee24..a3227dcf4b 100644 --- a/vyper/venom/passes/__init__.py +++ b/vyper/venom/passes/__init__.py @@ -3,6 +3,7 @@ from .dft import DFTPass from .float_allocas import FloatAllocas from .literals_codesize import ReduceLiteralsCodesize +from .load_elimination import LoadElimination from .lower_dload import LowerDloadPass from .make_ssa import MakeSSA from .mem2var import Mem2Var diff --git a/vyper/venom/passes/load_elimination.py b/vyper/venom/passes/load_elimination.py new file mode 100644 index 0000000000..6701b588fe --- /dev/null +++ b/vyper/venom/passes/load_elimination.py @@ -0,0 +1,50 @@ +from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis, VarEquivalenceAnalysis +from vyper.venom.effects import Effects +from vyper.venom.passes.base_pass import IRPass + + +class LoadElimination(IRPass): + """ + Eliminate sloads, mloads and tloads + """ + + # should this be renamed to EffectsElimination? + + def run_pass(self): + self.equivalence = self.analyses_cache.request_analysis(VarEquivalenceAnalysis) + + for bb in self.function.get_basic_blocks(): + self._process_bb(bb, Effects.MEMORY, "mload", "mstore") + self._process_bb(bb, Effects.TRANSIENT, "tload", "tstore") + self._process_bb(bb, Effects.STORAGE, "sload", "sstore") + + self.analyses_cache.invalidate_analysis(LivenessAnalysis) + self.analyses_cache.invalidate_analysis(DFGAnalysis) + + def equivalent(self, op1, op2): + return op1 == op2 or self.equivalence.equivalent(op1, op2) + + def _process_bb(self, bb, eff, load_opcode, store_opcode): + # not really a lattice even though it is not really inter-basic block; + # we may generalize in the future + lattice = () + + for inst in bb.instructions: + if eff in inst.get_write_effects(): + lattice = () + + if inst.opcode == store_opcode: + # mstore [val, ptr] + val, ptr = inst.operands + lattice = (ptr, val) + + if inst.opcode == load_opcode: + prev_lattice = lattice + (ptr,) = inst.operands + lattice = (ptr, inst.output) + if not prev_lattice: + continue + if not self.equivalent(ptr, prev_lattice[0]): + continue + inst.opcode = "store" + inst.operands = [prev_lattice[1]] diff --git a/vyper/venom/passes/store_elimination.py b/vyper/venom/passes/store_elimination.py index 97ab424cd6..22d4723013 100644 --- a/vyper/venom/passes/store_elimination.py +++ b/vyper/venom/passes/store_elimination.py @@ -9,6 +9,9 @@ class StoreElimination(IRPass): and removes the `store` instruction. """ + # TODO: consider renaming `store` instruction, since it is confusing + # with LoadElimination + def run_pass(self): self.analyses_cache.request_analysis(CFGAnalysis) dfg = self.analyses_cache.request_analysis(DFGAnalysis) From c9d8f5b8e8548b8af88581401a73fb1388040b14 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 25 Dec 2024 01:50:40 +0200 Subject: [PATCH 49/70] fix[venom]: fix `MakeSSA` with existing phis (#4423) This commit improves the `MakeSSA` pass to handle incoming `phi` instructions. Following the implementation of the venom parser it is possible to parse code with existing `phi` instructions in the code. The old code was expecting the `phi` instructions to have been self-placed and in "degenerate form" of output == all branch arguments. The new code does not make this assumption. This commit additionally: - refactors the existing `make_ssa` tests to use new test machinery - adds a phi parsing test (does not test the new code in this commit since it does not run passes, but does make sure we can at least parse phis) - expands the venom round-trip tests to check that we can both a) run venom passes on parsed venom, and b) bytecode generation from round-tripping venom produced by the vyper frontend is equivalent to bytecode generation from the regular pipeline (directly from vyper source code) --------- Co-authored-by: Charles Cooper --- tests/functional/venom/parser/test_parsing.py | 121 +++++++++++++++++- tests/functional/venom/test_venom_repr.py | 86 ++++++++++++- tests/unit/compiler/venom/test_make_ssa.py | 88 +++++++------ vyper/venom/basicblock.py | 17 ++- vyper/venom/passes/make_ssa.py | 8 +- 5 files changed, 263 insertions(+), 57 deletions(-) diff --git a/tests/functional/venom/parser/test_parsing.py b/tests/functional/venom/parser/test_parsing.py index f18a51fe76..bd536a8cfa 100644 --- a/tests/functional/venom/parser/test_parsing.py +++ b/tests/functional/venom/parser/test_parsing.py @@ -1,4 +1,4 @@ -from tests.venom_utils import assert_ctx_eq +from tests.venom_utils import assert_bb_eq, assert_ctx_eq from vyper.venom.basicblock import IRBasicBlock, IRLabel, IRLiteral, IRVariable from vyper.venom.context import DataItem, DataSection, IRContext from vyper.venom.function import IRFunction @@ -231,3 +231,122 @@ def test_multi_function_and_data(): ] assert_ctx_eq(parsed_ctx, expected_ctx) + + +def test_phis(): + # @external + # def _loop() -> uint256: + # res: uint256 = 9 + # for i: uint256 in range(res, bound=10): + # res = res + i + # return res + source = """ + function __main_entry { + __main_entry: ; IN=[] OUT=[fallback, 1_then] => {} + %27 = 0 + %1 = calldataload %27 + %28 = %1 + %29 = 224 + %2 = shr %29, %28 + %31 = %2 + %30 = 1729138561 + %4 = xor %30, %31 + %32 = %4 + jnz %32, @fallback, @1_then + ; (__main_entry) + + + 1_then: ; IN=[__main_entry] OUT=[4_condition] => {%11, %var8_0} + %6 = callvalue + %33 = %6 + %7 = iszero %33 + %34 = %7 + assert %34 + %var8_0 = 9 + %11 = 0 + nop + jmp @4_condition + ; (__main_entry) + + + 4_condition: ; IN=[1_then, 5_body] OUT=[5_body, 7_exit] => {%11:3, %var8_0:2} + %var8_0:2 = phi @1_then, %var8_0, @5_body, %var8_0:3 + %11:3 = phi @1_then, %11, @5_body, %11:4 + %35 = %11:3 + %36 = 9 + %15 = xor %36, %35 + %37 = %15 + jnz %37, @5_body, @7_exit + ; (__main_entry) + + + 5_body: ; IN=[4_condition] OUT=[4_condition] => {%11:4, %var8_0:3} + %38 = %11:3 + %39 = %var8_0:2 + %22 = add %39, %38 + %41 = %22 + %40 = %var8_0:2 + %24 = gt %40, %41 + %42 = %24 + %25 = iszero %42 + %43 = %25 + assert %43 + %var8_0:3 = %22 + %44 = %11:3 + %45 = 1 + %11:4 = add %45, %44 + jmp @4_condition + ; (__main_entry) + + + 7_exit: ; IN=[4_condition] OUT=[] => {} + %46 = %var8_0:2 + %47 = 64 + mstore %47, %46 + %48 = 32 + %49 = 64 + return %49, %48 + ; (__main_entry) + + + fallback: ; IN=[__main_entry] OUT=[] => {} + %50 = 0 + %51 = 0 + revert %51, %50 + stop + ; (__main_entry) + } ; close function __main_entry + """ + ctx = parse_venom(source) + + expected_ctx = IRContext() + expected_ctx.add_function(entry_fn := IRFunction(IRLabel("__main_entry"))) + + expect_bb = IRBasicBlock(IRLabel("4_condition"), entry_fn) + entry_fn.append_basic_block(expect_bb) + + expect_bb.append_instruction( + "phi", + IRLabel("1_then"), + IRVariable("%var8_0"), + IRLabel("5_body"), + IRVariable("%var8_0:3"), + ret=IRVariable("var8_0:2"), + ) + expect_bb.append_instruction( + "phi", + IRLabel("1_then"), + IRVariable("%11"), + IRLabel("5_body"), + IRVariable("%11:4"), + ret=IRVariable("11:3"), + ) + expect_bb.append_instruction("store", IRVariable("11:3"), ret=IRVariable("%35")) + expect_bb.append_instruction("store", IRLiteral(9), ret=IRVariable("%36")) + expect_bb.append_instruction("xor", IRVariable("%35"), IRVariable("%36"), ret=IRVariable("%15")) + expect_bb.append_instruction("store", IRVariable("%15"), ret=IRVariable("%37")) + expect_bb.append_instruction("jnz", IRVariable("%37"), IRLabel("5_body"), IRLabel("7_exit")) + # other basic blocks omitted for brevity + + parsed_fn = next(iter(ctx.functions.values())) + assert_bb_eq(parsed_fn.get_basic_block(expect_bb.label.name), expect_bb) diff --git a/tests/functional/venom/test_venom_repr.py b/tests/functional/venom/test_venom_repr.py index c25ce381d8..5136672a03 100644 --- a/tests/functional/venom/test_venom_repr.py +++ b/tests/functional/venom/test_venom_repr.py @@ -1,9 +1,13 @@ +import copy import glob +import textwrap import pytest from tests.venom_utils import assert_ctx_eq, parse_venom from vyper.compiler import compile_code +from vyper.compiler.phases import generate_bytecode +from vyper.venom import generate_assembly_experimental, run_passes_on from vyper.venom.context import IRContext """ @@ -16,15 +20,95 @@ def get_example_vy_filenames(): @pytest.mark.parametrize("vy_filename", get_example_vy_filenames()) -def test_round_trip(vy_filename, optimize, request): +def test_round_trip_examples(vy_filename, optimize, compiler_settings): + """ + Check all examples round trip + """ path = f"examples/{vy_filename}" with open(path) as f: vyper_source = f.read() + _round_trip_helper(vyper_source, optimize, compiler_settings) + + +# pure vyper sources +vyper_sources = [ + """ + @external + def _loop() -> uint256: + res: uint256 = 9 + for i: uint256 in range(res, bound=10): + res = res + i + return res + """ +] + + +@pytest.mark.parametrize("vyper_source", vyper_sources) +def test_round_trip_sources(vyper_source, optimize, compiler_settings): + """ + Test vyper_sources round trip + """ + vyper_source = textwrap.dedent(vyper_source) + _round_trip_helper(vyper_source, optimize, compiler_settings) + + +def _round_trip_helper(vyper_source, optimize, compiler_settings): + # helper function to test venom round-tripping thru the parser + # use two helpers because run_passes_on and + # generate_assembly_experimental are both destructive (mutating) on + # the IRContext + _helper1(vyper_source, optimize) + _helper2(vyper_source, optimize, compiler_settings) + + +def _helper1(vyper_source, optimize): + """ + Check that we are able to run passes on the round-tripped venom code + and that it is valid (generates bytecode) + """ + # note: compiling any later stage than bb_runtime like `asm` or + # `bytecode` modifies the bb_runtime data structure in place and results + # in normalization of the venom cfg (which breaks again make_ssa) out = compile_code(vyper_source, output_formats=["bb_runtime"]) + + bb_runtime = out["bb_runtime"] + venom_code = IRContext.__repr__(bb_runtime) + + ctx = parse_venom(venom_code) + + assert_ctx_eq(bb_runtime, ctx) + + # check it's valid to run venom passes+analyses + # (note this breaks bytecode equality, in the future we should + # test that separately) + run_passes_on(ctx, optimize) + + # test we can generate assembly+bytecode + asm = generate_assembly_experimental(ctx) + generate_bytecode(asm, compiler_metadata=None) + + +def _helper2(vyper_source, optimize, compiler_settings): + """ + Check that we can compile to bytecode, and without running venom passes, + that the output bytecode is equal to going through the normal vyper pipeline + """ + settings = copy.copy(compiler_settings) + # bytecode equivalence only makes sense if we use venom pipeline + settings.experimental_codegen = True + + out = compile_code(vyper_source, settings=settings, output_formats=["bb_runtime"]) bb_runtime = out["bb_runtime"] venom_code = IRContext.__repr__(bb_runtime) ctx = parse_venom(venom_code) assert_ctx_eq(bb_runtime, ctx) + + # test we can generate assembly+bytecode + asm = generate_assembly_experimental(ctx, optimize=optimize) + bytecode = generate_bytecode(asm, compiler_metadata=None) + + out = compile_code(vyper_source, settings=settings, output_formats=["bytecode_runtime"]) + assert "0x" + bytecode.hex() == out["bytecode_runtime"] diff --git a/tests/unit/compiler/venom/test_make_ssa.py b/tests/unit/compiler/venom/test_make_ssa.py index aa3fead6bf..7f6b2c0cba 100644 --- a/tests/unit/compiler/venom/test_make_ssa.py +++ b/tests/unit/compiler/venom/test_make_ssa.py @@ -1,48 +1,52 @@ +from tests.venom_utils import assert_ctx_eq, parse_venom from vyper.venom.analysis import IRAnalysesCache -from vyper.venom.basicblock import IRBasicBlock, IRLabel -from vyper.venom.context import IRContext from vyper.venom.passes import MakeSSA -def test_phi_case(): - ctx = IRContext() - fn = ctx.create_function("_global") - - bb = fn.get_basic_block() - - bb_cont = IRBasicBlock(IRLabel("condition"), fn) - bb_then = IRBasicBlock(IRLabel("then"), fn) - bb_else = IRBasicBlock(IRLabel("else"), fn) - bb_if_exit = IRBasicBlock(IRLabel("if_exit"), fn) - fn.append_basic_block(bb_cont) - fn.append_basic_block(bb_then) - fn.append_basic_block(bb_else) - fn.append_basic_block(bb_if_exit) - - v = bb.append_instruction("mload", 64) - bb_cont.append_instruction("jnz", v, bb_then.label, bb_else.label) - - bb_if_exit.append_instruction("add", v, 1, ret=v) - bb_if_exit.append_instruction("jmp", bb_cont.label) +def _check_pre_post(pre, post): + ctx = parse_venom(pre) + for fn in ctx.functions.values(): + ac = IRAnalysesCache(fn) + MakeSSA(ac, fn).run_pass() + assert_ctx_eq(ctx, parse_venom(post)) - bb_then.append_instruction("assert", bb_then.append_instruction("mload", 96)) - bb_then.append_instruction("jmp", bb_if_exit.label) - bb_else.append_instruction("jmp", bb_if_exit.label) - bb.append_instruction("jmp", bb_cont.label) - - ac = IRAnalysesCache(fn) - MakeSSA(ac, fn).run_pass() - - condition_block = fn.get_basic_block("condition") - assert len(condition_block.instructions) == 2 - - phi_inst = condition_block.instructions[0] - assert phi_inst.opcode == "phi" - assert phi_inst.operands[0].name == "_global" - assert phi_inst.operands[1].name == "%1" - assert phi_inst.operands[2].name == "if_exit" - assert phi_inst.operands[3].name == "%1" - assert phi_inst.output.name == "%1" - assert phi_inst.output.value != phi_inst.operands[1].value - assert phi_inst.output.value != phi_inst.operands[3].value +def test_phi_case(): + pre = """ + function loop { + main: + %v = mload 64 + jmp @test + test: + jnz %v, @then, @else + then: + %t = mload 96 + assert %t + jmp @if_exit + else: + jmp @if_exit + if_exit: + %v = add %v, 1 + jmp @test + } + """ + post = """ + function loop { + main: + %v = mload 64 + jmp @test + test: + %v:1 = phi @main, %v, @if_exit, %v:2 + jnz %v:1, @then, @else + then: + %t = mload 96 + assert %t + jmp @if_exit + else: + jmp @if_exit + if_exit: + %v:2 = add %v:1, 1 + jmp @test + } + """ + _check_pre_post(pre, post) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index b0f0b00341..4c75c67700 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -117,7 +117,7 @@ class IROperand: """ value: Any - _hash: Optional[int] + _hash: Optional[int] = None def __init__(self, value: Any) -> None: self.value = value @@ -149,9 +149,8 @@ class IRLiteral(IROperand): value: int def __init__(self, value: int) -> None: - super().__init__(value) assert isinstance(value, int), "value must be an int" - self.value = value + super().__init__(value) class IRVariable(IROperand): @@ -163,17 +162,17 @@ class IRVariable(IROperand): version: Optional[int] def __init__(self, name: str, version: int = 0) -> None: - super().__init__(name) assert isinstance(name, str) - assert isinstance(version, int | None) + # TODO: allow version to be None + assert isinstance(version, int) if not name.startswith("%"): name = f"%{name}" self._name = name self.version = version + value = name if version > 0: - self.value = f"{name}:{version}" - else: - self.value = name + value = f"{name}:{version}" + super().__init__(value) @property def name(self) -> str: @@ -193,8 +192,8 @@ class IRLabel(IROperand): def __init__(self, value: str, is_symbol: bool = False) -> None: assert isinstance(value, str), f"not a str: {value} ({type(value)})" assert len(value) > 0 - super().__init__(value) self.is_symbol = is_symbol + super().__init__(value) _IS_IDENTIFIER = re.compile("[0-9a-zA-Z_]*") diff --git a/vyper/venom/passes/make_ssa.py b/vyper/venom/passes/make_ssa.py index 56d3e1b7d3..ee013e0f1d 100644 --- a/vyper/venom/passes/make_ssa.py +++ b/vyper/venom/passes/make_ssa.py @@ -35,8 +35,8 @@ def _add_phi_nodes(self): Add phi nodes to the function. """ self._compute_defs() - work = {var: 0 for var in self.dom.dfs_walk} - has_already = {var: 0 for var in self.dom.dfs_walk} + work = {bb: 0 for bb in self.dom.dfs_walk} + has_already = {bb: 0 for bb in self.dom.dfs_walk} i = 0 # Iterate over all variables @@ -96,7 +96,6 @@ def _rename_vars(self, basic_block: IRBasicBlock): self.var_name_counters[v_name] = i + 1 inst.output = IRVariable(v_name, version=i) - # note - after previous line, inst.output.name != v_name outs.append(inst.output.name) for bb in basic_block.cfg_out: @@ -106,8 +105,9 @@ def _rename_vars(self, basic_block: IRBasicBlock): assert inst.output is not None, "Phi instruction without output" for i, op in enumerate(inst.operands): if op == basic_block.label: + var = inst.operands[i + 1] inst.operands[i + 1] = IRVariable( - inst.output.name, version=self.var_name_stacks[inst.output.name][-1] + var.name, version=self.var_name_stacks[var.name][-1] ) for bb in self.dom.dominated[basic_block]: From 7caa05590a4c8788b01795d3e18a7aefe1183ebf Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 26 Dec 2024 15:28:59 -0500 Subject: [PATCH 50/70] refactor[venom]: refactor mem2var (#4421) remove special cases which were necessary before introduction of `palloca`. now they represent useless variable read/writes which are safe to remove. results in removal of a few instructions on benchmark contracts. --------- Co-authored-by: Harry Kalogirou --- vyper/venom/passes/mem2var.py | 51 ++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/vyper/venom/passes/mem2var.py b/vyper/venom/passes/mem2var.py index f93924d449..9f985e2b0b 100644 --- a/vyper/venom/passes/mem2var.py +++ b/vyper/venom/passes/mem2var.py @@ -34,31 +34,30 @@ def _mk_varname(self, varname: str): def _process_alloca_var(self, dfg: DFGAnalysis, var: IRVariable): """ - Process alloca allocated variable. If it is only used by mstore/mload/return - instructions, it is promoted to a stack variable. Otherwise, it is left as is. + Process alloca allocated variable. If it is only used by + mstore/mload/return instructions, it is promoted to a stack variable. + Otherwise, it is left as is. """ uses = dfg.get_uses(var) - if all([inst.opcode == "mload" for inst in uses]): - return - elif all([inst.opcode == "mstore" for inst in uses]): + if not all([inst.opcode in ["mstore", "mload", "return"] for inst in uses]): return - elif all([inst.opcode in ["mstore", "mload", "return"] for inst in uses]): - var_name = self._mk_varname(var.name) - for inst in uses: - if inst.opcode == "mstore": - inst.opcode = "store" - inst.output = IRVariable(var_name) - inst.operands = [inst.operands[0]] - elif inst.opcode == "mload": - inst.opcode = "store" - inst.operands = [IRVariable(var_name)] - elif inst.opcode == "return": - bb = inst.parent - idx = len(bb.instructions) - 1 - assert inst == bb.instructions[idx] # sanity - bb.insert_instruction( - IRInstruction("mstore", [IRVariable(var_name), inst.operands[1]]), idx - ) + + var_name = self._mk_varname(var.name) + var = IRVariable(var_name) + for inst in uses: + if inst.opcode == "mstore": + inst.opcode = "store" + inst.output = var + inst.operands = [inst.operands[0]] + elif inst.opcode == "mload": + inst.opcode = "store" + inst.operands = [var] + elif inst.opcode == "return": + bb = inst.parent + idx = len(bb.instructions) - 1 + assert inst == bb.instructions[idx] # sanity + new_inst = IRInstruction("mstore", [var, inst.operands[1]]) + bb.insert_instruction(new_inst, idx) def _process_palloca_var(self, dfg: DFGAnalysis, palloca_inst: IRInstruction, var: IRVariable): """ @@ -70,16 +69,18 @@ def _process_palloca_var(self, dfg: DFGAnalysis, palloca_inst: IRInstruction, va return var_name = self._mk_varname(var.name) + var = IRVariable(var_name) + # some value given to us by the calling convention palloca_inst.opcode = "mload" palloca_inst.operands = [palloca_inst.operands[0]] - palloca_inst.output = IRVariable(var_name) + palloca_inst.output = var for inst in uses: if inst.opcode == "mstore": inst.opcode = "store" - inst.output = IRVariable(var_name) + inst.output = var inst.operands = [inst.operands[0]] elif inst.opcode == "mload": inst.opcode = "store" - inst.operands = [IRVariable(var_name)] + inst.operands = [var] From 705aa54ed62606cdb868ebbfab0924e71f1a3f3f Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Fri, 27 Dec 2024 21:10:30 +0100 Subject: [PATCH 51/70] feat[ux]: allow "compiling" `.vyi` files (#4290) This commit allows "compiling" `.vyi` files with ast, annotated_ast, interface, external_interface and abi output formats. Even though they don't produce bytecode, the above output formats can still be useful for users and tooling (even just to validate a `.vyi` file). --------- Co-authored-by: Charles Cooper --- tests/functional/codegen/test_interfaces.py | 105 ++++++++++++++++++ tests/unit/ast/test_ast_dict.py | 2 + .../cli/vyper_compile/test_compile_files.py | 29 +++++ vyper/ast/nodes.py | 2 +- vyper/ast/nodes.pyi | 1 + vyper/ast/parse.py | 8 +- vyper/compiler/__init__.py | 15 +++ vyper/compiler/output.py | 16 ++- vyper/compiler/phases.py | 3 + vyper/semantics/analysis/module.py | 2 +- vyper/semantics/types/user.py | 3 + 11 files changed, 179 insertions(+), 7 deletions(-) diff --git a/tests/functional/codegen/test_interfaces.py b/tests/functional/codegen/test_interfaces.py index e46a7d3dd4..e0b59ff668 100644 --- a/tests/functional/codegen/test_interfaces.py +++ b/tests/functional/codegen/test_interfaces.py @@ -876,3 +876,108 @@ def bar() -> uint256: input_bundle = make_input_bundle({"lib1.vy": lib1}) c = get_contract(main, input_bundle=input_bundle) assert c.bar() == 1 + + +def test_interface_with_flags(): + code = """ +struct MyStruct: + a: address + +flag Foo: + BOO + MOO + POO + +event Transfer: + sender: indexed(address) + +@external +def bar(): + pass +flag BAR: + BIZ + BAZ + BOO + +@external +@view +def foo(s: MyStruct) -> MyStruct: + return s + """ + + out = compile_code(code, contract_path="code.vy", output_formats=["interface"])["interface"] + + assert "# Flags" in out + assert "flag Foo:" in out + assert "flag BAR" in out + assert "BOO" in out + assert "MOO" in out + + compile_code(out, contract_path="code.vyi", output_formats=["interface"]) + + +vyi_filenames = [ + "test__test.vyi", + "test__t.vyi", + "t__test.vyi", + "t__t.vyi", + "t_t.vyi", + "test_test.vyi", + "t_test.vyi", + "test_t.vyi", + "_test_t__t_tt_.vyi", + "foo_bar_baz.vyi", +] + + +@pytest.mark.parametrize("vyi_filename", vyi_filenames) +def test_external_interface_names(vyi_filename): + code = """ +@external +def foo(): + ... + """ + + compile_code(code, contract_path=vyi_filename, output_formats=["external_interface"]) + + +def test_external_interface_with_flag(): + code = """ +flag Foo: + Blah + +@external +def foo() -> Foo: + ... + """ + + out = compile_code(code, contract_path="test__test.vyi", output_formats=["external_interface"])[ + "external_interface" + ] + assert "-> Foo:" in out + + +def test_external_interface_compiles_again(): + code = """ +@external +def foo() -> uint256: + ... +@external +def bar(a:int32) -> uint256: + ... + """ + + out = compile_code(code, contract_path="test.vyi", output_formats=["external_interface"])[ + "external_interface" + ] + compile_code(out, contract_path="test.vyi", output_formats=["external_interface"]) + + +@pytest.mark.xfail +def test_weird_interface_name(): + # based on comment https://github.com/vyperlang/vyper/pull/4290#discussion_r1884137428 + # we replace "_" for "" which results in an interface without name + out = compile_code("", contract_path="_.vyi", output_formats=["external_interface"])[ + "external_interface" + ] + assert "interface _:" in out diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index c9d7248809..196b1e24e6 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -399,6 +399,7 @@ def foo(): "node_id": 0, "path": "main.vy", "source_id": 1, + "is_interface": False, "type": { "name": "main.vy", "type_decl_node": {"node_id": 0, "source_id": 1}, @@ -1175,6 +1176,7 @@ def foo(): "node_id": 0, "path": "lib1.vy", "source_id": 0, + "is_interface": False, "type": { "name": "lib1.vy", "type_decl_node": {"node_id": 0, "source_id": 0}, diff --git a/tests/unit/cli/vyper_compile/test_compile_files.py b/tests/unit/cli/vyper_compile/test_compile_files.py index 3856aa3362..d8d9e56777 100644 --- a/tests/unit/cli/vyper_compile/test_compile_files.py +++ b/tests/unit/cli/vyper_compile/test_compile_files.py @@ -7,6 +7,7 @@ from vyper.cli.vyper_compile import compile_files from vyper.cli.vyper_json import compile_json +from vyper.compiler import INTERFACE_OUTPUT_FORMATS, OUTPUT_FORMATS from vyper.compiler.input_bundle import FilesystemInputBundle from vyper.compiler.output_bundle import OutputBundle from vyper.compiler.phases import CompilerData @@ -425,3 +426,31 @@ def test_archive_search_path(tmp_path_factory, make_file, chdir_tmp_path): used_dir = search_paths[-1].stem # either dir1 or dir2 assert output_bundle.used_search_paths == [".", "0/" + used_dir] + + +def test_compile_interface_file(make_file): + interface = """ +@view +@external +def foo() -> String[1]: + ... + +@view +@external +def bar() -> String[1]: + ... + +@external +def baz() -> uint8: + ... + + """ + file = make_file("interface.vyi", interface) + compile_files([file], INTERFACE_OUTPUT_FORMATS) + + # check unallowed output formats + for f in OUTPUT_FORMATS: + if f in INTERFACE_OUTPUT_FORMATS: + continue + with pytest.raises(ValueError): + compile_files([file], [f]) diff --git a/vyper/ast/nodes.py b/vyper/ast/nodes.py index 974685f403..ccc80947e4 100644 --- a/vyper/ast/nodes.py +++ b/vyper/ast/nodes.py @@ -638,7 +638,7 @@ class TopLevel(VyperNode): class Module(TopLevel): # metadata - __slots__ = ("path", "resolved_path", "source_id") + __slots__ = ("path", "resolved_path", "source_id", "is_interface") def to_dict(self): return dict(source_sha256sum=self.source_sha256sum, **super().to_dict()) diff --git a/vyper/ast/nodes.pyi b/vyper/ast/nodes.pyi index 783764271d..b00354c03a 100644 --- a/vyper/ast/nodes.pyi +++ b/vyper/ast/nodes.pyi @@ -71,6 +71,7 @@ class Module(TopLevel): path: str = ... resolved_path: str = ... source_id: int = ... + is_interface: bool = ... def namespace(self) -> Any: ... # context manager class FunctionDef(TopLevel): diff --git a/vyper/ast/parse.py b/vyper/ast/parse.py index d975aafac4..423b37721a 100644 --- a/vyper/ast/parse.py +++ b/vyper/ast/parse.py @@ -23,10 +23,11 @@ def parse_to_ast_with_settings( module_path: Optional[str] = None, resolved_path: Optional[str] = None, add_fn_node: Optional[str] = None, + is_interface: bool = False, ) -> tuple[Settings, vy_ast.Module]: try: return _parse_to_ast_with_settings( - vyper_source, source_id, module_path, resolved_path, add_fn_node + vyper_source, source_id, module_path, resolved_path, add_fn_node, is_interface ) except SyntaxException as e: e.resolved_path = resolved_path @@ -39,6 +40,7 @@ def _parse_to_ast_with_settings( module_path: Optional[str] = None, resolved_path: Optional[str] = None, add_fn_node: Optional[str] = None, + is_interface: bool = False, ) -> tuple[Settings, vy_ast.Module]: """ Parses a Vyper source string and generates basic Vyper AST nodes. @@ -62,6 +64,9 @@ def _parse_to_ast_with_settings( resolved_path: str, optional The resolved path of the source code Corresponds to FileInput.resolved_path + is_interface: bool + Indicates whether the source code should + be parsed as an interface file. Returns ------- @@ -106,6 +111,7 @@ def _parse_to_ast_with_settings( # Convert to Vyper AST. module = vy_ast.get_node(py_ast) assert isinstance(module, vy_ast.Module) # mypy hint + module.is_interface = is_interface return pre_parser.settings, module diff --git a/vyper/compiler/__init__.py b/vyper/compiler/__init__.py index d885599cec..57bd2f4096 100644 --- a/vyper/compiler/__init__.py +++ b/vyper/compiler/__init__.py @@ -46,6 +46,13 @@ "opcodes_runtime": output.build_opcodes_runtime_output, } +INTERFACE_OUTPUT_FORMATS = [ + "ast_dict", + "annotated_ast_dict", + "interface", + "external_interface", + "abi", +] UNKNOWN_CONTRACT_NAME = "" @@ -121,10 +128,18 @@ def outputs_from_compiler_data( output_formats = ("bytecode",) ret = {} + with anchor_settings(compiler_data.settings): for output_format in output_formats: if output_format not in OUTPUT_FORMATS: raise ValueError(f"Unsupported format type {repr(output_format)}") + + is_vyi = compiler_data.file_input.resolved_path.suffix == ".vyi" + if is_vyi and output_format not in INTERFACE_OUTPUT_FORMATS: + raise ValueError( + f"Unsupported format for compiling interface: {repr(output_format)}" + ) + try: formatter = OUTPUT_FORMATS[output_format] ret[output_format] = formatter(compiler_data) diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 1a6b3e9c07..ca951b8e39 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -108,9 +108,8 @@ def build_integrity(compiler_data: CompilerData) -> str: def build_external_interface_output(compiler_data: CompilerData) -> str: interface = compiler_data.annotated_vyper_module._metadata["type"].interface stem = PurePath(compiler_data.contract_path).stem - # capitalize words separated by '_' - # ex: test_interface.vy -> TestInterface - name = "".join([x.capitalize() for x in stem.split("_")]) + + name = stem.title().replace("_", "") out = f"\n# External Interfaces\ninterface {name}:\n" for func in interface.functions.values(): @@ -136,6 +135,14 @@ def build_interface_output(compiler_data: CompilerData) -> str: out += f" {member_name}: {member_type}\n" out += "\n\n" + if len(interface.flags) > 0: + out += "# Flags\n\n" + for flag in interface.flags.values(): + out += f"flag {flag.name}:\n" + for flag_value in flag._flag_members: + out += f" {flag_value}\n" + out += "\n\n" + if len(interface.events) > 0: out += "# Events\n\n" for event in interface.events.values(): @@ -282,7 +289,8 @@ def build_method_identifiers_output(compiler_data: CompilerData) -> dict: def build_abi_output(compiler_data: CompilerData) -> list: module_t = compiler_data.annotated_vyper_module._metadata["type"] - _ = compiler_data.ir_runtime # ensure _ir_info is generated + if not compiler_data.annotated_vyper_module.is_interface: + _ = compiler_data.ir_runtime # ensure _ir_info is generated abi = module_t.interface.to_toplevel_abi_dict() if module_t.init_function: diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 503281a867..4925d9971c 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -113,11 +113,14 @@ def contract_path(self): @cached_property def _generate_ast(self): + is_vyi = self.contract_path.suffix == ".vyi" + settings, ast = vy_ast.parse_to_ast_with_settings( self.source_code, self.source_id, module_path=self.contract_path.as_posix(), resolved_path=self.file_input.resolved_path.as_posix(), + is_interface=is_vyi, ) if self.original_settings: diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index 737f675b7c..534af4d633 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -53,7 +53,7 @@ def analyze_module(module_ast: vy_ast.Module) -> ModuleT: add all module-level objects to the namespace, type-check/validate semantics and annotate with type and analysis info """ - return _analyze_module_r(module_ast) + return _analyze_module_r(module_ast, module_ast.is_interface) def _analyze_module_r(module_ast: vy_ast.Module, is_interface: bool = False): diff --git a/vyper/semantics/types/user.py b/vyper/semantics/types/user.py index 73fa4878c7..d01ab23299 100644 --- a/vyper/semantics/types/user.py +++ b/vyper/semantics/types/user.py @@ -77,6 +77,9 @@ def get_type_member(self, key: str, node: vy_ast.VyperNode) -> "VyperType": self._helper.get_member(key, node) return self + def __str__(self): + return f"{self.name}" + def __repr__(self): arg_types = ",".join(repr(a) for a in self._flag_members) return f"flag {self.name}({arg_types})" From e9ac7cd623ebab98820cbafc93b8b4c1746d7070 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 28 Dec 2024 07:57:32 -0500 Subject: [PATCH 52/70] fix[stdlib]: fix `IERC4626` signatures (#4425) the signatures should not have default parameters; if used, they will create a different method id at the callsite than the ERC4626 standard actually accepts. - add `stdlib` to the list of valid PR scopes --- .github/workflows/pull-request.yaml | 2 ++ vyper/builtins/interfaces/IERC4626.vyi | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 2eb0113487..a2f4b5a0d1 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -32,6 +32,7 @@ jobs: # docs: documentation # test: test suite # lang: language changes + # stdlib: changes to the stdlib # ux: language changes (UX) # tool: integration # ir: (old) IR/codegen changes @@ -43,6 +44,7 @@ jobs: docs test lang + stdlib ux tool ir diff --git a/vyper/builtins/interfaces/IERC4626.vyi b/vyper/builtins/interfaces/IERC4626.vyi index 6d9e4c6ef7..0dd398d1f3 100644 --- a/vyper/builtins/interfaces/IERC4626.vyi +++ b/vyper/builtins/interfaces/IERC4626.vyi @@ -44,7 +44,7 @@ def previewDeposit(assets: uint256) -> uint256: ... @external -def deposit(assets: uint256, receiver: address=msg.sender) -> uint256: +def deposit(assets: uint256, receiver: address) -> uint256: ... @view @@ -58,7 +58,7 @@ def previewMint(shares: uint256) -> uint256: ... @external -def mint(shares: uint256, receiver: address=msg.sender) -> uint256: +def mint(shares: uint256, receiver: address) -> uint256: ... @view @@ -72,7 +72,7 @@ def previewWithdraw(assets: uint256) -> uint256: ... @external -def withdraw(assets: uint256, receiver: address=msg.sender, owner: address=msg.sender) -> uint256: +def withdraw(assets: uint256, receiver: address, owner: address) -> uint256: ... @view @@ -86,5 +86,5 @@ def previewRedeem(shares: uint256) -> uint256: ... @external -def redeem(shares: uint256, receiver: address=msg.sender, owner: address=msg.sender) -> uint256: +def redeem(shares: uint256, receiver: address, owner: address) -> uint256: ... From 614ea0da18110b9be4192dd96f3baad695b9e888 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 28 Dec 2024 16:45:49 -0500 Subject: [PATCH 53/70] fix[venom]: fix store elimination pass (#4428) this commit fixes the store elimination pass by updating the dfg in-place instead of relying on a stale dfg. this currently results in no bytecode changes. previously this was undetected because the order of items in the dfg happens to be "well-behaved", but if the dfg is built using a traversal of basic blocks in a different order (as may happen in upcoming passes), it can result in store instructions failing to be eliminated. note that we haven't rebuilt the dfg properly because `dfg.outputs` is invalid after this pass. we could modify `dfg.outputs` in place, but that results in a bytecode regression. this commit also removes the dependency on CFGAnalysis as it is not actually needed by the pass. --------- Co-authored-by: Harry Kalogirou --- vyper/venom/passes/store_elimination.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/vyper/venom/passes/store_elimination.py b/vyper/venom/passes/store_elimination.py index 22d4723013..a4f217505b 100644 --- a/vyper/venom/passes/store_elimination.py +++ b/vyper/venom/passes/store_elimination.py @@ -1,4 +1,4 @@ -from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, LivenessAnalysis +from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis from vyper.venom.basicblock import IRVariable from vyper.venom.passes.base_pass import IRPass @@ -13,31 +13,33 @@ class StoreElimination(IRPass): # with LoadElimination def run_pass(self): - self.analyses_cache.request_analysis(CFGAnalysis) - dfg = self.analyses_cache.request_analysis(DFGAnalysis) + self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) - for var, inst in dfg.outputs.items(): + for var, inst in self.dfg.outputs.items(): if inst.opcode != "store": continue - self._process_store(dfg, inst, var, inst.operands[0]) + self._process_store(inst, var, inst.operands[0]) self.analyses_cache.invalidate_analysis(LivenessAnalysis) self.analyses_cache.invalidate_analysis(DFGAnalysis) - def _process_store(self, dfg, inst, var: IRVariable, new_var: IRVariable): + def _process_store(self, inst, var: IRVariable, new_var: IRVariable): """ Process store instruction. If the variable is only used by a load instruction, forward the variable to the load instruction. """ - if any([inst.opcode == "phi" for inst in dfg.get_uses(new_var)]): + if any([inst.opcode == "phi" for inst in self.dfg.get_uses(new_var)]): return - uses = dfg.get_uses(var) + uses = self.dfg.get_uses(var) if any([inst.opcode == "phi" for inst in uses]): return - for use_inst in uses: + for use_inst in uses.copy(): for i, operand in enumerate(use_inst.operands): if operand == var: use_inst.operands[i] = new_var + self.dfg.add_use(new_var, use_inst) + self.dfg.remove_use(var, use_inst) + inst.parent.remove_instruction(inst) From 194d60ac6dbfe32d17d0073777817e8d95107d74 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 29 Dec 2024 15:35:15 -0500 Subject: [PATCH 54/70] fix[tool]: fix output formats for .vyz files (#4338) run the format name through the translate map. for instance, `annotated_ast` output format was not working for `.vyz` files. this commit has some additional fixes that were discovered when adding the integration tests and refactoring related to the settings not getting propagated uniformly across different entry points. --------- Co-authored-by: cyberthirst --- tests/functional/venom/test_venom_repr.py | 16 ++- .../cli/vyper_compile/test_compile_files.py | 99 +++++++++++++++++++ .../unit/cli/vyper_json/test_compile_json.py | 3 +- tests/unit/compiler/test_bytecode_runtime.py | 20 +++- vyper/cli/vyper_compile.py | 2 +- vyper/cli/vyper_json.py | 11 ++- vyper/compiler/phases.py | 2 + vyper/compiler/settings.py | 12 +-- vyper/ir/compile_ir.py | 5 + 9 files changed, 155 insertions(+), 15 deletions(-) diff --git a/tests/functional/venom/test_venom_repr.py b/tests/functional/venom/test_venom_repr.py index 5136672a03..1fb5d0486a 100644 --- a/tests/functional/venom/test_venom_repr.py +++ b/tests/functional/venom/test_venom_repr.py @@ -7,6 +7,7 @@ from tests.venom_utils import assert_ctx_eq, parse_venom from vyper.compiler import compile_code from vyper.compiler.phases import generate_bytecode +from vyper.compiler.settings import OptimizationLevel from vyper.venom import generate_assembly_experimental, run_passes_on from vyper.venom.context import IRContext @@ -20,7 +21,7 @@ def get_example_vy_filenames(): @pytest.mark.parametrize("vy_filename", get_example_vy_filenames()) -def test_round_trip_examples(vy_filename, optimize, compiler_settings): +def test_round_trip_examples(vy_filename, debug, optimize, compiler_settings, request): """ Check all examples round trip """ @@ -28,6 +29,11 @@ def test_round_trip_examples(vy_filename, optimize, compiler_settings): with open(path) as f: vyper_source = f.read() + if debug and optimize == OptimizationLevel.CODESIZE: + # FIXME: some round-trips fail when debug is enabled due to labels + # not getting pinned + request.node.add_marker(pytest.mark.xfail(strict=False)) + _round_trip_helper(vyper_source, optimize, compiler_settings) @@ -45,11 +51,17 @@ def _loop() -> uint256: @pytest.mark.parametrize("vyper_source", vyper_sources) -def test_round_trip_sources(vyper_source, optimize, compiler_settings): +def test_round_trip_sources(vyper_source, debug, optimize, compiler_settings, request): """ Test vyper_sources round trip """ vyper_source = textwrap.dedent(vyper_source) + + if debug and optimize == OptimizationLevel.CODESIZE: + # FIXME: some round-trips fail when debug is enabled due to labels + # not getting pinned + request.node.add_marker(pytest.mark.xfail(strict=False)) + _round_trip_helper(vyper_source, optimize, compiler_settings) diff --git a/tests/unit/cli/vyper_compile/test_compile_files.py b/tests/unit/cli/vyper_compile/test_compile_files.py index d8d9e56777..0fd938d519 100644 --- a/tests/unit/cli/vyper_compile/test_compile_files.py +++ b/tests/unit/cli/vyper_compile/test_compile_files.py @@ -361,6 +361,105 @@ def test_archive_b64_output(input_files): assert out[contract_file] == out2[archive_path] +def test_archive_compile_options(input_files): + tmpdir, _, _, contract_file = input_files + search_paths = [".", tmpdir] + + options = ["abi_python", "json", "ast", "annotated_ast", "ir_json"] + + for option in options: + out = compile_files([contract_file], ["archive_b64", option], paths=search_paths) + + archive_b64 = out[contract_file].pop("archive_b64") + + archive_path = Path("foo.zip.b64") + with archive_path.open("w") as f: + f.write(archive_b64) + + # compare compiling the two input bundles + out2 = compile_files([archive_path], [option]) + + if option in ["ast", "annotated_ast"]: + # would have to normalize paths and imports, so just verify it compiles + continue + + assert out[contract_file] == out2[archive_path] + + +format_options = [ + "bytecode", + "bytecode_runtime", + "blueprint_bytecode", + "abi", + "abi_python", + "source_map", + "source_map_runtime", + "method_identifiers", + "userdoc", + "devdoc", + "metadata", + "combined_json", + "layout", + "ast", + "annotated_ast", + "interface", + "external_interface", + "opcodes", + "opcodes_runtime", + "ir", + "ir_json", + "ir_runtime", + "asm", + "integrity", + "archive", + "solc_json", +] + + +def test_compile_vyz_with_options(input_files): + tmpdir, _, _, contract_file = input_files + search_paths = [".", tmpdir] + + for option in format_options: + out_archive = compile_files([contract_file], ["archive"], paths=search_paths) + + archive = out_archive[contract_file].pop("archive") + + archive_path = Path("foo.zip.out.vyz") + with archive_path.open("wb") as f: + f.write(archive) + + # compare compiling the two input bundles + out = compile_files([contract_file], [option], paths=search_paths) + out2 = compile_files([archive_path], [option]) + + if option in ["ast", "annotated_ast", "metadata"]: + # would have to normalize paths and imports, so just verify it compiles + continue + + if option in ["ir_runtime", "ir", "archive"]: + # ir+ir_runtime is different due to being different compiler runs + # archive is different due to different metadata (timestamps) + continue + + assert out[contract_file] == out2[archive_path] + + +def test_archive_compile_simultaneous_options(input_files): + tmpdir, _, _, contract_file = input_files + search_paths = [".", tmpdir] + + for option in format_options: + with pytest.raises(ValueError) as e: + _ = compile_files([contract_file], ["archive", option], paths=search_paths) + + err_opt = "archive" + if option in ("combined_json", "solc_json"): + err_opt = option + + assert f"If using {err_opt} it must be the only output format requested" in str(e.value) + + def test_solc_json_output(input_files): tmpdir, _, _, contract_file = input_files search_paths = [".", tmpdir] diff --git a/tests/unit/cli/vyper_json/test_compile_json.py b/tests/unit/cli/vyper_json/test_compile_json.py index 7e281bda2e..9044148aa9 100644 --- a/tests/unit/cli/vyper_json/test_compile_json.py +++ b/tests/unit/cli/vyper_json/test_compile_json.py @@ -73,7 +73,7 @@ def oopsie(a: uint256) -> bool: @pytest.fixture(scope="function") -def input_json(optimize, evm_version, experimental_codegen): +def input_json(optimize, evm_version, experimental_codegen, debug): return { "language": "Vyper", "sources": { @@ -87,6 +87,7 @@ def input_json(optimize, evm_version, experimental_codegen): "optimize": optimize.name.lower(), "evmVersion": evm_version, "experimentalCodegen": experimental_codegen, + "debug": debug, }, } diff --git a/tests/unit/compiler/test_bytecode_runtime.py b/tests/unit/compiler/test_bytecode_runtime.py index 1d38130c49..9fdc4c493f 100644 --- a/tests/unit/compiler/test_bytecode_runtime.py +++ b/tests/unit/compiler/test_bytecode_runtime.py @@ -54,7 +54,7 @@ def test_bytecode_runtime(): assert out["bytecode_runtime"].removeprefix("0x") in out["bytecode"].removeprefix("0x") -def test_bytecode_signature(): +def test_bytecode_signature(optimize, debug): out = vyper.compile_code( simple_contract_code, output_formats=["bytecode_runtime", "bytecode", "integrity"] ) @@ -65,10 +65,16 @@ def test_bytecode_signature(): metadata = _parse_cbor_metadata(initcode) integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + if debug and optimize == OptimizationLevel.CODESIZE: + # debug forces dense jumptable no matter the size of selector table + expected_data_section_lengths = [5, 7] + else: + expected_data_section_lengths = [] + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) - assert data_section_lengths == [] + assert data_section_lengths == expected_data_section_lengths assert immutables_len == 0 assert compiler == {"vyper": list(vyper.version.version_tuple)} @@ -119,7 +125,7 @@ def test_bytecode_signature_sparse_jumptable(): assert compiler == {"vyper": list(vyper.version.version_tuple)} -def test_bytecode_signature_immutables(): +def test_bytecode_signature_immutables(debug, optimize): out = vyper.compile_code( has_immutables, output_formats=["bytecode_runtime", "bytecode", "integrity"] ) @@ -130,10 +136,16 @@ def test_bytecode_signature_immutables(): metadata = _parse_cbor_metadata(initcode) integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + if debug and optimize == OptimizationLevel.CODESIZE: + # debug forces dense jumptable no matter the size of selector table + expected_data_section_lengths = [5, 7] + else: + expected_data_section_lengths = [] + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) - assert data_section_lengths == [] + assert data_section_lengths == expected_data_section_lengths assert immutables_len == 32 assert compiler == {"vyper": list(vyper.version.version_tuple)} diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index 046cac2c0b..390416799a 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -359,7 +359,7 @@ def compile_files( # we allow this instead of requiring a different mode (like # `--zip`) so that verifier pipelines do not need a different # workflow for archive files and single-file contracts. - output = compile_from_zip(file_name, output_formats, settings, no_bytecode_metadata) + output = compile_from_zip(file_name, final_formats, settings, no_bytecode_metadata) ret[file_path] = output continue except NotZipInput: diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index 9fcdf27baf..e7704b9398 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -272,8 +272,17 @@ def get_settings(input_dict: dict) -> Settings: else: assert optimize is None + debug = input_dict["settings"].get("debug", None) + + # TODO: maybe change these to camelCase for consistency + enable_decimals = input_dict["settings"].get("enable_decimals", None) + return Settings( - evm_version=evm_version, optimize=optimize, experimental_codegen=experimental_codegen + evm_version=evm_version, + optimize=optimize, + experimental_codegen=experimental_codegen, + debug=debug, + enable_decimals=enable_decimals, ) diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 4925d9971c..e6cb1c58d6 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -12,6 +12,7 @@ from vyper.compiler.input_bundle import FileInput, FilesystemInputBundle, InputBundle from vyper.compiler.settings import OptimizationLevel, Settings, anchor_settings, merge_settings from vyper.ir import compile_ir, optimizer +from vyper.ir.compile_ir import reset_symbols from vyper.semantics import analyze_module, set_data_positions, validate_compilation_target from vyper.semantics.analysis.data_positions import generate_layout_export from vyper.semantics.analysis.imports import resolve_imports @@ -310,6 +311,7 @@ def generate_ir_nodes(global_ctx: ModuleT, settings: Settings) -> tuple[IRnode, """ # make IR output the same between runs codegen.reset_names() + reset_symbols() with anchor_settings(settings): ir_nodes, ir_runtime = module.generate_ir_for_module(global_ctx) diff --git a/vyper/compiler/settings.py b/vyper/compiler/settings.py index a8e28c1ed1..e9840e8334 100644 --- a/vyper/compiler/settings.py +++ b/vyper/compiler/settings.py @@ -120,12 +120,12 @@ def _merge_one(lhs, rhs, helpstr): return lhs if rhs is None else rhs ret = Settings() - ret.evm_version = _merge_one(one.evm_version, two.evm_version, "evm version") - ret.optimize = _merge_one(one.optimize, two.optimize, "optimize") - ret.experimental_codegen = _merge_one( - one.experimental_codegen, two.experimental_codegen, "experimental codegen" - ) - ret.enable_decimals = _merge_one(one.enable_decimals, two.enable_decimals, "enable-decimals") + for field in dataclasses.fields(ret): + if field.name == "compiler_version": + continue + pretty_name = field.name.replace("_", "-") # e.g. evm_version -> evm-version + val = _merge_one(getattr(one, field.name), getattr(two, field.name), pretty_name) + setattr(ret, field.name, val) return ret diff --git a/vyper/ir/compile_ir.py b/vyper/ir/compile_ir.py index e87cf1b310..936e6d5d72 100644 --- a/vyper/ir/compile_ir.py +++ b/vyper/ir/compile_ir.py @@ -54,6 +54,11 @@ def mksymbol(name=""): return f"_sym_{name}{_next_symbol}" +def reset_symbols(): + global _next_symbol + _next_symbol = 0 + + def mkdebug(pc_debugger, ast_source): i = Instruction("DEBUG", ast_source) i.pc_debugger = pc_debugger From da5beb6bcf606b7262dbec5ca9a52484c604bb6e Mon Sep 17 00:00:00 2001 From: sudo rm -rf --no-preserve-root / Date: Tue, 31 Dec 2024 01:01:34 +0100 Subject: [PATCH 55/70] chore[docs]: `nonpayable` `internal` function behaviour (#4416) this commit adds a note to clarify that `nonpayable` `internal` functions can be called via `external` `payable` functions. --- docs/control-structures.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/control-structures.rst b/docs/control-structures.rst index d46e7a4a28..32fb80882a 100644 --- a/docs/control-structures.rst +++ b/docs/control-structures.rst @@ -75,6 +75,14 @@ Or for internal functions which are defined in :ref:`imported modules ` def calculate(amount: uint256) -> uint256: return calculator_library._times_two(amount) +Marking an internal function as ``payable`` specifies that the function can interact with ``msg.value``. A ``nonpayable`` internal function can be called from an external ``payable`` function, but it cannot access ``msg.value``. + +.. code-block:: vyper + + @payable + def _foo() -> uint256: + return msg.value % 2 + .. note:: As of v0.4.0, the ``@internal`` decorator is optional. That is, functions with no visibility decorator default to being ``internal``. @@ -110,7 +118,7 @@ You can optionally declare a function's mutability by using a :ref:`decorator Date: Tue, 31 Dec 2024 01:01:47 +0100 Subject: [PATCH 56/70] chore[docs]: abi function signature for default arguments (#4415) this commit adds a note on how the function selector is calculated if default arguments are used. --- docs/control-structures.rst | 11 ++++++++++- docs/interfaces.rst | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/control-structures.rst b/docs/control-structures.rst index 32fb80882a..6304637728 100644 --- a/docs/control-structures.rst +++ b/docs/control-structures.rst @@ -48,7 +48,16 @@ External functions (marked with the ``@external`` decorator) are a part of the c A Vyper contract cannot call directly between two external functions. If you must do this, you can use an :ref:`interface `. .. note:: - For external functions with default arguments like ``def my_function(x: uint256, b: uint256 = 1)`` the Vyper compiler will generate ``N+1`` overloaded function selectors based on ``N`` default arguments. + For external functions with default arguments like ``def my_function(x: uint256, b: uint256 = 1)`` the Vyper compiler will generate ``N+1`` overloaded function selectors based on ``N`` default arguments. Consequently, the ABI signature for a function (this includes interface functions) excludes optional arguments when their default values are used in the function call. + + .. code-block:: vyper + + from ethereum.ercs import IERC4626 + + @external + def foo(x: IERC4626): + extcall x.withdraw(0, self, self) # keccak256("withdraw(uint256,address,address)")[:4] = 0xb460af94 + extcall x.withdraw(0) # keccak256("withdraw(uint256)")[:4] = 0x2e1a7d4d .. _structure-functions-internal: diff --git a/docs/interfaces.rst b/docs/interfaces.rst index 9737d3c567..22a0874fa7 100644 --- a/docs/interfaces.rst +++ b/docs/interfaces.rst @@ -120,6 +120,10 @@ This imports the defined interface from the vyper file at ``an_interface.vyi`` ( Prior to v0.4.0, ``implements`` required that events defined in an interface were re-defined in the "implementing" contract. As of v0.4.0, this is no longer required because events can be used just by importing them. Any events used in a contract will automatically be exported in the ABI output. +.. note:: + + An interface function with default parameters (e.g. ``deposit(assets: uint256, receiver: address = msg.sender)``) implies that the contract being interfaced with supports these default arguments via the ABI-encoded function signatures (e.g. ``keccak256("deposit(uint256,address)")[:4]`` and ``keccak256("deposit(uint256)")[:4]``). It is the responsibility of the callee to implement the behavior associated with these defaults. + Standalone Interfaces ===================== From d67e57c1d9695ac0504ca8cb1425d8f16391de27 Mon Sep 17 00:00:00 2001 From: tserg <8017125+tserg@users.noreply.github.com> Date: Fri, 3 Jan 2025 23:24:42 +0800 Subject: [PATCH 57/70] feat[tool]: support storage layouts via `json` and `.vyz` inputs (#4370) This commit adds support for overriding the storage layout using `solc_json` and archive inputs, and consequently adding the storage layout if it was provided to these formats as output. This makes it possible for verifiers to verify code compiled with a storage layout override with no changes on their end. A design decision was made to have the storage layout override affect the integrity hash. This is so you can tell that a contract was compiled with storage layout override (even if it does not affect the bytecode). --------- Co-authored-by: Charles Cooper --- docs/compiling-a-contract.rst | 23 +++- .../test_storage_layout_overrides.py | 28 +++- .../cli/vyper_compile/test_compile_files.py | 120 ++++++++++++++---- .../unit/cli/vyper_json/test_compile_json.py | 50 +++++++- vyper/cli/compile_archive.py | 6 + vyper/cli/vyper_json.py | 20 +++ vyper/compiler/output.py | 2 +- vyper/compiler/output_bundle.py | 24 +++- vyper/compiler/phases.py | 33 +++-- vyper/semantics/analysis/imports.py | 4 +- 10 files changed, 262 insertions(+), 48 deletions(-) diff --git a/docs/compiling-a-contract.rst b/docs/compiling-a-contract.rst index 7132cff58d..c839e1e81d 100644 --- a/docs/compiling-a-contract.rst +++ b/docs/compiling-a-contract.rst @@ -106,7 +106,7 @@ Online Compilers Try VyperLang! ----------------- -`Try VyperLang! `_ is a JupterHub instance hosted by the Vyper team as a sandbox for developing and testing contracts in Vyper. It requires github for login, and supports deployment via the browser. +`Try VyperLang! `_ is a JupyterHub instance hosted by the Vyper team as a sandbox for developing and testing contracts in Vyper. It requires github for login, and supports deployment via the browser. Remix IDE --------- @@ -203,7 +203,7 @@ The following is a list of supported EVM versions, and changes in the compiler i Integrity Hash ============== -To help tooling detect whether two builds are the same, Vyper provides the ``-f integrity`` output, which outputs the integrity hash of a contract. The integrity hash is recursively defined as the sha256 of the source code with the integrity hashes of its dependencies (imports). +To help tooling detect whether two builds are the same, Vyper provides the ``-f integrity`` output, which outputs the integrity hash of a contract. The integrity hash is recursively defined as the sha256 of the source code with the integrity hashes of its dependencies (imports) and storage layout overrides (if provided). .. _vyper-archives: @@ -219,8 +219,9 @@ A Vyper archive is a compileable bundle of input sources and settings. Technical ├── compilation_targets ├── compiler_version ├── integrity + ├── settings.json ├── searchpaths - └── settings.json + └── storage_layout.json [OPTIONAL] * ``cli_settings.txt`` is a text representation of the settings that were used on the compilation run that generated this archive. * ``compilation_targets`` is a newline separated list of compilation targets. Currently only one compilation is supported @@ -228,6 +229,7 @@ A Vyper archive is a compileable bundle of input sources and settings. Technical * ``integrity`` is the :ref:`integrity hash ` of the input contract * ``searchpaths`` is a newline-separated list of the search paths used on this compilation run * ``settings.json`` is a json representation of the settings used on this compilation run. It is 1:1 with ``cli_settings.txt``, but both are provided as they are convenient for different workflows (typically, manually vs automated). +* ``storage_layout.json`` is a json representation of the storage layout overrides to be used on this compilation run. It is optional. A Vyper archive file can be produced by requesting the ``-f archive`` output format. The compiler can also produce the archive in base64 encoded form using the ``--base64`` flag. The Vyper compiler can accept both ``.vyz`` and base64-encoded Vyper archives directly as input. @@ -281,6 +283,14 @@ The following example describes the expected input format of ``vyper-json``. (Co } }, // Optional + // Storage layout overrides for the contracts that are compiled + "storage_layout_overrides": { + "contracts/foo.vy": { + "a": {"type": "uint256", "slot": 1, "n_slots": 1}, + "b": {"type": "uint256", "slot": 0, "n_slots": 1}, + } + }, + // Optional "settings": { "evmVersion": "cancun", // EVM version to compile for. Can be london, paris, shanghai or cancun (default). // optional, optimization mode @@ -364,6 +374,13 @@ The following example describes the output format of ``vyper-json``. Comments ar "formattedMessage": "line 5:11 Unsupported type conversion: int128 to bool" } ], + // Optional: not present if there are no storage layout overrides + "storage_layout_overrides": { + "contracts/foo.vy": { + "a": {"type": "uint256", "slot": 1, "n_slots": 1}, + "b": {"type": "uint256", "slot": 0, "n_slots": 1}, + } + }, // This contains the file-level outputs. Can be limited/filtered by the outputSelection settings. "sources": { "source_file.vy": { diff --git a/tests/unit/cli/storage_layout/test_storage_layout_overrides.py b/tests/unit/cli/storage_layout/test_storage_layout_overrides.py index f02a8471e2..d2c495a2fa 100644 --- a/tests/unit/cli/storage_layout/test_storage_layout_overrides.py +++ b/tests/unit/cli/storage_layout/test_storage_layout_overrides.py @@ -2,6 +2,7 @@ import pytest +from vyper.cli.vyper_json import compile_json from vyper.compiler import compile_code from vyper.evm.opcodes import version_check from vyper.exceptions import StorageLayoutException @@ -13,7 +14,7 @@ def test_storage_layout_overrides(): b: uint256""" storage_layout_overrides = { - "a": {"type": "uint256", "slot": 1, "n_slots": 1}, + "a": {"type": "uint256", "slot": 5, "n_slots": 1}, "b": {"type": "uint256", "slot": 0, "n_slots": 1}, } @@ -26,6 +27,31 @@ def test_storage_layout_overrides(): assert out["layout"] == expected_output +def test_storage_layout_overrides_json(): + code = """ +a: uint256 +b: uint256""" + + storage_layout_overrides = { + "a": {"type": "uint256", "slot": 5, "n_slots": 1}, + "b": {"type": "uint256", "slot": 0, "n_slots": 1}, + } + + input_json = { + "language": "Vyper", + "sources": {"contracts/foo.vy": {"content": code}}, + "storage_layout_overrides": {"contracts/foo.vy": storage_layout_overrides}, + "settings": {"outputSelection": {"*": ["*"]}}, + } + + out = compile_code( + code, output_formats=["layout"], storage_layout_override=storage_layout_overrides + ) + assert ( + compile_json(input_json)["contracts"]["contracts/foo.vy"]["foo"]["layout"] == out["layout"] + ) + + def test_storage_layout_for_more_complex(): code = """ foo: HashMap[address, uint256] diff --git a/tests/unit/cli/vyper_compile/test_compile_files.py b/tests/unit/cli/vyper_compile/test_compile_files.py index 0fd938d519..a1f5ca098c 100644 --- a/tests/unit/cli/vyper_compile/test_compile_files.py +++ b/tests/unit/cli/vyper_compile/test_compile_files.py @@ -1,18 +1,27 @@ import contextlib +import json import sys +import warnings import zipfile from pathlib import Path import pytest +from vyper.cli.compile_archive import compiler_data_from_zip from vyper.cli.vyper_compile import compile_files -from vyper.cli.vyper_json import compile_json +from vyper.cli.vyper_json import compile_from_input_dict, compile_json from vyper.compiler import INTERFACE_OUTPUT_FORMATS, OUTPUT_FORMATS from vyper.compiler.input_bundle import FilesystemInputBundle from vyper.compiler.output_bundle import OutputBundle from vyper.compiler.phases import CompilerData from vyper.utils import sha256sum +TAMPERED_INTEGRITY_SUM = sha256sum("tampered integrity sum") + +INTEGRITY_WARNING = f"Mismatched integrity sum! Expected {TAMPERED_INTEGRITY_SUM}" +INTEGRITY_WARNING += " but got {integrity}." # noqa: FS003 +INTEGRITY_WARNING += " (This likely indicates a corrupted archive)" + def test_combined_json_keys(chdir_tmp_path, make_file): make_file("bar.vy", "") @@ -298,6 +307,9 @@ def foo() -> uint256: import lib import jsonabi +a: uint256 +b: uint256 + @external def foo() -> uint256: return lib.foo() @@ -306,28 +318,52 @@ def foo() -> uint256: def bar(x: uint256) -> uint256: return extcall jsonabi(msg.sender).test_json(x) """ + storage_layout_overrides = { + "a": {"type": "uint256", "n_slots": 1, "slot": 5}, + "b": {"type": "uint256", "n_slots": 1, "slot": 0}, + } + storage_layout_source = json.dumps(storage_layout_overrides) + tmpdir = tmp_path_factory.mktemp("fake-package") with open(tmpdir / "lib.vy", "w") as f: f.write(library_source) with open(tmpdir / "jsonabi.json", "w") as f: f.write(json_source) + with open(tmpdir / "layout.json", "w") as f: + f.write(storage_layout_source) contract_file = make_file("contract.vy", contract_source) - return (tmpdir, tmpdir / "lib.vy", tmpdir / "jsonabi.json", contract_file) + contract_hash = sha256sum(contract_source) + library_hash = sha256sum(library_source) + jsonabi_hash = sha256sum(json_source) + resolved_imports_hash = sha256sum(contract_hash + sha256sum(library_hash) + jsonabi_hash) + storage_layout_hash = sha256sum(storage_layout_source) + expected_integrity = sha256sum(storage_layout_hash + resolved_imports_hash) + + return ( + tmpdir, + tmpdir / "lib.vy", + tmpdir / "jsonabi.json", + tmpdir / "layout.json", + contract_file, + expected_integrity, + ) def test_import_sys_path(input_files): - tmpdir, _, _, contract_file = input_files + tmpdir, _, _, _, contract_file, _ = input_files with mock_sys_path(tmpdir): assert compile_files([contract_file], ["combined_json"]) is not None def test_archive_output(input_files): - tmpdir, _, _, contract_file = input_files + tmpdir, library_file, jsonabi_file, storage_layout_path, contract_file, integrity = input_files search_paths = [".", tmpdir] - s = compile_files([contract_file], ["archive"], paths=search_paths) + s = compile_files( + [contract_file], ["archive"], paths=search_paths, storage_layout_paths=[storage_layout_path] + ) archive_bytes = s[contract_file]["archive"] archive_path = Path("foo.zip") @@ -337,13 +373,28 @@ def test_archive_output(input_files): assert zipfile.is_zipfile(archive_path) # compare compiling the two input bundles - out = compile_files([contract_file], ["integrity", "bytecode"], paths=search_paths) - out2 = compile_files([archive_path], ["integrity", "bytecode"]) + out = compile_files( + [contract_file], + ["integrity", "bytecode", "layout"], + paths=search_paths, + storage_layout_paths=[storage_layout_path], + ) + out2 = compile_files([archive_path], ["integrity", "bytecode", "layout"]) assert out[contract_file] == out2[archive_path] + # tamper with the integrity sum + archive_compiler_data = compiler_data_from_zip(archive_path, None, False) + archive_compiler_data.expected_integrity_sum = TAMPERED_INTEGRITY_SUM + + with warnings.catch_warnings(record=True) as w: + assert archive_compiler_data.integrity_sum is not None + + assert len(w) == 1, [s.message for s in w] + assert str(w[0].message).startswith(INTEGRITY_WARNING.format(integrity=integrity)) + def test_archive_b64_output(input_files): - tmpdir, _, _, contract_file = input_files + tmpdir, _, _, _, contract_file, _ = input_files search_paths = [".", tmpdir] out = compile_files( @@ -362,7 +413,7 @@ def test_archive_b64_output(input_files): def test_archive_compile_options(input_files): - tmpdir, _, _, contract_file = input_files + tmpdir, _, _, _, contract_file, _ = input_files search_paths = [".", tmpdir] options = ["abi_python", "json", "ast", "annotated_ast", "ir_json"] @@ -417,7 +468,7 @@ def test_archive_compile_options(input_files): def test_compile_vyz_with_options(input_files): - tmpdir, _, _, contract_file = input_files + tmpdir, _, _, _, contract_file, _ = input_files search_paths = [".", tmpdir] for option in format_options: @@ -446,7 +497,7 @@ def test_compile_vyz_with_options(input_files): def test_archive_compile_simultaneous_options(input_files): - tmpdir, _, _, contract_file = input_files + tmpdir, _, _, _, contract_file, _ = input_files search_paths = [".", tmpdir] for option in format_options: @@ -461,40 +512,55 @@ def test_archive_compile_simultaneous_options(input_files): def test_solc_json_output(input_files): - tmpdir, _, _, contract_file = input_files + tmpdir, _, _, storage_layout_path, contract_file, integrity = input_files search_paths = [".", tmpdir] - out = compile_files([contract_file], ["solc_json"], paths=search_paths) - + out = compile_files( + [contract_file], + ["solc_json"], + paths=search_paths, + storage_layout_paths=[storage_layout_path], + ) json_input = out[contract_file]["solc_json"] # check that round-tripping solc_json thru standard json produces # the same as compiling directly json_out = compile_json(json_input)["contracts"]["contract.vy"] json_out_bytecode = json_out["contract"]["evm"]["bytecode"]["object"] + json_out_layout = json_out["contract"]["layout"]["storage_layout"] - out2 = compile_files([contract_file], ["integrity", "bytecode"], paths=search_paths) + out2 = compile_files( + [contract_file], + ["integrity", "bytecode", "layout"], + paths=search_paths, + storage_layout_paths=[storage_layout_path], + ) assert out2[contract_file]["bytecode"] == json_out_bytecode + assert out2[contract_file]["layout"]["storage_layout"] == json_out_layout + + # tamper with the integrity sum + json_input["integrity"] = TAMPERED_INTEGRITY_SUM + _, warn_data = compile_from_input_dict(json_input) + + w = warn_data[Path("contract.vy")] + assert len(w) == 1, [s.message for s in w] + assert str(w[0].message).startswith(INTEGRITY_WARNING.format(integrity=integrity)) # maybe this belongs in tests/unit/compiler? def test_integrity_sum(input_files): - tmpdir, library_file, jsonabi_file, contract_file = input_files + tmpdir, library_file, jsonabi_file, storage_layout_path, contract_file, integrity = input_files search_paths = [".", tmpdir] - out = compile_files([contract_file], ["integrity"], paths=search_paths) - - with library_file.open() as f, contract_file.open() as g, jsonabi_file.open() as h: - library_contents = f.read() - contract_contents = g.read() - jsonabi_contents = h.read() + out = compile_files( + [contract_file], + ["integrity"], + paths=search_paths, + storage_layout_paths=[storage_layout_path], + ) - contract_hash = sha256sum(contract_contents) - library_hash = sha256sum(library_contents) - jsonabi_hash = sha256sum(jsonabi_contents) - expected = sha256sum(contract_hash + sha256sum(library_hash) + jsonabi_hash) - assert out[contract_file]["integrity"] == expected + assert out[contract_file]["integrity"] == integrity # does this belong in tests/unit/compiler? diff --git a/tests/unit/cli/vyper_json/test_compile_json.py b/tests/unit/cli/vyper_json/test_compile_json.py index 9044148aa9..f921d250a4 100644 --- a/tests/unit/cli/vyper_json/test_compile_json.py +++ b/tests/unit/cli/vyper_json/test_compile_json.py @@ -20,6 +20,9 @@ import contracts.library as library +a: uint256 +b: uint256 + @external def foo(a: address) -> bool: return extcall IBar(a).bar(1) @@ -29,16 +32,29 @@ def baz() -> uint256: return self.balance + library.foo() """ +FOO_STORAGE_LAYOUT_OVERRIDES = { + "a": {"type": "uint256", "n_slots": 1, "slot": 5}, + "b": {"type": "uint256", "n_slots": 1, "slot": 0}, +} + BAR_CODE = """ import contracts.ibar as IBar implements: IBar +c: uint256 +d: uint256 + @external def bar(a: uint256) -> bool: return True """ +BAR_STORAGE_LAYOUT_OVERRIDES = { + "c": {"type": "uint256", "n_slots": 1, "slot": 13}, + "d": {"type": "uint256", "n_slots": 1, "slot": 7}, +} + BAR_VYI = """ @external def bar(a: uint256) -> bool: @@ -89,6 +105,10 @@ def input_json(optimize, evm_version, experimental_codegen, debug): "experimentalCodegen": experimental_codegen, "debug": debug, }, + "storage_layout_overrides": { + "contracts/foo.vy": FOO_STORAGE_LAYOUT_OVERRIDES, + "contracts/bar.vy": BAR_STORAGE_LAYOUT_OVERRIDES, + }, } @@ -128,7 +148,10 @@ def test_compile_json(input_json, input_bundle): del output_formats["cfg"] del output_formats["cfg_runtime"] foo = compile_from_file_input( - foo_input, output_formats=output_formats, input_bundle=input_bundle + foo_input, + output_formats=output_formats, + input_bundle=input_bundle, + storage_layout_override=FOO_STORAGE_LAYOUT_OVERRIDES, ) library_input = input_bundle.load_file("contracts/library.vy") @@ -138,7 +161,10 @@ def test_compile_json(input_json, input_bundle): bar_input = input_bundle.load_file("contracts/bar.vy") bar = compile_from_file_input( - bar_input, output_formats=output_formats, input_bundle=input_bundle + bar_input, + output_formats=output_formats, + input_bundle=input_bundle, + storage_layout_override=BAR_STORAGE_LAYOUT_OVERRIDES, ) compile_code_results = { @@ -171,6 +197,7 @@ def test_compile_json(input_json, input_bundle): "interface": data["interface"], "ir": data["ir_dict"], "userdoc": data["userdoc"], + "layout": data["layout"], "metadata": data["metadata"], "evm": { "bytecode": { @@ -218,7 +245,16 @@ def test_different_outputs(input_bundle, input_json): foo = contracts["contracts/foo.vy"]["foo"] bar = contracts["contracts/bar.vy"]["bar"] - assert sorted(bar.keys()) == ["abi", "devdoc", "evm", "interface", "ir", "metadata", "userdoc"] + assert sorted(bar.keys()) == [ + "abi", + "devdoc", + "evm", + "interface", + "ir", + "layout", + "metadata", + "userdoc", + ] assert sorted(foo.keys()) == ["evm"] @@ -270,6 +306,14 @@ def test_exc_handler_to_dict_compiler(input_json): assert error["type"] == "TypeMismatch" +def test_unknown_storage_layout_overrides(input_json): + unknown_contract_path = "contracts/baz.vy" + input_json["storage_layout_overrides"] = {unknown_contract_path: FOO_STORAGE_LAYOUT_OVERRIDES} + with pytest.raises(JSONError) as e: + compile_json(input_json) + assert e.value.args[0] == f"unknown target for storage layout override: {unknown_contract_path}" + + def test_source_ids_increment(input_json): input_json["settings"]["outputSelection"] = {"*": ["ast", "evm.deployedBytecode.sourceMap"]} result = compile_json(input_json) diff --git a/vyper/cli/compile_archive.py b/vyper/cli/compile_archive.py index c6d07de9f1..d1dd2588ad 100644 --- a/vyper/cli/compile_archive.py +++ b/vyper/cli/compile_archive.py @@ -45,6 +45,11 @@ def compiler_data_from_zip(file_name, settings, no_bytecode_metadata): fcontents = archive.read("MANIFEST/compilation_targets").decode("utf-8") compilation_targets = fcontents.splitlines() + storage_layout_path = "MANIFEST/storage_layout.json" + storage_layout = None + if storage_layout_path in archive.namelist(): + storage_layout = json.loads(archive.read(storage_layout_path).decode("utf-8")) + if len(compilation_targets) != 1: raise BadArchive("Multiple compilation targets not supported!") @@ -68,6 +73,7 @@ def compiler_data_from_zip(file_name, settings, no_bytecode_metadata): return CompilerData( file, input_bundle=input_bundle, + storage_layout=storage_layout, integrity_sum=integrity, settings=settings, no_bytecode_metadata=no_bytecode_metadata, diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index e7704b9398..5f632f4167 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -12,6 +12,7 @@ from vyper.compiler.settings import OptimizationLevel, Settings from vyper.evm.opcodes import EVM_VERSIONS from vyper.exceptions import JSONError +from vyper.typing import StorageLayout from vyper.utils import OrderedSet, keccak256 TRANSLATE_MAP = { @@ -206,6 +207,19 @@ def get_inputs(input_dict: dict) -> dict[PurePath, Any]: return ret +def get_storage_layout_overrides(input_dict: dict) -> dict[PurePath, StorageLayout]: + storage_layout_overrides: dict[PurePath, StorageLayout] = {} + + for path, value in input_dict.get("storage_layout_overrides", {}).items(): + if path not in input_dict["sources"]: + raise JSONError(f"unknown target for storage layout override: {path}") + + path = PurePath(path) + storage_layout_overrides[path] = value + + return storage_layout_overrides + + # get unique output formats for each contract, given the input_dict # NOTE: would maybe be nice to raise on duplicated output formats def get_output_formats(input_dict: dict) -> dict[PurePath, list[str]]: @@ -299,6 +313,7 @@ def compile_from_input_dict( integrity = input_dict.get("integrity") sources = get_inputs(input_dict) + storage_layout_overrides = get_storage_layout_overrides(input_dict) output_formats = get_output_formats(input_dict) compilation_targets = list(output_formats.keys()) search_paths = get_search_paths(input_dict) @@ -308,6 +323,7 @@ def compile_from_input_dict( res, warnings_dict = {}, {} warnings.simplefilter("always") for contract_path in compilation_targets: + storage_layout_override = storage_layout_overrides.get(contract_path) with warnings.catch_warnings(record=True) as caught_warnings: try: # use load_file to get a unique source_id @@ -317,6 +333,7 @@ def compile_from_input_dict( file, input_bundle=input_bundle, output_formats=output_formats[contract_path], + storage_layout_override=storage_layout_override, integrity_sum=integrity, settings=settings, no_bytecode_metadata=no_bytecode_metadata, @@ -356,6 +373,9 @@ def format_to_output_dict(compiler_data: dict) -> dict: if key in data: output_contracts[key] = data[key] + if "layout" in data: + output_contracts["layout"] = data["layout"] + if "method_identifiers" in data: output_contracts["evm"] = {"methodIdentifiers": data["method_identifiers"]} diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index ca951b8e39..b6a0e8ac8c 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -102,7 +102,7 @@ def build_archive_b64(compiler_data: CompilerData) -> str: def build_integrity(compiler_data: CompilerData) -> str: - return compiler_data.resolved_imports.integrity_sum + return compiler_data.integrity_sum def build_external_interface_output(compiler_data: CompilerData) -> str: diff --git a/vyper/compiler/output_bundle.py b/vyper/compiler/output_bundle.py index 24a0d070cc..8af1b72289 100644 --- a/vyper/compiler/output_bundle.py +++ b/vyper/compiler/output_bundle.py @@ -12,6 +12,7 @@ from vyper.compiler.settings import Settings from vyper.exceptions import CompilerPanic from vyper.semantics.analysis.imports import _is_builtin +from vyper.typing import StorageLayout from vyper.utils import get_long_version, safe_relpath # data structures and routines for constructing "output bundles", @@ -134,6 +135,11 @@ def bundle(self): def write_sources(self, sources: dict[str, CompilerInput]): raise NotImplementedError(f"write_sources: {self.__class__}") + def write_storage_layout_overrides( + self, compilation_target_path: str, storage_layout_override: StorageLayout + ): + raise NotImplementedError(f"write_storage_layout_overrides: {self.__class__}") + def write_search_paths(self, search_paths: list[str]): raise NotImplementedError(f"write_search_paths: {self.__class__}") @@ -158,8 +164,12 @@ def write(self): self.write_compilation_target([self.bundle.compilation_target_path]) self.write_search_paths(self.bundle.used_search_paths) self.write_settings(self.compiler_data.original_settings) - self.write_integrity(self.compiler_data.resolved_imports.integrity_sum) + self.write_integrity(self.compiler_data.integrity_sum) self.write_sources(self.bundle.compiler_inputs) + if self.compiler_data.storage_layout_override is not None: + self.write_storage_layout_overrides( + self.bundle.compilation_target_path, self.compiler_data.storage_layout_override + ) class SolcJSONWriter(OutputBundleWriter): @@ -175,6 +185,13 @@ def write_sources(self, sources: dict[str, CompilerInput]): self._output["sources"].update(out) + def write_storage_layout_overrides( + self, compilation_target_path: str, storage_layout_override: StorageLayout + ): + self._output["storage_layout_overrides"] = { + compilation_target_path: storage_layout_override + } + def write_search_paths(self, search_paths: list[str]): self._output["settings"]["search_paths"] = search_paths @@ -237,6 +254,11 @@ def write_sources(self, sources: dict[str, CompilerInput]): for path, c in sources.items(): self.archive.writestr(_anonymize(path), c.contents) + def write_storage_layout_overrides( + self, compilation_target_path: str, storage_layout_override: StorageLayout + ): + self.archive.writestr("MANIFEST/storage_layout.json", json.dumps(storage_layout_override)) + def write_search_paths(self, search_paths: list[str]): self.archive.writestr("MANIFEST/searchpaths", "\n".join(search_paths)) diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index e6cb1c58d6..17812ee535 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -1,4 +1,5 @@ import copy +import json import warnings from functools import cached_property from pathlib import Path, PurePath @@ -19,7 +20,7 @@ from vyper.semantics.types.function import ContractFunctionT from vyper.semantics.types.module import ModuleT from vyper.typing import StorageLayout -from vyper.utils import ERC5202_PREFIX, vyper_warn +from vyper.utils import ERC5202_PREFIX, sha256sum, vyper_warn from vyper.venom import generate_assembly_experimental, generate_ir DEFAULT_CONTRACT_PATH = PurePath("VyperContract.vy") @@ -151,29 +152,41 @@ def vyper_module(self): _, ast = self._generate_ast return ast + def _compute_integrity_sum(self, imports_integrity_sum: str) -> str: + if self.storage_layout_override is not None: + layout_sum = sha256sum(json.dumps(self.storage_layout_override)) + return sha256sum(layout_sum + imports_integrity_sum) + return imports_integrity_sum + @cached_property def _resolve_imports(self): # deepcopy so as to not interfere with `-f ast` output vyper_module = copy.deepcopy(self.vyper_module) with self.input_bundle.search_path(Path(vyper_module.resolved_path).parent): - return vyper_module, resolve_imports(vyper_module, self.input_bundle) + imports = resolve_imports(vyper_module, self.input_bundle) - @cached_property - def resolved_imports(self): - imports = self._resolve_imports[1] + # check integrity sum + integrity_sum = self._compute_integrity_sum(imports._integrity_sum) expected = self.expected_integrity_sum - - if expected is not None and imports.integrity_sum != expected: + if expected is not None and integrity_sum != expected: # warn for now. strict/relaxed mode was considered but it costs # interface and testing complexity to add another feature flag. vyper_warn( f"Mismatched integrity sum! Expected {expected}" - f" but got {imports.integrity_sum}." + f" but got {integrity_sum}." " (This likely indicates a corrupted archive)" ) - return imports + return vyper_module, imports, integrity_sum + + @cached_property + def integrity_sum(self): + return self._resolve_imports[2] + + @cached_property + def resolved_imports(self): + return self._resolve_imports[1] @cached_property def _annotate(self) -> tuple[natspec.NatspecOutput, vy_ast.Module]: @@ -271,7 +284,7 @@ def assembly_runtime(self) -> list: def bytecode(self) -> bytes: metadata = None if not self.no_bytecode_metadata: - metadata = bytes.fromhex(self.resolved_imports.integrity_sum) + metadata = bytes.fromhex(self.integrity_sum) return generate_bytecode(self.assembly, compiler_metadata=metadata) @cached_property diff --git a/vyper/semantics/analysis/imports.py b/vyper/semantics/analysis/imports.py index 4f8daefa97..bcd62feb07 100644 --- a/vyper/semantics/analysis/imports.py +++ b/vyper/semantics/analysis/imports.py @@ -79,14 +79,14 @@ def __init__(self, input_bundle: InputBundle, graph: _ImportGraph): self.seen: set[int] = set() - self.integrity_sum = None + self._integrity_sum = None # should be all system paths + topmost module path self.absolute_search_paths = input_bundle.search_paths.copy() def resolve_imports(self, module_ast: vy_ast.Module): self._resolve_imports_r(module_ast) - self.integrity_sum = self._calculate_integrity_sum_r(module_ast) + self._integrity_sum = self._calculate_integrity_sum_r(module_ast) def _calculate_integrity_sum_r(self, module_ast: vy_ast.Module): acc = [sha256sum(module_ast.full_source_code)] From 4507d2a6abbf168c2cbed7681fdf816ee8abc1d1 Mon Sep 17 00:00:00 2001 From: sandbubbles <160503471+sandbubbles@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:44:14 +0100 Subject: [PATCH 58/70] fix[ux]: improve error message on failed imports (#4409) Previously, when an import failed, the error message only displayed the paths that were attempted, but did not point to the specific import statement which caused the exception. To address this, we wrap the main node-handling loop in `ImportAnalyzer` with `tag_exception`, which propagates information with the specific line and file to the error message. --- tests/functional/syntax/test_import.py | 9 ++++++--- tests/functional/syntax/test_interfaces.py | 15 ++++++++++----- vyper/semantics/analysis/imports.py | 10 ++++++---- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/functional/syntax/test_import.py b/tests/functional/syntax/test_import.py index 07b1a336c3..acc556206e 100644 --- a/tests/functional/syntax/test_import.py +++ b/tests/functional/syntax/test_import.py @@ -28,8 +28,9 @@ def foo(): ) file_input = input_bundle.load_file("top.vy") - with pytest.raises(ModuleNotFound): + with pytest.raises(ModuleNotFound) as e: compiler.compile_from_file_input(file_input, input_bundle=input_bundle) + assert "lib0.vy:" in str(e.value) def test_implicitly_relative_import_crashes_2(make_input_bundle): @@ -44,8 +45,9 @@ def foo(): ) file_input = input_bundle.load_file("top.vy") - with pytest.raises(ModuleNotFound): + with pytest.raises(ModuleNotFound) as e: compiler.compile_from_file_input(file_input, input_bundle=input_bundle) + assert "lib0.vy:" in str(e.value) def test_relative_import_searches_only_current_path(make_input_bundle): @@ -70,8 +72,9 @@ def foo(): input_bundle = make_input_bundle({"top.vy": top, "a.vy": a, "subdir/b.vy": b}) file_input = input_bundle.load_file("top.vy") - with pytest.raises(ModuleNotFound): + with pytest.raises(ModuleNotFound) as e: compiler.compile_from_file_input(file_input, input_bundle=input_bundle) + assert "b.vy:" in str(e.value) def test_absolute_import_within_relative_import(make_input_bundle): diff --git a/tests/functional/syntax/test_interfaces.py b/tests/functional/syntax/test_interfaces.py index baf0c73c30..fcfa5ba0c9 100644 --- a/tests/functional/syntax/test_interfaces.py +++ b/tests/functional/syntax/test_interfaces.py @@ -411,26 +411,31 @@ def foobar(): assert compiler.compile_code(code, input_bundle=input_bundle) is not None -def test_builtins_not_found(): +def test_builtins_not_found(make_input_bundle): code = """ from vyper.interfaces import foobar """ + input_bundle = make_input_bundle({"code.vy": code}) + file_input = input_bundle.load_file("code.vy") with pytest.raises(ModuleNotFound) as e: - compiler.compile_code(code) - + compiler.compile_from_file_input(file_input, input_bundle=input_bundle) assert e.value._message == "vyper.interfaces.foobar" assert e.value._hint == "try renaming `vyper.interfaces` to `ethereum.ercs`" + assert "code.vy:" in str(e.value) @pytest.mark.parametrize("erc", ("ERC20", "ERC721", "ERC4626")) -def test_builtins_not_found2(erc): +def test_builtins_not_found2(erc, make_input_bundle): code = f""" from ethereum.ercs import {erc} """ + input_bundle = make_input_bundle({"code.vy": code}) + file_input = input_bundle.load_file("code.vy") with pytest.raises(ModuleNotFound) as e: - compiler.compile_code(code) + compiler.compile_from_file_input(file_input, input_bundle=input_bundle) assert e.value._message == f"ethereum.ercs.{erc}" assert e.value._hint == f"try renaming `{erc}` to `I{erc}`" + assert "code.vy:" in str(e.value) def test_interface_body_check(make_input_bundle): diff --git a/vyper/semantics/analysis/imports.py b/vyper/semantics/analysis/imports.py index bcd62feb07..148205f5f8 100644 --- a/vyper/semantics/analysis/imports.py +++ b/vyper/semantics/analysis/imports.py @@ -19,6 +19,7 @@ ImportCycle, ModuleNotFound, StructureException, + tag_exceptions, ) from vyper.semantics.analysis.base import ImportInfo from vyper.utils import safe_relpath, sha256sum @@ -106,10 +107,11 @@ def _resolve_imports_r(self, module_ast: vy_ast.Module): return with self.graph.enter_path(module_ast): for node in module_ast.body: - if isinstance(node, vy_ast.Import): - self._handle_Import(node) - elif isinstance(node, vy_ast.ImportFrom): - self._handle_ImportFrom(node) + with tag_exceptions(node): + if isinstance(node, vy_ast.Import): + self._handle_Import(node) + elif isinstance(node, vy_ast.ImportFrom): + self._handle_ImportFrom(node) self.seen.add(id(module_ast)) def _handle_Import(self, node: vy_ast.Import): From 66272e6637864b8b02f0a941c2006f69bc7b3d3c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 9 Jan 2025 10:59:05 -0500 Subject: [PATCH 59/70] fix[ux]: fix error message for "staticall" typo (#4438) the "staticall" typo is very common, and was raising a hard-to-read error message from the python parser like "invalid syntax. Perhaps you forgot a comma?" this commit adds logic to `ast/parse.py`, so we can add a hint to the exception after we have successfully gone through the python parser and have the ability to add source info and a hint. --- .../exceptions/test_syntax_exception.py | 30 +++++++++++++++++-- vyper/ast/parse.py | 11 ++++++- vyper/exceptions.py | 5 ++-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/tests/functional/syntax/exceptions/test_syntax_exception.py b/tests/functional/syntax/exceptions/test_syntax_exception.py index 80f499ac89..b95e63f598 100644 --- a/tests/functional/syntax/exceptions/test_syntax_exception.py +++ b/tests/functional/syntax/exceptions/test_syntax_exception.py @@ -1,5 +1,6 @@ import pytest +from vyper.compiler import compile_code from vyper.exceptions import SyntaxException fail_list = [ @@ -107,5 +108,30 @@ def foo(): @pytest.mark.parametrize("bad_code", fail_list) -def test_syntax_exception(assert_compile_failed, get_contract, bad_code): - assert_compile_failed(lambda: get_contract(bad_code), SyntaxException) +def test_syntax_exception(bad_code): + with pytest.raises(SyntaxException): + compile_code(bad_code) + + +def test_bad_staticcall_keyword(): + bad_code = """ +from ethereum.ercs import IERC20Detailed + +def foo(): + staticcall ERC20(msg.sender).transfer(msg.sender, staticall IERC20Detailed(msg.sender).decimals()) + """ # noqa + with pytest.raises(SyntaxException) as e: + compile_code(bad_code) + + expected_error = """ +invalid syntax. Perhaps you forgot a comma? (, line 5) + + contract ":5", line 5:54 + 4 def foo(): + ---> 5 staticcall ERC20(msg.sender).transfer(msg.sender, staticall IERC20Detailed(msg.sender).decimals()) + -------------------------------------------------------------^ + 6 + + (hint: did you mean `staticcall`?) + """ # noqa + assert str(e.value) == expected_error.strip() diff --git a/vyper/ast/parse.py b/vyper/ast/parse.py index 423b37721a..1a416acd55 100644 --- a/vyper/ast/parse.py +++ b/vyper/ast/parse.py @@ -86,7 +86,16 @@ def _parse_to_ast_with_settings( # SyntaxError offset is 1-based, not 0-based (see: # https://docs.python.org/3/library/exceptions.html#SyntaxError.offset) offset -= 1 - raise SyntaxException(str(e.msg), vyper_source, e.lineno, offset) from None + new_e = SyntaxException(str(e), vyper_source, e.lineno, offset) + + likely_errors = ("staticall", "staticcal") + tmp = str(new_e) + for s in likely_errors: + if s in tmp: + new_e._hint = "did you mean `staticcall`?" + break + + raise new_e from None # Add dummy function node to ensure local variables are treated as `AnnAssign` # instead of state variables (`VariableDecl`) diff --git a/vyper/exceptions.py b/vyper/exceptions.py index 990dbf7953..04a60b6306 100644 --- a/vyper/exceptions.py +++ b/vyper/exceptions.py @@ -192,15 +192,14 @@ class VyperException(_BaseVyperException): class SyntaxException(VyperException): - """Invalid syntax.""" - def __init__(self, message, source_code, lineno, col_offset): + def __init__(self, message, source_code, lineno, col_offset, hint=None): item = types.SimpleNamespace() # TODO: Create an actual object for this item.lineno = lineno item.col_offset = col_offset item.full_source_code = source_code - super().__init__(message, item) + super().__init__(message, item, hint=hint) class DecimalOverrideException(VyperException): From 9db1546272660bde375409147856ddeada1979b9 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 9 Jan 2025 11:51:23 -0500 Subject: [PATCH 60/70] fix[parser]: fix bad tokenization of hex strings (#4406) this commit fixes parsing of hex strings. there were several issues with the hex string pre-parser, including: - modification of the string locations - incorrectly not exiting the state machine if a non-string token is encountered. this commit fixes the state machine, changes the pre-parser to leave the locations of hex strings unmodified as to minimize the changes to locations in the reformatted code vs source code. to see the effect, print out the reformatted code of the test cases included in this PR before and after this commit. this commit additionally adds several sanity checks to the pre-parser so that the chance of future tokenization bugs is minimized. --- .github/workflows/pull-request.yaml | 2 + tests/functional/codegen/types/test_bytes.py | 15 +++++++ tests/functional/syntax/test_bytes.py | 30 +++++++++++++- vyper/ast/parse.py | 5 +++ vyper/ast/pre_parser.py | 43 +++++++++++--------- 5 files changed, 74 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index a2f4b5a0d1..75a8762d04 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -34,6 +34,7 @@ jobs: # lang: language changes # stdlib: changes to the stdlib # ux: language changes (UX) + # parser: parser changes # tool: integration # ir: (old) IR/codegen changes # codegen: lowering from vyper AST to codegen @@ -46,6 +47,7 @@ jobs: lang stdlib ux + parser tool ir codegen diff --git a/tests/functional/codegen/types/test_bytes.py b/tests/functional/codegen/types/test_bytes.py index 6473be4348..8bd7cb6a2c 100644 --- a/tests/functional/codegen/types/test_bytes.py +++ b/tests/functional/codegen/types/test_bytes.py @@ -1,5 +1,6 @@ import pytest +from vyper.compiler import compile_code from vyper.exceptions import TypeMismatch @@ -281,6 +282,20 @@ def test2(l: Bytes[{m}] = x"{val}") -> bool: assert c.test2(vyper_literal) is True +def test_hex_literal_parser_edge_case(): + # see GH issue 4405 example 2 + code = """ +interface FooBar: + def test(a: Bytes[2], b: String[4]): payable + +@deploy +def __init__(ext: FooBar): + extcall ext.test(x'6161', x'6161') #ext.test(b'\x61\61', '6161') gets called + """ + with pytest.raises(TypeMismatch): + compile_code(code) + + def test_zero_padding_with_private(get_contract): code = """ counter: uint256 diff --git a/tests/functional/syntax/test_bytes.py b/tests/functional/syntax/test_bytes.py index 9df2962f2e..06c3c1f443 100644 --- a/tests/functional/syntax/test_bytes.py +++ b/tests/functional/syntax/test_bytes.py @@ -81,7 +81,17 @@ def test() -> Bytes[1]: """ @external def test() -> Bytes[2]: - a: Bytes[2] = x"abc" + a: Bytes[2] = x"abc" # non-hex nibbles + return a + """, + SyntaxException, + ), + ( + """ +@external +def test() -> Bytes[10]: + # GH issue 4405 example 1 + a: Bytes[10] = x x x x x x"61" # messed up hex prefix return a """, SyntaxException, @@ -107,6 +117,24 @@ def test_bytes_fail(bad_code): compiler.compile_code(bad_code) +@pytest.mark.xfail +def test_hexbytes_offset(): + good_code = """ + event X: + a: Bytes[2] + +@deploy +def __init__(): + # GH issue 4405, example 1 + # + # log changes offset of HexString, and the hex_string_locations tracked + # location is incorrect when visiting ast + log X(a = x"6161") + """ + # move this to valid list once it passes. + assert compiler.compile_code(good_code) is not None + + valid_list = [ """ @external diff --git a/vyper/ast/parse.py b/vyper/ast/parse.py index 1a416acd55..8df295c9eb 100644 --- a/vyper/ast/parse.py +++ b/vyper/ast/parse.py @@ -117,6 +117,10 @@ def _parse_to_ast_with_settings( # postcondition: consumed all the for loop annotations assert len(pre_parser.for_loop_annotations) == 0 + # postcondition: we have used all the hex strings found by the + # pre-parser + assert len(pre_parser.hex_string_locations) == 0 + # Convert to Vyper AST. module = vy_ast.get_node(py_ast) assert isinstance(module, vy_ast.Module) # mypy hint @@ -440,6 +444,7 @@ def visit_Constant(self, node): node.col_offset, ) node.ast_type = "HexBytes" + self._pre_parser.hex_string_locations.remove(key) else: node.ast_type = "Str" elif isinstance(node.value, bytes): diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index dbeb6181f9..5cbddffed8 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -109,37 +109,40 @@ def consume(self, token): class HexStringParser: def __init__(self): self.locations = [] - self._current_x = None + self._tokens = [] self._state = ParserState.NOT_RUNNING def consume(self, token, result): # prepare to check if the next token is a STRING - if token.type == NAME and token.string == "x": - self._state = ParserState.RUNNING - self._current_x = token - return True - if self._state == ParserState.NOT_RUNNING: + if token.type == NAME and token.string == "x": + self._tokens.append(token) + self._state = ParserState.RUNNING + return True + return False - if self._state == ParserState.RUNNING: - current_x = self._current_x - self._current_x = None - self._state = ParserState.NOT_RUNNING + assert self._state == ParserState.RUNNING, "unreachable" - toks = [current_x] + self._state = ParserState.NOT_RUNNING - # drop the leading x token if the next token is a STRING to avoid a python - # parser error - if token.type == STRING: - self.locations.append(current_x.start) - toks = [TokenInfo(STRING, token.string, current_x.start, token.end, token.line)] - result.extend(toks) - return True + if token.type != STRING: + # flush the tokens we have accumulated and move on + result.extend(self._tokens) + self._tokens = [] + return False - result.extend(toks) + # mark hex string in locations for later processing + self.locations.append(token.start) - return False + # discard the `x` token and apply sanity checks - + # we should only be discarding one token. + assert len(self._tokens) == 1 + assert (x_tok := self._tokens[0]).type == NAME and x_tok.string == "x" + self._tokens = [] # discard tokens + + result.append(token) + return True # compound statements that are replaced with `class` From f444c8fa3b02f34181cfa8768bcf572aedc29659 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 11 Jan 2025 15:14:12 -0500 Subject: [PATCH 61/70] chore[ci]: update codecov github action to v5 (#4437) might fix some rate limiting errors we have been seeing recently --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa2155db20..aba60ba391 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,7 +155,7 @@ jobs: tests/ - name: Upload Coverage - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml @@ -215,7 +215,7 @@ jobs: tests/ - name: Upload Coverage - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml From 9b5523e6131335c81714e7e8af63cc49404f5ce7 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 12 Jan 2025 11:19:30 -0500 Subject: [PATCH 62/70] refactor[lang]: remove VyperNode `__hash__()` and `__eq__()` implementations (#4433) it is a performance and correctness footgun for `VyperNode`'s hash and eq implementations to recurse. for instance, two nodes from different source files should never compare equal. several tests rely on the recursive behavior of the eq implementation; a utility function `deepequals()` is added in this PR so that tests can perform the recursive check on AST nodes where needed. nowhere in the compiler itself (`vyper/` directory) is the recursive definition relied on. this commit also slightly refactors the import analyzer so that uses the new hash and eq implementations instead of the previous workaround to avoid recursion (which was to use `id(module_ast)`. --- tests/ast_utils.py | 25 ++++++++++++++++++++++ tests/unit/ast/nodes/test_binary.py | 3 ++- tests/unit/ast/nodes/test_compare_nodes.py | 11 +++++----- tests/unit/ast/test_ast_dict.py | 3 ++- tests/unit/ast/test_parser.py | 5 +++-- vyper/ast/nodes.py | 16 -------------- vyper/semantics/analysis/imports.py | 10 ++++----- 7 files changed, 43 insertions(+), 30 deletions(-) create mode 100644 tests/ast_utils.py diff --git a/tests/ast_utils.py b/tests/ast_utils.py new file mode 100644 index 0000000000..e4be35adb2 --- /dev/null +++ b/tests/ast_utils.py @@ -0,0 +1,25 @@ +from vyper.ast.nodes import VyperNode + + +def deepequals(node: VyperNode, other: VyperNode): + # checks two nodes are recursively equal, ignoring metadata + # like line info. + if not isinstance(other, type(node)): + return False + + if isinstance(node, list): + if len(node) != len(other): + return False + return all(deepequals(a, b) for a, b in zip(node, other)) + + if not isinstance(node, VyperNode): + return node == other + + if getattr(node, "node_id", None) != getattr(other, "node_id", None): + return False + for field_name in (i for i in node.get_fields() if i not in VyperNode.__slots__): + lhs = getattr(node, field_name, None) + rhs = getattr(other, field_name, None) + if not deepequals(lhs, rhs): + return False + return True diff --git a/tests/unit/ast/nodes/test_binary.py b/tests/unit/ast/nodes/test_binary.py index d7662bc4bb..4bebe0abc2 100644 --- a/tests/unit/ast/nodes/test_binary.py +++ b/tests/unit/ast/nodes/test_binary.py @@ -1,5 +1,6 @@ import pytest +from tests.ast_utils import deepequals from vyper import ast as vy_ast from vyper.exceptions import SyntaxException @@ -18,7 +19,7 @@ def x(): """ ) - assert expected == mutated + assert deepequals(expected, mutated) def test_binary_length(): diff --git a/tests/unit/ast/nodes/test_compare_nodes.py b/tests/unit/ast/nodes/test_compare_nodes.py index 164cd3d371..d228e40bd1 100644 --- a/tests/unit/ast/nodes/test_compare_nodes.py +++ b/tests/unit/ast/nodes/test_compare_nodes.py @@ -1,3 +1,4 @@ +from tests.ast_utils import deepequals from vyper import ast as vy_ast @@ -6,21 +7,21 @@ def test_compare_different_node_clases(): left = vyper_ast.body[0].target right = vyper_ast.body[0].value - assert left != right + assert not deepequals(left, right) def test_compare_different_nodes_same_class(): vyper_ast = vy_ast.parse_to_ast("[1, 2]") left, right = vyper_ast.body[0].value.elements - assert left != right + assert not deepequals(left, right) def test_compare_different_nodes_same_value(): vyper_ast = vy_ast.parse_to_ast("[1, 1]") left, right = vyper_ast.body[0].value.elements - assert left != right + assert not deepequals(left, right) def test_compare_similar_node(): @@ -28,11 +29,11 @@ def test_compare_similar_node(): left = vy_ast.Int(value=1) right = vy_ast.Int(value=1) - assert left == right + assert deepequals(left, right) def test_compare_same_node(): vyper_ast = vy_ast.parse_to_ast("42") node = vyper_ast.body[0].value - assert node == node + assert deepequals(node, node) diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index 196b1e24e6..cfad0795bc 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -1,6 +1,7 @@ import copy import json +from tests.ast_utils import deepequals from vyper import compiler from vyper.ast.nodes import NODE_SRC_ATTRIBUTES from vyper.ast.parse import parse_to_ast @@ -138,7 +139,7 @@ def test() -> int128: new_dict = json.loads(out_json) new_ast = dict_to_ast(new_dict) - assert new_ast == original_ast + assert deepequals(new_ast, original_ast) # strip source annotations like lineno, we don't care for inspecting diff --git a/tests/unit/ast/test_parser.py b/tests/unit/ast/test_parser.py index e0bfcbc2ef..96df6cf245 100644 --- a/tests/unit/ast/test_parser.py +++ b/tests/unit/ast/test_parser.py @@ -1,3 +1,4 @@ +from tests.ast_utils import deepequals from vyper.ast.parse import parse_to_ast @@ -12,7 +13,7 @@ def test() -> int128: ast1 = parse_to_ast(code) ast2 = parse_to_ast("\n \n" + code + "\n\n") - assert ast1 == ast2 + assert deepequals(ast1, ast2) def test_ast_unequal(): @@ -32,4 +33,4 @@ def test() -> int128: ast1 = parse_to_ast(code1) ast2 = parse_to_ast(code2) - assert ast1 != ast2 + assert not deepequals(ast1, ast2) diff --git a/vyper/ast/nodes.py b/vyper/ast/nodes.py index ccc80947e4..3c8feec786 100644 --- a/vyper/ast/nodes.py +++ b/vyper/ast/nodes.py @@ -331,26 +331,10 @@ def get_fields(cls) -> set: slot_fields = [x for i in cls.__mro__ for x in getattr(i, "__slots__", [])] return set(i for i in slot_fields if not i.startswith("_")) - def __hash__(self): - values = [getattr(self, i, None) for i in VyperNode._public_slots] - return hash(tuple(values)) - def __deepcopy__(self, memo): # default implementation of deepcopy is a hotspot return pickle.loads(pickle.dumps(self)) - def __eq__(self, other): - # CMC 2024-03-03 I'm not sure it makes much sense to compare AST - # nodes, especially if they come from other modules - if not isinstance(other, type(self)): - return False - if getattr(other, "node_id", None) != getattr(self, "node_id", None): - return False - for field_name in (i for i in self.get_fields() if i not in VyperNode.__slots__): - if getattr(self, field_name, None) != getattr(other, field_name, None): - return False - return True - def __repr__(self): cls = type(self) class_repr = f"{cls.__module__}.{cls.__qualname__}" diff --git a/vyper/semantics/analysis/imports.py b/vyper/semantics/analysis/imports.py index 148205f5f8..7f02bce79d 100644 --- a/vyper/semantics/analysis/imports.py +++ b/vyper/semantics/analysis/imports.py @@ -59,8 +59,7 @@ def push_path(self, module_ast: vy_ast.Module) -> None: def pop_path(self, expected: vy_ast.Module) -> None: popped = self._path.pop() - if expected != popped: - raise CompilerPanic("unreachable") + assert expected is popped, "unreachable" self._imports.pop() @contextlib.contextmanager @@ -78,7 +77,7 @@ def __init__(self, input_bundle: InputBundle, graph: _ImportGraph): self.graph = graph self._ast_of: dict[int, vy_ast.Module] = {} - self.seen: set[int] = set() + self.seen: set[vy_ast.Module] = set() self._integrity_sum = None @@ -103,7 +102,7 @@ def _calculate_integrity_sum_r(self, module_ast: vy_ast.Module): return sha256sum("".join(acc)) def _resolve_imports_r(self, module_ast: vy_ast.Module): - if id(module_ast) in self.seen: + if module_ast in self.seen: return with self.graph.enter_path(module_ast): for node in module_ast.body: @@ -112,7 +111,8 @@ def _resolve_imports_r(self, module_ast: vy_ast.Module): self._handle_Import(node) elif isinstance(node, vy_ast.ImportFrom): self._handle_ImportFrom(node) - self.seen.add(id(module_ast)) + + self.seen.add(module_ast) def _handle_Import(self, node: vy_ast.Import): # import x.y[name] as y[alias] From 43259f8953672ef7a19167c6c048d020d82e05da Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 12 Jan 2025 11:32:46 -0500 Subject: [PATCH 63/70] fix[lang]: fix encoding of string literals (#3091) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this commit fixes bad runtime encoding of unicode strings. `parse_Str` used the utility function `string_to_bytes`, which rejects characters with value larger than 255 and otherwise produces the ascii encoding of the string. the issue is that bytes in the range 128-255 specify different characters in utf-8 than in ascii encodings, resulting in different values at runtime than at compile-time. this can be seen from differing compile-vs-runtime behavior of `keccak256` (this example was provided in GH issue 3088): ```vyper @external @view def compile_hash() -> bytes32: return keccak256("è") @external @view def runtime_hash() -> bytes32: s: String[1] = "è" return keccak256(s) ``` this commit fixes and simplifies `parse_Str` by using python's `str.encode()` builtin, which encodes using utf-8 by default. it also increases strictness of string validation to reject bytes in the range 128-255, since in utf-8 these can encode multibyte characters, which we reject in vyper (see more discussion in GH issue 2338). --- tests/functional/syntax/test_string.py | 45 ++++++++++++++++---------- vyper/ast/nodes.py | 5 ++- vyper/codegen/expr.py | 29 ++++++----------- vyper/utils.py | 13 +------- 4 files changed, 43 insertions(+), 49 deletions(-) diff --git a/tests/functional/syntax/test_string.py b/tests/functional/syntax/test_string.py index 77cb7eaee6..1dc354f773 100644 --- a/tests/functional/syntax/test_string.py +++ b/tests/functional/syntax/test_string.py @@ -1,7 +1,7 @@ import pytest from vyper import compiler -from vyper.exceptions import StructureException +from vyper.exceptions import InvalidLiteral, StructureException valid_list = [ """ @@ -11,25 +11,13 @@ def foo() -> String[10]: """, """ @external -def foo(): - x: String[11] = "¡très bien!" - """, - """ -@external def foo() -> bool: - x: String[15] = "¡très bien!" + x: String[15] = "tres bien!" y: String[15] = "test" return x != y """, """ @external -def foo() -> bool: - x: String[15] = "¡très bien!" - y: String[12] = "test" - return x != y - """, - """ -@external def test() -> String[100]: return "hello world!" """, @@ -46,13 +34,36 @@ def test_string_success(good_code): """ @external def foo(): + # invalid type annotation - should be String[N] a: String = "abc" """, StructureException, - ) + ), + ( + """ +@external +@view +def compile_hash() -> bytes32: + # GH issue #3088 - ord("è") == 232 + return keccak256("è") + """, + InvalidLiteral, + ), + ( + """ +@external +def foo() -> bool: + # ord("¡") == 161 + x: String[15] = "¡très bien!" + y: String[12] = "test" + return x != y + """, + InvalidLiteral, + ), ] @pytest.mark.parametrize("bad_code,exc", invalid_list) -def test_string_fail(assert_compile_failed, get_contract, bad_code, exc): - assert_compile_failed(lambda: get_contract(bad_code), exc) +def test_string_fail(get_contract, bad_code, exc): + with pytest.raises(exc): + compiler.compile_code(bad_code) diff --git a/vyper/ast/nodes.py b/vyper/ast/nodes.py index 3c8feec786..24a0f9ade3 100644 --- a/vyper/ast/nodes.py +++ b/vyper/ast/nodes.py @@ -873,7 +873,10 @@ class Str(Constant): def validate(self): for c in self.value: - if ord(c) >= 256: + # in utf-8, bytes in the 128 and up range deviate from latin1 and + # can be control bytes, allowing multi-byte characters. + # reject them here. + if ord(c) >= 128: raise InvalidLiteral(f"'{c}' is not an allowed string literal character", self) diff --git a/vyper/codegen/expr.py b/vyper/codegen/expr.py index 3a09bbe6c0..d3059e4245 100644 --- a/vyper/codegen/expr.py +++ b/vyper/codegen/expr.py @@ -61,13 +61,7 @@ from vyper.semantics.types.bytestrings import _BytestringT from vyper.semantics.types.function import ContractFunctionT, MemberFunctionT from vyper.semantics.types.shortcuts import BYTES32_T, UINT256_T -from vyper.utils import ( - DECIMAL_DIVISOR, - bytes_to_int, - is_checksum_encoded, - string_to_bytes, - vyper_warn, -) +from vyper.utils import DECIMAL_DIVISOR, bytes_to_int, is_checksum_encoded, vyper_warn ENVIRONMENT_VARIABLES = {"block", "msg", "tx", "chain"} @@ -135,24 +129,21 @@ def parse_Hex(self): # String literals def parse_Str(self): - bytez, bytez_length = string_to_bytes(self.expr.value) - typ = StringT(bytez_length) - return self._make_bytelike(typ, bytez, bytez_length) + bytez = self.expr.value.encode("utf-8") + return self._make_bytelike(StringT, bytez) # Byte literals def parse_Bytes(self): - return self._parse_bytes() + return self._make_bytelike(BytesT, self.expr.value) def parse_HexBytes(self): - return self._parse_bytes() - - def _parse_bytes(self): - bytez = self.expr.value - bytez_length = len(self.expr.value) - typ = BytesT(bytez_length) - return self._make_bytelike(typ, bytez, bytez_length) + # HexBytes already has value as bytes + assert isinstance(self.expr.value, bytes) + return self._make_bytelike(BytesT, self.expr.value) - def _make_bytelike(self, btype, bytez, bytez_length): + def _make_bytelike(self, typeclass, bytez): + bytez_length = len(bytez) + btype = typeclass(bytez_length) placeholder = self.context.new_internal_variable(btype) seq = [] seq.append(["mstore", placeholder, bytez_length]) diff --git a/vyper/utils.py b/vyper/utils.py index db50626713..999e211acb 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -11,7 +11,7 @@ import warnings from typing import Generic, List, TypeVar, Union -from vyper.exceptions import CompilerPanic, DecimalOverrideException, InvalidLiteral, VyperException +from vyper.exceptions import CompilerPanic, DecimalOverrideException, VyperException _T = TypeVar("_T") @@ -310,17 +310,6 @@ def round_towards_zero(d: decimal.Decimal) -> int: return int(d.to_integral_exact(decimal.ROUND_DOWN)) -# Converts string to bytes -def string_to_bytes(str): - bytez = b"" - for c in str: - if ord(c) >= 256: - raise InvalidLiteral(f"Cannot insert special character {c} into byte array") - bytez += bytes([ord(c)]) - bytez_length = len(bytez) - return bytez, bytez_length - - # Converts a provided hex string to an integer def hex_to_int(inp): if inp[:2] == "0x": From 10e91d5a2ba6eaab2f7194fd86cefb7a0ff19964 Mon Sep 17 00:00:00 2001 From: tserg <8017125+tserg@users.noreply.github.com> Date: Mon, 13 Jan 2025 00:34:14 +0800 Subject: [PATCH 64/70] chore[ci]: bump upload-artifact action to v4 (#4445) This commit bumps the upload-artifact action to v4 because v3 will be deprecated on 30 Jan 2025. --- .github/workflows/build.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 515be87c25..5dd98413a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,8 +49,9 @@ jobs: - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: + name: vyper-${{ runner.os }} path: dist/vyper.* windows-build: @@ -81,8 +82,9 @@ jobs: ./make.cmd freeze - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: + name: vyper-${{ runner.os }} path: dist/vyper.* publish-release-assets: @@ -92,14 +94,13 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts/ + merge-multiple: true - name: Upload assets - # fun - artifacts are downloaded into "artifact/". - # TODO: this needs to be tested when upgrading to upload-artifact v4 - working-directory: artifacts/artifact + working-directory: artifacts run: | set -Eeuxo pipefail for BIN_NAME in $(ls) From db8dcc713168b16977b5b07267653c9024f6acea Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 12 Jan 2025 12:01:49 -0500 Subject: [PATCH 65/70] refactor[parser]: remove `ASTTokens` (#4364) this commit removes `asttokens` from the parse machinery, since the method is buggy (see below bugs) and slow. this commit brings down parse time (time spent in ast generation) between 40-70%. the `mark_tokens()` machinery is replaced with a modified version of `python.ast`'s `fix_missing_locations()` function, which recurses through the AST and adds missing line info based on the parent node. it also changes to a more consistent method for updating source offsets that are modified by the `pre_parse` step, which fixes several outstanding bugs with source location reporting. there were some exceptions to the line info fixup working, the issues and corresponding workarounds are described as follows: - some python AST nodes returned by `ast.parse()` are singletons, which we work around by deepcopying the AST before operating on it. - notably, there is an interaction between our AST annotation and `coverage.py` in the case of `USub`. in this commit we paper over the issue by simply always overriding line info for `USub` nodes. in the future, we should refactor `VyperNode` generation by bypassing the python AST annotation step entirely, which is a more proper fix to the problems encountered in this PR. the `asttokens` package is not removed entirely since it still has a limited usage inside of the natspec parser. we could remove it in a future PR; for now it is out-of-scope. referenced bugs: - https://github.com/vyperlang/vyper/issues/2258 - https://github.com/vyperlang/vyper/issues/3059 - https://github.com/vyperlang/vyper/issues/3430 - https://github.com/vyperlang/vyper/issues/4139 --- tests/unit/ast/test_tokenizer.py | 94 ++++++++++++++++++ vyper/ast/natspec.py | 1 + vyper/ast/parse.py | 165 +++++++++++++++++++------------ vyper/ast/pre_parser.py | 61 +++++++----- vyper/utils.py | 4 +- 5 files changed, 236 insertions(+), 89 deletions(-) create mode 100644 tests/unit/ast/test_tokenizer.py diff --git a/tests/unit/ast/test_tokenizer.py b/tests/unit/ast/test_tokenizer.py new file mode 100644 index 0000000000..f6000e0425 --- /dev/null +++ b/tests/unit/ast/test_tokenizer.py @@ -0,0 +1,94 @@ +""" +Tests that the tokenizer / parser are passing correct source location +info to the AST +""" +import pytest + +from vyper.ast.parse import parse_to_ast +from vyper.compiler import compile_code +from vyper.exceptions import UndeclaredDefinition + + +def test_log_token_aligned(): + # GH issue 3430 + code = """ +event A: + b: uint256 + +@external +def f(): + log A(b=d) + """ + with pytest.raises(UndeclaredDefinition) as e: + compile_code(code) + + expected = """ + 'd' has not been declared. + + function "f", line 7:12 + 6 def f(): + ---> 7 log A(b=d) + -------------------^ + 8 + """ # noqa: W291 + assert expected.strip() == str(e.value).strip() + + +def test_log_token_aligned2(): + # GH issue 3059 + code = """ +interface Contract: + def foo(): nonpayable + +event MyEvent: + a: address + +@external +def foo(c: Contract): + log MyEvent(a=c.address) + """ + compile_code(code) + + +def test_log_token_aligned3(): + # https://github.com/vyperlang/vyper/pull/3808#pullrequestreview-1900570163 + code = """ +import ITest + +implements: ITest + +event Foo: + a: address + +@external +def foo(u: uint256): + log Foo(empty(address)) + log i.Foo(empty(address)) + """ + # not semantically valid code, check we can at least parse it + assert parse_to_ast(code) is not None + + +def test_log_token_aligned4(): + # GH issue 4139 + code = """ +b: public(uint256) + +event Transfer: + random: indexed(uint256) + shi: uint256 + +@external +def transfer(): + log Transfer(T(self).b(), 10) + return + """ + # not semantically valid code, check we can at least parse it + assert parse_to_ast(code) is not None + + +def test_long_string_non_coding_token(): + # GH issue 2258 + code = '\r[[]]\ndef _(e:[],l:[]):\n """"""""""""""""""""""""""""""""""""""""""""""""""""""\n f.n()' # noqa: E501 + # not valid code, but should at least parse + assert parse_to_ast(code) is not None diff --git a/vyper/ast/natspec.py b/vyper/ast/natspec.py index f65a361338..f5487e8a91 100644 --- a/vyper/ast/natspec.py +++ b/vyper/ast/natspec.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from typing import Optional, Tuple +# NOTE: this is our only use of asttokens -- consider vendoring in the implementation. from asttokens import LineNumbers from vyper.ast import nodes as vy_ast diff --git a/vyper/ast/parse.py b/vyper/ast/parse.py index 8df295c9eb..a7cd0464ed 100644 --- a/vyper/ast/parse.py +++ b/vyper/ast/parse.py @@ -1,10 +1,10 @@ import ast as python_ast +import pickle import tokenize from decimal import Decimal +from functools import cached_property from typing import Any, Dict, List, Optional, Union -import asttokens - from vyper.ast import nodes as vy_ast from vyper.ast.pre_parser import PreParser from vyper.compiler.settings import Settings @@ -80,12 +80,16 @@ def _parse_to_ast_with_settings( try: py_ast = python_ast.parse(pre_parser.reformatted_code) except SyntaxError as e: - # TODO: Ensure 1-to-1 match of source_code:reformatted_code SyntaxErrors offset = e.offset if offset is not None: # SyntaxError offset is 1-based, not 0-based (see: # https://docs.python.org/3/library/exceptions.html#SyntaxError.offset) offset -= 1 + + # adjust the column of the error if it was modified by the pre-parser + if e.lineno is not None: # help mypy + offset += pre_parser.adjustments.get((e.lineno, offset), 0) + new_e = SyntaxException(str(e), vyper_source, e.lineno, offset) likely_errors = ("staticall", "staticcal") @@ -97,6 +101,11 @@ def _parse_to_ast_with_settings( raise new_e from None + # some python AST node instances are singletons and are reused between + # parse() invocations. copy the python AST so that we are using fresh + # objects. + py_ast = _deepcopy_ast(py_ast) + # Add dummy function node to ensure local variables are treated as `AnnAssign` # instead of state variables (`VariableDecl`) if add_fn_node: @@ -129,6 +138,9 @@ def _parse_to_ast_with_settings( return pre_parser.settings, module +LINE_INFO_FIELDS = ("lineno", "col_offset", "end_lineno", "end_col_offset") + + def ast_to_dict(ast_struct: Union[vy_ast.VyperNode, List]) -> Union[Dict, List]: """ Converts a Vyper AST node, or list of nodes, into a dictionary suitable for @@ -155,7 +167,7 @@ def dict_to_ast(ast_struct: Union[Dict, List]) -> Union[vy_ast.VyperNode, List]: def annotate_python_ast( - parsed_ast: python_ast.AST, + parsed_ast: python_ast.Module, vyper_source: str, pre_parser: PreParser, source_id: int = 0, @@ -178,22 +190,19 @@ def annotate_python_ast( ------- The annotated and optimized AST. """ - tokens = asttokens.ASTTokens(vyper_source) - assert isinstance(parsed_ast, python_ast.Module) # help mypy - tokens.mark_tokens(parsed_ast) visitor = AnnotatingVisitor( - vyper_source, - pre_parser, - tokens, - source_id, - module_path=module_path, - resolved_path=resolved_path, + vyper_source, pre_parser, source_id, module_path=module_path, resolved_path=resolved_path ) - visitor.visit(parsed_ast) + visitor.start(parsed_ast) return parsed_ast +def _deepcopy_ast(ast_node: python_ast.AST): + # pickle roundtrip is faster than copy.deepcopy() here. + return pickle.loads(pickle.dumps(ast_node)) + + class AnnotatingVisitor(python_ast.NodeTransformer): _source_code: str _pre_parser: PreParser @@ -202,12 +211,10 @@ def __init__( self, source_code: str, pre_parser: PreParser, - tokens: asttokens.ASTTokens, source_id: int, module_path: Optional[str] = None, resolved_path: Optional[str] = None, ): - self._tokens = tokens self._source_id = source_id self._module_path = module_path self._resolved_path = resolved_path @@ -216,6 +223,58 @@ def __init__( self.counter: int = 0 + @cached_property + def source_lines(self): + return self._source_code.splitlines(keepends=True) + + @cached_property + def line_offsets(self): + ofst = 0 + # ensure line_offsets has at least 1 entry for 0-line source + ret = {1: ofst} + for lineno, line in enumerate(self.source_lines): + ret[lineno + 1] = ofst + ofst += len(line) + return ret + + def start(self, node: python_ast.Module): + self._fix_missing_locations(node) + self.visit(node) + + def _fix_missing_locations(self, ast_node: python_ast.Module): + """ + adapted from cpython Lib/ast.py. adds line/col info to ast, + but unlike Lib/ast.py, adjusts *all* ast nodes, not just the + one that python defines to have line/col info. + https://github.com/python/cpython/blob/62729d79206014886f5d/Lib/ast.py#L228 + """ + assert isinstance(ast_node, python_ast.Module) + ast_node.lineno = 1 + ast_node.col_offset = 0 + ast_node.end_lineno = max(1, len(self.source_lines)) + + if len(self.source_lines) > 0: + ast_node.end_col_offset = len(self.source_lines[-1]) + else: + ast_node.end_col_offset = 0 + + def _fix(node, parent=None): + for field in LINE_INFO_FIELDS: + if parent is not None: + val = getattr(node, field, None) + # special case for USub - heisenbug when coverage is + # enabled in the test suite. + if val is None or isinstance(node, python_ast.USub): + val = getattr(parent, field) + setattr(node, field, val) + else: + assert hasattr(node, field), node + + for child in python_ast.iter_child_nodes(node): + _fix(child, node) + + _fix(ast_node) + def generic_visit(self, node): """ Annotate a node with information that simplifies Vyper node generation. @@ -223,38 +282,28 @@ def generic_visit(self, node): # Decorate every node with the original source code to allow pretty-printing errors node.full_source_code = self._source_code node.node_id = self.counter - node.ast_type = node.__class__.__name__ self.counter += 1 + node.ast_type = node.__class__.__name__ - # Decorate every node with source end offsets - start = (None, None) - if hasattr(node, "first_token"): - start = node.first_token.start - end = (None, None) - if hasattr(node, "last_token"): - end = node.last_token.end - if node.last_token.type == 4: - # token type 4 is a `\n`, some nodes include a trailing newline - # here we ignore it when building the node offsets - end = (end[0], end[1] - 1) - - node.lineno = start[0] - node.col_offset = start[1] - node.end_lineno = end[0] - node.end_col_offset = end[1] - - # TODO: adjust end_lineno and end_col_offset when this node is in - # modification_offsets - - if hasattr(node, "last_token"): - start_pos = node.first_token.startpos - end_pos = node.last_token.endpos - - if node.last_token.type == 4: - # ignore trailing newline once more - end_pos -= 1 - node.src = f"{start_pos}:{end_pos-start_pos}:{self._source_id}" - node.node_source_code = self._source_code[start_pos:end_pos] + adjustments = self._pre_parser.adjustments + + # Load and Store behave differently inside of fix_missing_locations; + # we don't use them in the vyper AST so just skip adjusting the line + # info. + if isinstance(node, (python_ast.Load, python_ast.Store)): + return super().generic_visit(node) + + adj = adjustments.get((node.lineno, node.col_offset), 0) + node.col_offset += adj + + adj = adjustments.get((node.end_lineno, node.end_col_offset), 0) + node.end_col_offset += adj + + start_pos = self.line_offsets[node.lineno] + node.col_offset + end_pos = self.line_offsets[node.end_lineno] + node.end_col_offset + + node.src = f"{start_pos}:{end_pos-start_pos}:{self._source_id}" + node.node_source_code = self._source_code[start_pos:end_pos] return super().generic_visit(node) @@ -288,12 +337,6 @@ def visit_Module(self, node): return self._visit_docstring(node) def visit_FunctionDef(self, node): - if node.decorator_list: - # start the source highlight at `def` to improve annotation readability - decorator_token = node.decorator_list[-1].last_token - def_token = self._tokens.find_token(decorator_token, tokenize.NAME, tok_str="def") - node.first_token = def_token - return self._visit_docstring(node) def visit_ClassDef(self, node): @@ -306,7 +349,7 @@ def visit_ClassDef(self, node): """ self.generic_visit(node) - node.ast_type = self._pre_parser.modification_offsets[(node.lineno, node.col_offset)] + node.ast_type = self._pre_parser.keyword_translations[(node.lineno, node.col_offset)] return node def visit_For(self, node): @@ -349,16 +392,13 @@ def visit_For(self, node): try: fake_node = python_ast.parse(annotation_str).body[0] + # do we need to fix location info here? + fake_node = _deepcopy_ast(fake_node) except SyntaxError as e: raise SyntaxException( "invalid type annotation", self._source_code, node.lineno, node.col_offset ) from e - # fill in with asttokens info. note we can use `self._tokens` because - # it is indented to exactly the same position where it appeared - # in the original source! - self._tokens.mark_tokens(fake_node) - # replace the dummy target name with the real target name. fake_node.target = node.target # replace the For node target with the new ann_assign @@ -383,14 +423,14 @@ def visit_Expr(self, node): # CMC 2024-03-03 consider unremoving this from the enclosing Expr node = node.value key = (node.lineno, node.col_offset) - node.ast_type = self._pre_parser.modification_offsets[key] + node.ast_type = self._pre_parser.keyword_translations[key] return node def visit_Await(self, node): - start_pos = node.lineno, node.col_offset # grab these before generic_visit modifies them + start_pos = node.lineno, node.col_offset self.generic_visit(node) - node.ast_type = self._pre_parser.modification_offsets[start_pos] + node.ast_type = self._pre_parser.keyword_translations[start_pos] return node def visit_Call(self, node): @@ -410,6 +450,9 @@ def visit_Call(self, node): assert len(dict_.keys) == len(dict_.values) for key, value in zip(dict_.keys, dict_.values): replacement_kw_node = python_ast.keyword(key.id, value) + # set locations + for attr in LINE_INFO_FIELDS: + setattr(replacement_kw_node, attr, getattr(key, attr)) kw_list.append(replacement_kw_node) node.args = [] diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index 5cbddffed8..8e221fb7e6 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -164,8 +164,14 @@ def consume(self, token, result): class PreParser: # Compilation settings based on the directives in the source code settings: Settings - # A mapping of class names to their original class types. - modification_offsets: dict[tuple[int, int], str] + + # A mapping of offsets to new class names + keyword_translations: dict[tuple[int, int], str] + + # Map from offsets in the original vyper source code to offsets + # in the new ("reformatted", i.e. python-compatible) source code + adjustments: dict[tuple[int, int], int] + # A mapping of line/column offsets of `For` nodes to the annotation of the for loop target for_loop_annotations: dict[tuple[int, int], list[TokenInfo]] # A list of line/column offsets of hex string literals @@ -199,8 +205,9 @@ def parse(self, code: str): raise SyntaxException(e.args[0], code, e.args[1][0], e.args[1][1]) from e def _parse(self, code: str): + adjustments: dict = {} result: list[TokenInfo] = [] - modification_offsets: dict[tuple[int, int], str] = {} + keyword_translations: dict[tuple[int, int], str] = {} settings = Settings() for_parser = ForParser(code) hex_string_parser = HexStringParser() @@ -219,6 +226,12 @@ def _parse(self, code: str): end = token.end line = token.line + # handle adjustments + lineno, col = token.start + adj = _col_adjustments[lineno] + newstart = lineno, col - adj + adjustments[lineno, col - adj] = adj + if typ == COMMENT: contents = string[1:].strip() if contents.startswith("@version"): @@ -275,37 +288,32 @@ def _parse(self, code: str): ) if typ == NAME: + # see if it's a keyword we need to replace + new_keyword = None if string in VYPER_CLASS_TYPES and start[1] == 0: - toks = [TokenInfo(NAME, "class", start, end, line)] - modification_offsets[start] = VYPER_CLASS_TYPES[string] + new_keyword = "class" + vyper_type = VYPER_CLASS_TYPES[string] elif string in CUSTOM_STATEMENT_TYPES: new_keyword = "yield" - adjustment = len(new_keyword) - len(string) - # adjustments for following staticcall/extcall modification_offsets - _col_adjustments[start[0]] += adjustment - toks = [TokenInfo(NAME, new_keyword, start, end, line)] - modification_offsets[start] = CUSTOM_STATEMENT_TYPES[string] + vyper_type = CUSTOM_STATEMENT_TYPES[string] elif string in CUSTOM_EXPRESSION_TYPES: - # a bit cursed technique to get untokenize to put - # the new tokens in the right place so that modification_offsets - # will work correctly. - # (recommend comparing the result of parse with the - # source code side by side to visualize the whitespace) new_keyword = "await" vyper_type = CUSTOM_EXPRESSION_TYPES[string] - lineno, col_offset = start - - # fixup for when `extcall/staticcall` follows `log` - adjustment = _col_adjustments[lineno] - new_start = (lineno, col_offset + adjustment) - modification_offsets[new_start] = vyper_type + if new_keyword is not None: + keyword_translations[newstart] = vyper_type - # tells untokenize to add whitespace, preserving locations - diff = len(new_keyword) - len(string) - new_end = end[0], end[1] + diff + adjustment = len(string) - len(new_keyword) + # adjustments for following tokens + lineno, col = start + _col_adjustments[lineno] += adjustment - toks = [TokenInfo(NAME, new_keyword, start, new_end, line)] + # a bit cursed technique to get untokenize to put + # the new tokens in the right place so that + # `keyword_translations` will work correctly. + # (recommend comparing the result of parse with the + # source code side by side to visualize the whitespace) + toks = [TokenInfo(NAME, new_keyword, start, end, line)] if (typ, string) == (OP, ";"): raise SyntaxException("Semi-colon statements not allowed", code, start[0], start[1]) @@ -317,8 +325,9 @@ def _parse(self, code: str): for k, v in for_parser.annotations.items(): for_loop_annotations[k] = v.copy() + self.adjustments = adjustments self.settings = settings - self.modification_offsets = modification_offsets + self.keyword_translations = keyword_translations self.for_loop_annotations = for_loop_annotations self.hex_string_locations = hex_string_parser.locations self.reformatted_code = untokenize(result).decode("utf-8") diff --git a/vyper/utils.py b/vyper/utils.py index 999e211acb..5bebca7776 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -518,7 +518,7 @@ def timeit(msg): # pragma: nocover yield end_time = time.perf_counter() total_time = end_time - start_time - print(f"{msg}: Took {total_time:.4f} seconds", file=sys.stderr) + print(f"{msg}: Took {total_time:.6f} seconds", file=sys.stderr) _CUMTIMES = None @@ -527,7 +527,7 @@ def timeit(msg): # pragma: nocover def _dump_cumtime(): # pragma: nocover global _CUMTIMES for msg, total_time in _CUMTIMES.items(): - print(f"{msg}: Cumulative time {total_time:.4f} seconds", file=sys.stderr) + print(f"{msg}: Cumulative time {total_time:.3f} seconds", file=sys.stderr) @contextlib.contextmanager From c208b954564e8fffdd4c86cc3c497e0c3df1aeec Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 14 Jan 2025 12:09:31 -0500 Subject: [PATCH 66/70] chore[docs]: update readme about testing (#4448) - add comments to `quicktest.sh` explaining usage. - remove `tests_require` from `setup.py` as that has been deprecated - remove `pytest-runner` from `setup.py` - update readme to reference quicktest.sh --- README.md | 2 +- quicktest.sh | 11 ++++++++++- setup.py | 3 +-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 84c2948ceb..827d40d549 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ be a bit behind the latest version found in the master branch of this repository ```bash make dev-init -python setup.py test +./quicktest.sh -m "not fuzzing" ``` ## Developing (working on the compiler) diff --git a/quicktest.sh b/quicktest.sh index cd3aad1f15..af928f5d7c 100755 --- a/quicktest.sh +++ b/quicktest.sh @@ -2,8 +2,17 @@ # examples: # ./quicktest.sh +# ./quicktest.sh -m "not fuzzing" +# ./quicktest.sh -m "not fuzzing" -n (this is the most useful) +# ./quicktest.sh -m "not fuzzing" -n0 # ./quicktest.sh tests/.../mytest.py # run pytest but bail out on first error -# useful for dev workflow +# useful for dev workflow. + pytest -q -s --instafail -x --disable-warnings "$@" + +# useful options include: +# -n0 (uses only one core but faster startup) +# -nauto (uses only one core but faster startup) +# -m "not fuzzing" - skip slow/fuzzing tests diff --git a/setup.py b/setup.py index 5b1ae1b81a..e6d4c5763d 100644 --- a/setup.py +++ b/setup.py @@ -98,8 +98,7 @@ def _global_version(version): "importlib-metadata", "wheel", ], - setup_requires=["pytest-runner", "setuptools_scm>=7.1.0,<8.0.0"], - tests_require=extras_require["test"], + setup_requires=["setuptools_scm>=7.1.0,<8.0.0"], extras_require=extras_require, entry_points={ "console_scripts": [ From d7f50dfe6a361557968a4a01278d8ea0d96e227e Mon Sep 17 00:00:00 2001 From: tserg <8017125+tserg@users.noreply.github.com> Date: Tue, 21 Jan 2025 00:14:26 +0800 Subject: [PATCH 67/70] feat[ci]: use `coverage combine` to reduce codecov uploads (#4452) This commit updates the CI to combine code coverage to a single file and perform just a single upload after all tests finish (rather than each test run uploading its own coverage report). This should reduce failures / rate limiting on the codecov app, and also prevent the codecov app from producing an inaccurate coverage report before all the tests finish. references: - https://coverage.readthedocs.io/en/7.6.10/cmd.html#cmd-combine - https://coverage.readthedocs.io/en/7.6.10/cmd.html#re-mapping-paths - https://coverage.readthedocs.io/en/7.6.10/config.html#config-run-relative-files --------- Co-authored-by: Charles Cooper --- .github/workflows/test.yml | 61 ++++++++++++++++++++++++++++++-------- setup.cfg | 12 ++++++++ 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aba60ba391..c9705d5e87 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -149,16 +149,17 @@ jobs: --evm-backend ${{ matrix.evm-backend || 'revm' }} ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }} ${{ matrix.experimental-codegen && '--experimental-codegen' || '' }} - --cov-branch - --cov-report xml:coverage.xml + --cov-config=setup.cfg --cov=vyper tests/ - - name: Upload Coverage - uses: codecov/codecov-action@v5 + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.xml + name: coverage-files-${{ github.job }}-${{ strategy.job-index }} + include-hidden-files: true + path: .coverage + if-no-files-found: error core-tests-success: if: always() @@ -209,16 +210,17 @@ jobs: --splits 120 \ --group ${{ matrix.group }} \ --splitting-algorithm least_duration \ - --cov-branch \ - --cov-report xml:coverage.xml \ + --cov-config=setup.cfg \ --cov=vyper \ tests/ - - name: Upload Coverage - uses: codecov/codecov-action@v5 + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.xml + name: coverage-files-${{ github.job }}-${{ strategy.job-index }} + include-hidden-files: true + path: .coverage + if-no-files-found: error slow-tests-success: if: always() @@ -231,3 +233,38 @@ jobs: - name: Check slow tests all succeeded if: ${{ needs.fuzzing.result != 'success' }} run: exit 1 + + consolidate-coverage: + # Consolidate code coverage using `coverage combine` and upload + # to the codecov app + runs-on: ubuntu-latest + needs: [tests, fuzzing] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Install coverage + run: pip install coverage + + - name: Download coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: coverage-files-* + path: coverage-files + + - name: Combine coverage + run: | + coverage combine coverage-files/**/.coverage + coverage xml + + - name: Upload Coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml diff --git a/setup.cfg b/setup.cfg index 5998961ee8..4cce85034d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,3 +33,15 @@ markers = fuzzing: Run Hypothesis fuzz test suite (deselect with '-m "not fuzzing"') requires_evm_version(version): Mark tests that require at least a specific EVM version and would throw `EvmVersionException` otherwise venom_xfail: mark a test case as a regression (expected to fail) under the venom pipeline + + +[coverage:run] +branch = True +source = vyper + +# this is not available on the CI step that performs `coverage combine` +omit = vyper/version.py + +# allow `coverage combine` to combine reports from heterogeneous OSes. +# (mainly important for consolidating coverage reports in the CI). +relative_files = True From 7136eab0a254aa2ff7ddca41cc05f2ee1fa99caf Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 20 Jan 2025 11:51:21 -0500 Subject: [PATCH 68/70] fix[codegen]: fix assertions for certain precompiles (#4451) this commit fixes a flaw in code generation for certain precompiles. specifically, some calls to the ecrecover (0x01) and identity (0x04) precompiles were not checked for success. in 93a957947af1088addc, the assert for memory copying calls to the identity precompile was optimized out; the reasoning being that if the identity precompile fails due to OOG, the contract would also likely fail with OOG. however, due to the 63/64ths rule, there are cases where just enough gas was supplied to the current call context so that the subcall to the precompile could fail with OOG, but the contract has enough gas to continue execution after it shouldn't (which is undefined behavior) and then successfully return out of the call context. (note that even prior to 93a957947af1088addc, some calls to the identity precompile did not check the success flag. cf. commit cf03d27be6a74c0c33de. the call to ecrecover was unchecked since inception - db44cde626919ed8bebf). note also that since cancun, memory copies are implemented using the `mcopy` instruction, so the bug as it pertains to the identity precompile only affects pre-cancun compilation targets. this commit fixes the flaw by converting the relevant unchecked calls to checked calls. it also adds tests that trigger the behavior by running the call, and then performing the exact same call again but providing `gas_used` back to the contract, which is the minimum amount of gas for the call to the contract to finish execution. the specific amount of gas left at the point of the subcall is small enough to cause the subcall to fail (and the check around the subcall success to revert, which is what is tested for in the new tests). in these tests, it also adds a static check that the IR is well-formed (that all relevant calls to precompiles are appropriately checked). references: - https://github.com/vyperlang/vyper/security/advisories/GHSA-vgf2-gvx8-xwc3 --- .../builtins/codegen/test_ecrecover.py | 42 +++++++++- .../codegen/types/test_dynamic_array.py | 60 ++++++++++++++- tests/functional/codegen/types/test_lists.py | 76 ++++++++++++++++++- tests/functional/codegen/types/test_string.py | 58 ++++++++++++++ tests/utils.py | 22 ++++++ vyper/builtins/functions.py | 2 +- vyper/codegen/core.py | 2 +- 7 files changed, 257 insertions(+), 5 deletions(-) diff --git a/tests/functional/builtins/codegen/test_ecrecover.py b/tests/functional/builtins/codegen/test_ecrecover.py index 8db51fdd07..47a225068d 100644 --- a/tests/functional/builtins/codegen/test_ecrecover.py +++ b/tests/functional/builtins/codegen/test_ecrecover.py @@ -1,7 +1,10 @@ +import contextlib + from eth_account import Account from eth_account._utils.signing import to_bytes32 -from tests.utils import ZERO_ADDRESS +from tests.utils import ZERO_ADDRESS, check_precompile_asserts +from vyper.compiler.settings import OptimizationLevel def test_ecrecover_test(get_contract): @@ -86,3 +89,40 @@ def test_ecrecover() -> bool: """ c = get_contract(code) assert c.test_ecrecover() is True + + +def test_ecrecover_oog_handling(env, get_contract, tx_failed, optimize, experimental_codegen): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +@view +def do_ecrecover(hash: bytes32, v: uint256, r:uint256, s:uint256) -> address: + return ecrecover(hash, v, r, s) + """ + check_precompile_asserts(code) + + c = get_contract(code) + + h = b"\x35" * 32 + local_account = Account.from_key(b"\x46" * 32) + sig = local_account.signHash(h) + v, r, s = sig.v, sig.r, sig.s + + assert c.do_ecrecover(h, v, r, s) == local_account.address + + gas_used = env.last_result.gas_used + + if optimize == OptimizationLevel.NONE and not experimental_codegen: + # if optimizations are off, enough gas is used by the contract + # that the gas provided to ecrecover (63/64ths rule) is enough + # for it to succeed + ctx = contextlib.nullcontext + else: + # in other cases, the gas forwarded is small enough for ecrecover + # to fail with oog, which we handle by reverting. + ctx = tx_failed + + with ctx(): + # provide enough spare gas for the top-level call to not oog but + # not enough for ecrecover to succeed + c.do_ecrecover(h, v, r, s, gas=gas_used) diff --git a/tests/functional/codegen/types/test_dynamic_array.py b/tests/functional/codegen/types/test_dynamic_array.py index 2f647ac38c..3289b6a9dd 100644 --- a/tests/functional/codegen/types/test_dynamic_array.py +++ b/tests/functional/codegen/types/test_dynamic_array.py @@ -1,10 +1,12 @@ +import contextlib import itertools from typing import Any, Callable import pytest -from tests.utils import decimal_to_int +from tests.utils import check_precompile_asserts, decimal_to_int from vyper.compiler import compile_code +from vyper.evm.opcodes import version_check from vyper.exceptions import ( ArgumentException, ArrayIndexException, @@ -1901,3 +1903,59 @@ def foo(): c = get_contract(code) with tx_failed(): c.foo() + + +def test_dynarray_copy_oog(env, get_contract, tx_failed): + # GHSA-vgf2-gvx8-xwc3 + code = """ + +@external +def foo(a: DynArray[uint256, 4000]) -> uint256: + b: DynArray[uint256, 4000] = a + return b[0] + """ + check_precompile_asserts(code) + + c = get_contract(code) + dynarray = [2] * 4000 + assert c.foo(dynarray) == 2 + + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(dynarray, gas=gas_used) + + +def test_dynarray_copy_oog2(env, get_contract, tx_failed): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +@view +def foo(x: String[1000000], y: String[1000000]) -> DynArray[String[1000000], 2]: + z: DynArray[String[1000000], 2] = [x, y] + # Some code + return z + """ + check_precompile_asserts(code) + + c = get_contract(code) + calldata0 = "a" * 10 + calldata1 = "b" * 1000000 + assert c.foo(calldata0, calldata1) == [calldata0, calldata1] + + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(calldata0, calldata1, gas=gas_used) diff --git a/tests/functional/codegen/types/test_lists.py b/tests/functional/codegen/types/test_lists.py index 953a9a9f9f..26cd16ed32 100644 --- a/tests/functional/codegen/types/test_lists.py +++ b/tests/functional/codegen/types/test_lists.py @@ -1,8 +1,12 @@ +import contextlib import itertools import pytest -from tests.utils import decimal_to_int +from tests.evm_backends.base_env import EvmError +from tests.utils import check_precompile_asserts, decimal_to_int +from vyper.compiler.settings import OptimizationLevel +from vyper.evm.opcodes import version_check from vyper.exceptions import ArrayIndexException, OverflowException, TypeMismatch @@ -848,3 +852,73 @@ def foo() -> {return_type}: return MY_CONSTANT[0][0] """ assert_compile_failed(lambda: get_contract(code), TypeMismatch) + + +def test_array_copy_oog(env, get_contract, tx_failed, optimize, experimental_codegen, request): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@internal +def bar(x: uint256[3000]) -> uint256[3000]: + a: uint256[3000] = x + return a + +@external +def foo(x: uint256[3000]) -> uint256: + s: uint256[3000] = self.bar(x) + return s[0] + """ + check_precompile_asserts(code) + + if optimize == OptimizationLevel.NONE and not experimental_codegen: + # fails in bytecode generation due to jumpdests too large + with pytest.raises(AssertionError): + get_contract(code) + return + + c = get_contract(code) + array = [2] * 3000 + assert c.foo(array) == array[0] + + # get the minimum gas for the contract complete execution + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(array, gas=gas_used) + + +def test_array_copy_oog2(env, get_contract, tx_failed, optimize, experimental_codegen, request): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +def foo(x: uint256[2500]) -> uint256: + s: uint256[2500] = x + t: uint256[2500] = s + return t[0] + """ + check_precompile_asserts(code) + + if optimize == OptimizationLevel.NONE and not experimental_codegen: + # fails in creating contract due to code too large + with tx_failed(EvmError): + get_contract(code) + return + + c = get_contract(code) + array = [2] * 2500 + assert c.foo(array) == array[0] + + # get the minimum gas for the contract complete execution + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(array, gas=gas_used) diff --git a/tests/functional/codegen/types/test_string.py b/tests/functional/codegen/types/test_string.py index 1c186eeb6e..b4e6919ea7 100644 --- a/tests/functional/codegen/types/test_string.py +++ b/tests/functional/codegen/types/test_string.py @@ -1,5 +1,10 @@ +import contextlib + import pytest +from tests.utils import check_precompile_asserts +from vyper.evm.opcodes import version_check + def test_string_return(get_contract): code = """ @@ -359,3 +364,56 @@ def compare_var_storage_not_equal_false() -> bool: assert c.compare_var_storage_equal_false() is False assert c.compare_var_storage_not_equal_true() is True assert c.compare_var_storage_not_equal_false() is False + + +def test_string_copy_oog(env, get_contract, tx_failed): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +@view +def foo(x: String[1000000]) -> String[1000000]: + return x + """ + check_precompile_asserts(code) + + c = get_contract(code) + calldata = "a" * 1000000 + assert c.foo(calldata) == calldata + + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(calldata, gas=gas_used) + + +def test_string_copy_oog2(env, get_contract, tx_failed): + # GHSA-vgf2-gvx8-xwc3 + code = """ +@external +@view +def foo(x: String[1000000]) -> uint256: + y: String[1000000] = x + return len(y) + """ + check_precompile_asserts(code) + + c = get_contract(code) + calldata = "a" * 1000000 + assert c.foo(calldata) == len(calldata) + + gas_used = env.last_result.gas_used + if version_check(begin="cancun"): + ctx = contextlib.nullcontext + else: + ctx = tx_failed + + with ctx(): + # depends on EVM version. pre-cancun, will revert due to checking + # success flag from identity precompile. + c.foo(calldata, gas=gas_used) diff --git a/tests/utils.py b/tests/utils.py index 8548c4f47a..b9dc443c0d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,6 +3,7 @@ import os from vyper import ast as vy_ast +from vyper.compiler.phases import CompilerData from vyper.semantics.analysis.constant_folding import constant_fold from vyper.utils import DECIMAL_EPSILON, round_towards_zero @@ -28,3 +29,24 @@ def parse_and_fold(source_code): def decimal_to_int(*args): s = decimal.Decimal(*args) return round_towards_zero(s / DECIMAL_EPSILON) + + +def check_precompile_asserts(source_code): + # common sanity check for some tests, that calls to precompiles + # are correctly wrapped in an assert. + + compiler_data = CompilerData(source_code) + deploy_ir = compiler_data.ir_nodes + runtime_ir = compiler_data.ir_runtime + + def _check(ir_node, parent=None): + if ir_node.value == "staticcall": + precompile_addr = ir_node.args[1] + if isinstance(precompile_addr.value, int) and precompile_addr.value < 10: + assert parent is not None and parent.value == "assert" + for arg in ir_node.args: + _check(arg, ir_node) + + _check(deploy_ir) + # technically runtime_ir is contained in deploy_ir, but check it anyways. + _check(runtime_ir) diff --git a/vyper/builtins/functions.py b/vyper/builtins/functions.py index 62539872bc..55d5443a8f 100644 --- a/vyper/builtins/functions.py +++ b/vyper/builtins/functions.py @@ -781,7 +781,7 @@ def build_IR(self, expr, args, kwargs, context): ["mstore", add_ofst(input_buf, 32), args[1]], ["mstore", add_ofst(input_buf, 64), args[2]], ["mstore", add_ofst(input_buf, 96), args[3]], - ["staticcall", "gas", 1, input_buf, 128, output_buf, 32], + ["assert", ["staticcall", "gas", 1, input_buf, 128, output_buf, 32]], ["mload", output_buf], ], typ=AddressT(), diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 0ad7fa79c6..aaf6f35047 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -326,7 +326,7 @@ def copy_bytes(dst, src, length, length_bound): copy_op = ["mcopy", dst, src, length] gas_bound = _mcopy_gas_bound(length_bound) else: - copy_op = ["staticcall", "gas", 4, src, length, dst, length] + copy_op = ["assert", ["staticcall", "gas", 4, src, length, dst, length]] gas_bound = _identity_gas_bound(length_bound) elif src.location == CALLDATA: copy_op = ["calldatacopy", dst, src, length] From 9a2cb2ed95d0c970ca4c2012593f8851787e547e Mon Sep 17 00:00:00 2001 From: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:04:06 +0000 Subject: [PATCH 69/70] feat[venom]: add binop optimizations (#4281) this commit adds additional binop optimizations to venom. these are the analog of `optimize_binop` in the legacy optimizer and take us one step closer to being able to bypass the legacy optimizer in the venom pipeline. the rules have been added to the algebraic optimizations pass. adding them to the SCCP pass was considered, but to keep the rules all in one place and to keep each pass more self-contained, they were added to the algebraic optimizations pass, with additional SCCP passes performed for branch/cfg reduction. some additional machinery has been added, including `lit_eq` which compares two IRLiterals in modular arithmetic, and InstructionUpdator, which updates instructions while also updating the DFG in-place. this should be useful to use for other passes in the future (although out of scope for this PR). additional passes have been added since they were found to improve codesize. in particular, running AlgebraicOptimization multiple times, and running SCCP after LoadElimination, seem to have some additional benefits. --------- Co-authored-by: Charles Cooper --- .../functional/builtins/codegen/test_slice.py | 19 +- .../codegen/features/test_clampers.py | 2 - .../codegen/types/test_dynamic_array.py | 14 +- .../syntax/test_unbalanced_return.py | 2 +- .../compiler/venom/test_algebraic_binopt.py | 584 ++++++++++++++++++ tests/venom_utils.py | 7 +- vyper/cli/vyper_compile.py | 19 +- vyper/utils.py | 27 +- vyper/venom/__init__.py | 12 +- vyper/venom/analysis/dfg.py | 3 + vyper/venom/basicblock.py | 16 +- vyper/venom/passes/algebraic_optimization.py | 402 +++++++++++- vyper/venom/passes/remove_unused_variables.py | 4 +- vyper/venom/passes/sccp/eval.py | 25 +- vyper/venom/passes/sccp/sccp.py | 24 +- vyper/venom/venom_to_assembly.py | 4 +- 16 files changed, 1084 insertions(+), 80 deletions(-) create mode 100644 tests/unit/compiler/venom/test_algebraic_binopt.py diff --git a/tests/functional/builtins/codegen/test_slice.py b/tests/functional/builtins/codegen/test_slice.py index d5d1efca0f..3f2ce44e1a 100644 --- a/tests/functional/builtins/codegen/test_slice.py +++ b/tests/functional/builtins/codegen/test_slice.py @@ -5,7 +5,12 @@ from vyper.compiler import compile_code from vyper.compiler.settings import OptimizationLevel, Settings from vyper.evm.opcodes import version_check -from vyper.exceptions import ArgumentException, CompilerPanic, TypeMismatch +from vyper.exceptions import ( + ArgumentException, + CompilerPanic, + StaticAssertionException, + TypeMismatch, +) _fun_bytes32_bounds = [(0, 32), (3, 29), (27, 5), (0, 5), (5, 3), (30, 2)] @@ -533,9 +538,15 @@ def do_slice(): @pytest.mark.parametrize("bad_code", oob_fail_list) def test_slice_buffer_oob_reverts(bad_code, get_contract, tx_failed): - c = get_contract(bad_code) - with tx_failed(): - c.do_slice() + try: + c = get_contract(bad_code) + with tx_failed(): + c.do_slice() + except StaticAssertionException: + # it should be ok if we + # catch the assert in compile time + # since it supposed to be revert + pass # tests all 3 adhoc locations: `msg.data`, `self.code`, `
.code` diff --git a/tests/functional/codegen/features/test_clampers.py b/tests/functional/codegen/features/test_clampers.py index b82a771962..2b015a1cce 100644 --- a/tests/functional/codegen/features/test_clampers.py +++ b/tests/functional/codegen/features/test_clampers.py @@ -5,7 +5,6 @@ from eth_utils import keccak from tests.utils import ZERO_ADDRESS, decimal_to_int -from vyper.exceptions import StackTooDeep from vyper.utils import int_bounds @@ -502,7 +501,6 @@ def foo(b: DynArray[int128, 10]) -> DynArray[int128, 10]: @pytest.mark.parametrize("value", [0, 1, -1, 2**127 - 1, -(2**127)]) -@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression") def test_multidimension_dynarray_clamper_passing(get_contract, value): code = """ @external diff --git a/tests/functional/codegen/types/test_dynamic_array.py b/tests/functional/codegen/types/test_dynamic_array.py index 3289b6a9dd..e35bec9dbc 100644 --- a/tests/functional/codegen/types/test_dynamic_array.py +++ b/tests/functional/codegen/types/test_dynamic_array.py @@ -14,6 +14,7 @@ ImmutableViolation, OverflowException, StateAccessViolation, + StaticAssertionException, TypeMismatch, ) @@ -1863,9 +1864,16 @@ def should_revert() -> DynArray[String[65], 2]: @pytest.mark.parametrize("code", dynarray_length_no_clobber_cases) def test_dynarray_length_no_clobber(get_contract, tx_failed, code): # check that length is not clobbered before dynarray data copy happens - c = get_contract(code) - with tx_failed(): - c.should_revert() + try: + c = get_contract(code) + with tx_failed(): + c.should_revert() + except StaticAssertionException: + # this test should create + # assert error so if it is + # detected in compile time + # we can continue + pass def test_dynarray_make_setter_overlap(get_contract): diff --git a/tests/functional/syntax/test_unbalanced_return.py b/tests/functional/syntax/test_unbalanced_return.py index 04835bb0f0..a1faa1c6a5 100644 --- a/tests/functional/syntax/test_unbalanced_return.py +++ b/tests/functional/syntax/test_unbalanced_return.py @@ -195,7 +195,7 @@ def test() -> int128: if 1 == 1 : return 1 else: - assert msg.sender != msg.sender + assert msg.sender != self return 0 """, """ diff --git a/tests/unit/compiler/venom/test_algebraic_binopt.py b/tests/unit/compiler/venom/test_algebraic_binopt.py new file mode 100644 index 0000000000..5486787225 --- /dev/null +++ b/tests/unit/compiler/venom/test_algebraic_binopt.py @@ -0,0 +1,584 @@ +import pytest + +from tests.venom_utils import assert_ctx_eq, parse_from_basic_block +from vyper.venom.analysis import IRAnalysesCache +from vyper.venom.passes import AlgebraicOptimizationPass, StoreElimination + +""" +Test abstract binop+unop optimizations in algebraic optimizations pass +""" + + +def _sccp_algebraic_runner(pre, post): + ctx = parse_from_basic_block(pre) + + for fn in ctx.functions.values(): + ac = IRAnalysesCache(fn) + StoreElimination(ac, fn).run_pass() + AlgebraicOptimizationPass(ac, fn).run_pass() + StoreElimination(ac, fn).run_pass() + + assert_ctx_eq(ctx, parse_from_basic_block(post)) + + +def test_sccp_algebraic_opt_sub_xor(): + # x - x -> 0 + # x ^ x -> 0 + pre = """ + _global: + %par = param + %1 = sub %par, %par + %2 = xor %par, %par + return %1, %2 + """ + post = """ + _global: + %par = param + return 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_zero_sub_add_xor(): + # x + 0 == x - 0 == x ^ 0 -> x + # (this cannot be done for 0 - x) + pre = """ + _global: + %par = param + %1 = sub %par, 0 + %2 = xor %par, 0 + %3 = add %par, 0 + %4 = sub 0, %par + %5 = add 0, %par + %6 = xor 0, %par + return %1, %2, %3, %4, %5, %6 + """ + post = """ + _global: + %par = param + %4 = sub 0, %par + return %par, %par, %par, %4, %par, %par + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_sub_xor_max(): + # x ^ 0xFF..FF -> not x + # -1 - x -> ~x + pre = """ + _global: + %par = param + %tmp = -1 + %1 = xor -1, %par + %2 = xor %par, -1 + + %3 = sub -1, %par + + return %1, %2, %3 + """ + post = """ + _global: + %par = param + %1 = not %par + %2 = not %par + %3 = not %par + return %1, %2, %3 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_shift(): + # x << 0 == x >> 0 == x (sar) 0 -> x + # sar is right arithmetic shift + pre = """ + _global: + %par = param + %1 = shl 0, %par + %2 = shr 0, %1 + %3 = sar 0, %2 + return %1, %2, %3 + """ + post = """ + _global: + %par = param + return %par, %par, %par + """ + + _sccp_algebraic_runner(pre, post) + + +@pytest.mark.parametrize("opcode", ("mul", "and", "div", "sdiv", "mod", "smod")) +def test_mul_by_zero(opcode): + # x * 0 == 0 * x == x % 0 == 0 % x == x // 0 == 0 // x == x & 0 == 0 & x -> 0 + pre = f""" + _global: + %par = param + %1 = {opcode} 0, %par + %2 = {opcode} %par, 0 + return %1, %2 + """ + post = """ + _global: + %par = param + return 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_multi_neutral_elem(): + # x * 1 == 1 * x == x / 1 -> x + # checks for non comutative ops + pre = """ + _global: + %par = param + %1_1 = mul 1, %par + %1_2 = mul %par, 1 + %2_1 = div 1, %par + %2_2 = div %par, 1 + %3_1 = sdiv 1, %par + %3_2 = sdiv %par, 1 + return %1_1, %1_2, %2_1, %2_2, %3_1, %3_2 + """ + post = """ + _global: + %par = param + %2_1 = div 1, %par + %3_1 = sdiv 1, %par + return %par, %par, %2_1, %par, %3_1, %par + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_mod_zero(): + # x % 1 -> 0 + pre = """ + _global: + %par = param + %1 = mod %par, 1 + %2 = smod %par, 1 + return %1, %2 + """ + post = """ + _global: + %par = param + return 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_and_max(): + # x & 0xFF..FF == 0xFF..FF & x -> x + max_uint256 = 2**256 - 1 + pre = f""" + _global: + %par = param + %tmp = {max_uint256} + %1 = and %par, %tmp + %2 = and %tmp, %par + return %1, %2 + """ + post = """ + _global: + %par = param + return %par, %par + """ + + _sccp_algebraic_runner(pre, post) + + +# test powers of 2 from n==2 to n==255. +# (skip 1 since there are specialized rules for n==1) +@pytest.mark.parametrize("n", range(2, 256)) +def test_sccp_algebraic_opt_mul_div_to_shifts(n): + # x * 2**n -> x << n + # x / 2**n -> x >> n + y = 2**n + pre = f""" + _global: + %par = param + %1 = mul %par, {y} + %2 = mod %par, {y} + %3 = div %par, {y} + %4 = mul {y}, %par + %5 = mod {y}, %par ; note: this is blocked! + %6 = div {y}, %par ; blocked! + return %1, %2, %3, %4, %5, %6 + """ + post = f""" + _global: + %par = param + %1 = shl {n}, %par + %2 = and {y - 1}, %par + %3 = shr {n}, %par + %4 = shl {n}, %par + %5 = mod {y}, %par + %6 = div {y}, %par + return %1, %2, %3, %4, %5, %6 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_exp(): + # x ** 0 == 0 ** x -> 1 + # x ** 1 -> x + pre = """ + _global: + %par = param + %1 = exp %par, 0 + %2 = exp 1, %par + %3 = exp 0, %par + %4 = exp %par, 1 + return %1, %2, %3, %4 + """ + post = """ + _global: + %par = param + %3 = iszero %par + return 1, 1, %3, %par + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_compare_self(): + # x < x == x > x -> 0 + pre = """ + _global: + %par = param + %tmp = %par + %1 = gt %tmp, %par + %2 = sgt %tmp, %par + %3 = lt %tmp, %par + %4 = slt %tmp, %par + return %1, %2, %3, %4 + """ + post = """ + _global: + %par = param + return 0, 0, 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_or(): + # x | 0 -> x + # x | 0xFF..FF -> 0xFF..FF + max_uint256 = 2**256 - 1 + pre = f""" + _global: + %par = param + %1 = or %par, 0 + %2 = or %par, {max_uint256} + %3 = or 0, %par + %4 = or {max_uint256}, %par + return %1, %2, %3, %4 + """ + post = f""" + _global: + %par = param + return %par, {max_uint256}, %par, {max_uint256} + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_eq(): + # (x == 0) == (0 == x) -> iszero x + # x == x -> 1 + # x == 0xFFFF..FF -> iszero(not x) + pre = """ + global: + %par = param + %1 = eq %par, 0 + %2 = eq 0, %par + + %3 = eq %par, -1 + %4 = eq -1, %par + + %5 = eq %par, %par + return %1, %2, %3, %4, %5 + """ + post = """ + global: + %par = param + %1 = iszero %par + %2 = iszero %par + %6 = not %par + %3 = iszero %6 + %7 = not %par + %4 = iszero %7 + return %1, %2, %3, %4, 1 + """ + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_boolean_or(): + # x | (non zero) -> 1 if it is only used as boolean + some_nonzero = 123 + pre = f""" + _global: + %par = param + %1 = or %par, {some_nonzero} + %2 = or %par, {some_nonzero} + assert %1 + %3 = or {some_nonzero}, %par + %4 = or {some_nonzero}, %par + assert %3 + return %2, %4 + """ + post = f""" + _global: + %par = param + %2 = or {some_nonzero}, %par + assert 1 + %4 = or {some_nonzero}, %par + assert 1 + return %2, %4 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_sccp_algebraic_opt_boolean_eq(): + # x == y -> iszero (x ^ y) if it is only used as boolean + pre = """ + _global: + %par = param + %par2 = param + %1 = eq %par, %par2 + %2 = eq %par, %par2 + assert %1 + return %2 + + """ + post = """ + _global: + %par = param + %par2 = param + %3 = xor %par, %par2 + %1 = iszero %3 + %2 = eq %par, %par2 + assert %1 + return %2 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_compare_never(): + # unsigned x > 0xFF..FF == x < 0 -> 0 + # signed: x > MAX_SIGNED (0x3F..FF) == x < MIN_SIGNED (0xF0..00) -> 0 + min_int256 = -(2**255) + max_int256 = 2**255 - 1 + min_uint256 = 0 + max_uint256 = 2**256 - 1 + pre = f""" + _global: + %par = param + + %1 = slt %par, {min_int256} + %2 = sgt %par, {max_int256} + %3 = lt %par, {min_uint256} + %4 = gt %par, {max_uint256} + + return %1, %2, %3, %4 + """ + post = """ + _global: + %par = param + return 0, 0, 0, 0 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_comparison_zero(): + # x > 0 => iszero(iszero x) + # 0 < x => iszero(iszero x) + pre = """ + _global: + %par = param + %1 = lt 0, %par + %2 = gt %par, 0 + return %1, %2 + """ + post = """ + _global: + %par = param + %3 = iszero %par + %1 = iszero %3 + %4 = iszero %par + %2 = iszero %4 + return %1, %2 + """ + + _sccp_algebraic_runner(pre, post) + + +def test_comparison_almost_never(): + # unsigned: + # x < 1 => eq x 0 => iszero x + # MAX_UINT - 1 < x => eq x MAX_UINT => iszero(not x) + # signed + # x < MIN_INT + 1 => eq x MIN_INT + # MAX_INT - 1 < x => eq x MAX_INT + + max_uint256 = 2**256 - 1 + max_int256 = 2**255 - 1 + min_int256 = -(2**255) + pre1 = f""" + _global: + %par = param + %1 = lt %par, 1 + %2 = gt %par, {max_uint256 - 1} + %3 = sgt %par, {max_int256 - 1} + %4 = slt %par, {min_int256 + 1} + + return %1, %2, %3, %4 + """ + # commuted versions - produce same output + pre2 = f""" + _global: + %par = param + %1 = gt 1, %par + %2 = lt {max_uint256 - 1}, %par + %3 = slt {max_int256 - 1}, %par + %4 = sgt {min_int256 + 1}, %par + return %1, %2, %3, %4 + """ + post = f""" + _global: + %par = param + ; lt %par, 1 => eq 0, %par => iszero %par + %1 = iszero %par + ; x > MAX_UINT256 - 1 => eq MAX_UINT x => iszero(not x) + %5 = not %par + %2 = iszero %5 + %3 = eq {max_int256}, %par + %4 = eq {min_int256}, %par + return %1, %2, %3, %4 + """ + + _sccp_algebraic_runner(pre1, post) + _sccp_algebraic_runner(pre2, post) + + +def test_comparison_almost_always(): + # unsigned + # x > 0 => iszero(iszero x) + # 0 < x => iszero(iszero x) + # x < MAX_UINT => iszero(eq x MAX_UINT) => iszero(iszero(not x)) + # signed + # x < MAX_INT => iszero(eq MAX_INT) => iszero(iszero(xor MAX_INT x)) + + max_uint256 = 2**256 - 1 + max_int256 = 2**255 - 1 + min_int256 = -(2**255) + + pre1 = f""" + _global: + %par = param + %1 = gt %par, 0 + %2 = lt %par, {max_uint256} + assert %2 + %3 = slt %par, {max_int256} + assert %3 + %4 = sgt %par, {min_int256} + assert %4 + return %1 + """ + # commuted versions + pre2 = f""" + _global: + %par = param + %1 = lt 0, %par + %2 = gt {max_uint256}, %par + assert %2 + %3 = sgt {max_int256}, %par + assert %3 + %4 = slt {min_int256}, %par + assert %4 + return %1 + """ + post = f""" + _global: + %par = param + %5 = iszero %par + %1 = iszero %5 + %9 = not %par ; (eq -1 x) => (iszero (not x)) + %6 = iszero %9 + %2 = iszero %6 + assert %2 + %10 = xor %par, {max_int256} + %7 = iszero %10 + %3 = iszero %7 + assert %3 + %11 = xor %par, {min_int256} + %8 = iszero %11 + %4 = iszero %8 + assert %4 + return %1 + """ + + _sccp_algebraic_runner(pre1, post) + _sccp_algebraic_runner(pre2, post) + + +@pytest.mark.parametrize("val", (100, 2, 3, -100)) +def test_comparison_ge_le(val): + # iszero(x < 100) => 99 < x + # iszero(x > 100) => 101 > x + + up = val + 1 + down = val - 1 + + abs_val = abs(val) + abs_up = abs_val + 1 + abs_down = abs_val - 1 + + pre1 = f""" + _global: + %par = param + %1 = lt %par, {abs_val} + %3 = gt %par, {abs_val} + %2 = iszero %1 + %4 = iszero %3 + %5 = slt %par, {val} + %7 = sgt %par, {val} + %6 = iszero %5 + %8 = iszero %7 + return %2, %4, %6, %8 + """ + pre2 = f""" + _global: + %par = param + %1 = gt {abs_val}, %par + %3 = lt {abs_val}, %par + %2 = iszero %1 + %4 = iszero %3 + %5 = sgt {val}, %par + %7 = slt {val}, %par + %6 = iszero %5 + %8 = iszero %7 + return %2, %4, %6, %8 + """ + post = f""" + _global: + %par = param + %1 = lt {abs_down}, %par + %3 = gt {abs_up}, %par + %5 = slt {down}, %par + %7 = sgt {up}, %par + return %1, %3, %5, %7 + """ + + _sccp_algebraic_runner(pre1, post) + _sccp_algebraic_runner(pre2, post) diff --git a/tests/venom_utils.py b/tests/venom_utils.py index 85298ccb87..6ddc61f615 100644 --- a/tests/venom_utils.py +++ b/tests/venom_utils.py @@ -18,9 +18,12 @@ def instructions_eq(i1: IRInstruction, i2: IRInstruction) -> bool: def assert_bb_eq(bb1: IRBasicBlock, bb2: IRBasicBlock): assert bb1.label.value == bb2.label.value - assert len(bb1.instructions) == len(bb2.instructions) for i1, i2 in zip(bb1.instructions, bb2.instructions): - assert instructions_eq(i1, i2), f"[{i1}] != [{i2}]" + assert instructions_eq(i1, i2), (bb1, f"[{i1}] != [{i2}]") + + # assert after individual instruction checks, makes it easier to debug + # if there is a difference. + assert len(bb1.instructions) == len(bb2.instructions) def assert_fn_eq(fn1: IRFunction, fn2: IRFunction): diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index 390416799a..09f8324dcf 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -5,7 +5,7 @@ import sys import warnings from pathlib import Path -from typing import Any, Iterable, Iterator, Optional, Set, TypeVar +from typing import Any, Optional import vyper import vyper.codegen.ir_node as ir_node @@ -15,8 +15,7 @@ from vyper.compiler.input_bundle import FileInput, FilesystemInputBundle from vyper.compiler.settings import VYPER_TRACEBACK_LIMIT, OptimizationLevel, Settings from vyper.typing import ContractPath, OutputFormats - -T = TypeVar("T") +from vyper.utils import uniq format_options_help = """Format to print, one or more of: bytecode (default) - Deployable bytecode @@ -263,20 +262,6 @@ def _parse_args(argv): _cli_helper(f, output_formats, compiled) -def uniq(seq: Iterable[T]) -> Iterator[T]: - """ - Yield unique items in ``seq`` in order. - """ - seen: Set[T] = set() - - for x in seq: - if x in seen: - continue - - seen.add(x) - yield x - - def exc_handler(contract_path: ContractPath, exception: Exception) -> None: print(f"Error compiling: {contract_path}") raise exception diff --git a/vyper/utils.py b/vyper/utils.py index 5bebca7776..39d3093478 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -9,7 +9,7 @@ import time import traceback import warnings -from typing import Generic, List, TypeVar, Union +from typing import Generic, Iterable, Iterator, List, Set, TypeVar, Union from vyper.exceptions import CompilerPanic, DecimalOverrideException, VyperException @@ -129,6 +129,20 @@ def intersection(cls, *sets): return cls(tmp) +def uniq(seq: Iterable[_T]) -> Iterator[_T]: + """ + Yield unique items in ``seq`` in original sequence order. + """ + seen: Set[_T] = set() + + for x in seq: + if x in seen: + continue + + seen.add(x) + yield x + + class StringEnum(enum.Enum): # Must be first, or else won't work, specifies what .value is @staticmethod @@ -234,6 +248,13 @@ def int_to_fourbytes(n: int) -> bytes: return n.to_bytes(4, byteorder="big") +def wrap256(val: int, signed=False) -> int: + ret = val % (2**256) + if signed: + ret = unsigned_to_signed(ret, 256, strict=True) + return ret + + def signed_to_unsigned(int_, bits, strict=False): """ Reinterpret a signed integer with n bits as an unsigned integer. @@ -243,7 +264,7 @@ def signed_to_unsigned(int_, bits, strict=False): """ if strict: lo, hi = int_bounds(signed=True, bits=bits) - assert lo <= int_ <= hi + assert lo <= int_ <= hi, int_ if int_ < 0: return int_ + 2**bits return int_ @@ -258,7 +279,7 @@ def unsigned_to_signed(int_, bits, strict=False): """ if strict: lo, hi = int_bounds(signed=False, bits=bits) - assert lo <= int_ <= hi + assert lo <= int_ <= hi, int_ if int_ > (2 ** (bits - 1)) - 1: return int_ - (2**bits) return int_ diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index ddd9065194..bb3fe58a8d 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -56,18 +56,24 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: SimplifyCFGPass(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() + # run algebraic opts before mem2var to reduce some pointer arithmetic + AlgebraicOptimizationPass(ac, fn).run_pass() StoreElimination(ac, fn).run_pass() Mem2Var(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() SCCP(ac, fn).run_pass() + SimplifyCFGPass(ac, fn).run_pass() + StoreElimination(ac, fn).run_pass() + AlgebraicOptimizationPass(ac, fn).run_pass() LoadElimination(ac, fn).run_pass() + SCCP(ac, fn).run_pass() StoreElimination(ac, fn).run_pass() - MemMergePass(ac, fn).run_pass() + SimplifyCFGPass(ac, fn).run_pass() + MemMergePass(ac, fn).run_pass() LowerDloadPass(ac, fn).run_pass() - AlgebraicOptimizationPass(ac, fn).run_pass() # NOTE: MakeSSA is after algebraic optimization it currently produces # smaller code by adding some redundant phi nodes. This is not a # problem for us, but we need to be aware of it, and should be @@ -76,6 +82,8 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: # MakeSSA again. MakeSSA(ac, fn).run_pass() BranchOptimizationPass(ac, fn).run_pass() + + AlgebraicOptimizationPass(ac, fn).run_pass() RemoveUnusedVariablesPass(ac, fn).run_pass() StoreExpansionPass(ac, fn).run_pass() diff --git a/vyper/venom/analysis/dfg.py b/vyper/venom/analysis/dfg.py index a2e050094d..e528284422 100644 --- a/vyper/venom/analysis/dfg.py +++ b/vyper/venom/analysis/dfg.py @@ -30,6 +30,9 @@ def get_uses_in_bb(self, op: IRVariable, bb: IRBasicBlock): def get_producing_instruction(self, op: IRVariable) -> Optional[IRInstruction]: return self._dfg_outputs.get(op) + def set_producing_instruction(self, op: IRVariable, inst: IRInstruction): + self._dfg_outputs[op] = inst + def add_use(self, op: IRVariable, inst: IRInstruction): uses = self._dfg_inputs.setdefault(op, OrderedSet()) uses.add(inst) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index 4c75c67700..8d86da73e7 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -4,6 +4,7 @@ import vyper.venom.effects as effects from vyper.codegen.ir_node import IRnode +from vyper.exceptions import CompilerPanic from vyper.utils import OrderedSet # instructions which can terminate a basic block @@ -92,6 +93,15 @@ from vyper.venom.function import IRFunction +def flip_comparison_opcode(opcode): + if opcode in ("gt", "sgt"): + return opcode.replace("g", "l") + elif opcode in ("lt", "slt"): + return opcode.replace("l", "g") + + raise CompilerPanic(f"unreachable {opcode}") # pragma: nocover + + class IRDebugInfo: """ IRDebugInfo represents debug information in IR, used to annotate IR @@ -318,10 +328,8 @@ def flip(self): if self.is_commutative: return - if self.opcode in ("gt", "sgt"): - self.opcode = self.opcode.replace("g", "l") - else: - self.opcode = self.opcode.replace("l", "g") + assert self.opcode in COMPARATOR_INSTRUCTIONS # sanity + self.opcode = flip_comparison_opcode(self.opcode) def replace_operands(self, replacements: dict) -> None: """ diff --git a/vyper/venom/passes/algebraic_optimization.py b/vyper/venom/passes/algebraic_optimization.py index 5d4291667e..b4f4104d5f 100644 --- a/vyper/venom/passes/algebraic_optimization.py +++ b/vyper/venom/passes/algebraic_optimization.py @@ -1,16 +1,105 @@ -from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis -from vyper.venom.basicblock import IRInstruction, IRLabel, IRLiteral, IROperand +from vyper.utils import SizeLimits, int_bounds, int_log2, is_power_of_two, wrap256 +from vyper.venom.analysis.dfg import DFGAnalysis +from vyper.venom.analysis.liveness import LivenessAnalysis +from vyper.venom.basicblock import ( + COMPARATOR_INSTRUCTIONS, + IRInstruction, + IRLabel, + IRLiteral, + IROperand, + IRVariable, + flip_comparison_opcode, +) from vyper.venom.passes.base_pass import IRPass +TRUTHY_INSTRUCTIONS = ("iszero", "jnz", "assert", "assert_unreachable") + + +def lit_eq(op: IROperand, val: int) -> bool: + return isinstance(op, IRLiteral) and wrap256(op.value) == wrap256(val) + + +class InstructionUpdater: + """ + A helper class for updating instructions which also updates the + basic block and dfg in place + """ + + def __init__(self, dfg: DFGAnalysis): + self.dfg = dfg + + def _update_operands(self, inst: IRInstruction, replace_dict: dict[IROperand, IROperand]): + old_operands = inst.operands + new_operands = [replace_dict[op] if op in replace_dict else op for op in old_operands] + self._update(inst, inst.opcode, new_operands) + + def _update(self, inst: IRInstruction, opcode: str, new_operands: list[IROperand]): + assert opcode != "phi" + # sanity + assert all(isinstance(op, IROperand) for op in new_operands) + + old_operands = inst.operands + + for op in old_operands: + if not isinstance(op, IRVariable): + continue + uses = self.dfg.get_uses(op) + if inst in uses: + uses.remove(inst) + + for op in new_operands: + if isinstance(op, IRVariable): + self.dfg.add_use(op, inst) + + inst.opcode = opcode + inst.operands = new_operands + + def _store(self, inst: IRInstruction, op: IROperand): + self._update(inst, "store", [op]) + + def _add_before(self, inst: IRInstruction, opcode: str, args: list[IROperand]) -> IRVariable: + """ + Insert another instruction before the given instruction + """ + assert opcode != "phi" + index = inst.parent.instructions.index(inst) + var = inst.parent.parent.get_next_variable() + operands = list(args) + new_inst = IRInstruction(opcode, operands, output=var) + inst.parent.insert_instruction(new_inst, index) + for op in new_inst.operands: + if isinstance(op, IRVariable): + self.dfg.add_use(op, new_inst) + self.dfg.add_use(var, inst) + self.dfg.set_producing_instruction(var, new_inst) + return var + class AlgebraicOptimizationPass(IRPass): """ This pass reduces algebraic evaluatable expressions. It currently optimizes: - * iszero chains + - iszero chains + - binops + - offset adds """ + dfg: DFGAnalysis + updater: InstructionUpdater + + def run_pass(self): + self.dfg = self.analyses_cache.request_analysis(DFGAnalysis) # type: ignore + self.updater = InstructionUpdater(self.dfg) + self._handle_offset() + + self._algebraic_opt() + self._optimize_iszero_chains() + self._algebraic_opt() + + self.analyses_cache.invalidate_analysis(DFGAnalysis) + self.analyses_cache.invalidate_analysis(LivenessAnalysis) + def _optimize_iszero_chains(self) -> None: fn = self.function for bb in fn.get_basic_blocks(): @@ -23,7 +112,8 @@ def _optimize_iszero_chains(self) -> None: if iszero_count == 0: continue - for use_inst in self.dfg.get_uses(inst.output): + assert isinstance(inst.output, IRVariable) + for use_inst in self.dfg.get_uses(inst.output).copy(): opcode = use_inst.opcode if opcode == "iszero": @@ -42,12 +132,14 @@ def _optimize_iszero_chains(self) -> None: continue out_var = iszero_chain[keep_count].operands[0] - use_inst.replace_operands({inst.output: out_var}) + self.updater._update_operands(use_inst, {inst.output: out_var}) def _get_iszero_chain(self, op: IROperand) -> list[IRInstruction]: chain: list[IRInstruction] = [] while True: + if not isinstance(op, IRVariable): + break inst = self.dfg.get_producing_instruction(op) if inst is None or inst.opcode != "iszero": break @@ -57,24 +149,302 @@ def _get_iszero_chain(self, op: IROperand) -> list[IRInstruction]: chain.reverse() return chain - def _handle_offsets(self): + def _handle_offset(self): for bb in self.function.get_basic_blocks(): for inst in bb.instructions: - # check if the instruction is of the form - # `add