diff --git a/planetarium/ASSUMPTIONS.md b/planetarium/ASSUMPTIONS.md index 2a6ab26..2e960ef 100644 --- a/planetarium/ASSUMPTIONS.md +++ b/planetarium/ASSUMPTIONS.md @@ -3,6 +3,8 @@ For each domain, the following assumptions are made regarding the types of problems evaluated. We also assume that problems are solvable via the domain's actions. +Our equivalence check is based on the assumption that the problems are solvable already (i.e. our evaluator checks solvability before equivalence). + ## Blocksworld Generally, since Blocks World has reversible actions, essentially all problems are evaluable. @@ -23,15 +25,8 @@ Generally, since Grippers has reversible actions, essentially all problems are e - All grippers have only one location - All grippers hold up to 1 ball -## Rover -Rover has the capability of being a much more complex domain, but for the purposes of this benchmark, we assume all problems will have symmetry of traversal. -This changes our `fullySpecify` function from essentially a multi-agent traveling salesman problem into simple pathfinding between two points. -Our goal is not to be *complete*, but to be *correct* for the problems we do support evaluating. +## Rover Single +Rover has the capability of being a much more complex domain, but for the purposes of this benchmark, we work only with a single rover and a single lander. `:init` conditions: -- All objects have 1 type -- All rovers only have one location -- All landers only have one location -- If `can_traverse ?rover ?x ?y`, then `can_traverse ?rover ?y ?x` - -**Note**: For the special case of up to 1 rover and up to 1 sample per analysis (rock, soil, imaging), we do not require symmmetry. \ No newline at end of file +- No double `at_*` predicates (rover and lander can only be in one location at a time) \ No newline at end of file diff --git a/planetarium/oracle.py b/planetarium/oracle.py index 7a08a9b..8a15f57 100644 --- a/planetarium/oracle.py +++ b/planetarium/oracle.py @@ -17,7 +17,11 @@ _fully_specify_gripper, _plan_gripper, ) -from .oracles.rover_single import _reduce_rover_single, _inflate_rover_single, _fully_specify_rover_single +from .oracles.rover_single import ( + _reduce_rover_single, + _inflate_rover_single, + _fully_specify_rover_single, +) plan_template = jinja.Template( diff --git a/planetarium/oracles/rover_single.py b/planetarium/oracles/rover_single.py index 0976620..3b28bdc 100644 --- a/planetarium/oracles/rover_single.py +++ b/planetarium/oracles/rover_single.py @@ -299,7 +299,6 @@ def rover_subgraph(scene: ReducedSceneGraph) -> ReducedSceneGraph: subgraph.add_edge(v, u, copy.deepcopy(edge)) # add visible edges from waypoint to lander for waypoint, in_edge in scene.in_edges(v): - # print(in_edge, waypoint.node, u.node, "IN EDGE") if in_edge.predicate == "visible": subgraph.add_edge(waypoint, u, copy.deepcopy(in_edge)) @@ -406,7 +405,7 @@ def _fully_specify_rover_single( for pred in inflated_goal.predicates: params = pred["parameters"] match pred["typing"].split("_"): - case [_, sample, _] if sample in ("rock", "soil", "image"): + case ["communicated", sample, "data"]: have_pred = { "typing": f"have_{sample}{'_analysis' if sample != 'image' else ''}", "parameters": params, @@ -424,7 +423,6 @@ def _fully_specify_rover_single( for pred in inflated_goal.predicates: match pred["typing"].split("_"): case ["have", sample, *_] if sample in ("rock", "soil", "image"): - print(pred) waypoint = pred["parameters"][0] # check if the rover needs to communicate the data com_pred = { diff --git a/tests/problem_fixtures.py b/tests/problem_fixtures.py index 4328d43..8607e51 100644 --- a/tests/problem_fixtures.py +++ b/tests/problem_fixtures.py @@ -1916,3 +1916,262 @@ def rover_single_line_fully_specified_4a(): ) ) )""" + + +@pytest.fixture +def rover_single_line_equiv(): + return """ + (define (problem rover) + (:domain rover-single) + (:objects + site1 site2 site3 site4 site5 site6 - waypoint + store1 store2 - store + rgb - mode + camera1 camera2 camera3 - camera + objective1 objective2 objective3 - objective + ) + (:init + (can_traverse site1 site2) + (can_traverse site2 site3) + (can_traverse site3 site4) + (can_traverse site4 site5) + (can_traverse site5 site6) + (visible site1 site2) + (visible site2 site3) + (visible site3 site4) + (visible site4 site5) + (visible site5 site6) + (at_rover site1) + (available) + (at_lander site6) + (empty store1) + (full store2) + (supports camera1 rgb) + (supports camera2 rgb) + (supports camera3 rgb) + (visible_from objective1 site5) + (channel_free) + ) + (:goal + (and + (at_lander site6) + (communicated_image_data objective1 rgb) + ) + ) + )""" + + +@pytest.fixture +def rover_single_line_equiva(): + return """ + (define (problem rover) + (:domain rover-single) + (:objects + site1 site2 site3 site4 site5 site6 - waypoint + store1 store2 - store + rgb - mode + camera1 camera2 camera3 - camera + objective1 objective2 objective3 - objective + ) + (:init + (can_traverse site1 site2) + (can_traverse site2 site3) + (can_traverse site3 site4) + (can_traverse site4 site5) + (can_traverse site5 site6) + (visible site1 site2) + (visible site2 site3) + (visible site3 site4) + (visible site4 site5) + (visible site5 site6) + (at_rover site1) + (available) + (at_lander site6) + (empty store1) + (full store2) + (supports camera1 rgb) + (supports camera2 rgb) + (supports camera3 rgb) + (visible_from objective1 site5) + (channel_free) + ) + (:goal + (and + (can_traverse site1 site2) + (can_traverse site2 site3) + (can_traverse site3 site4) + (can_traverse site4 site5) + (can_traverse site5 site6) + (visible site1 site2) + (visible site2 site3) + (visible site3 site4) + (visible site4 site5) + (visible site5 site6) + (available) + (supports camera1 rgb) + (supports camera2 rgb) + (supports camera3 rgb) + (visible_from objective1 site5) + (channel_free) + (have_image objective1 rgb) + (communicated_image_data objective1 rgb) + ) + ) + )""" + + +@pytest.fixture +def rover_single_line_equiv_1(): + return """ + (define (problem rover) + (:domain rover-single) + (:objects + site1 site2 site3 site4 site5 site6 - waypoint + store1 store2 - store + rgb - mode + camera1 camera2 camera3 - camera + objective1 objective2 objective3 - objective + ) + (:init + (can_traverse site1 site2) + (can_traverse site2 site3) + (can_traverse site3 site4) + (can_traverse site4 site5) + (can_traverse site5 site6) + (visible site1 site2) + (visible site2 site3) + (visible site3 site4) + (visible site4 site5) + (visible site5 site6) + (visible site6 site6) + (at_rover site1) + (available) + (at_lander site6) + (at_rock_sample site6) + (empty store1) + (full store2) + (supports camera1 rgb) + (supports camera2 rgb) + (supports camera3 rgb) + (visible_from objective1 site5) + (channel_free) + ) + (:goal + (and + (can_traverse site1 site2) + (can_traverse site2 site3) + (can_traverse site3 site4) + (can_traverse site4 site5) + (can_traverse site5 site6) + (visible site1 site2) + (visible site2 site3) + (visible site3 site4) + (visible site4 site5) + (visible site5 site6) + (visible site6 site6) + (available) + (at_lander site6) + (supports camera1 rgb) + (supports camera2 rgb) + (supports camera3 rgb) + (visible_from objective1 site5) + (channel_free) + (at_rover site6) + (have_rock_analysis site6) + (communicated_rock_data site6) + (have_image objective1 rgb) + (communicated_image_data objective1 rgb) + ) + ) + )""" + + +@pytest.fixture +def rover_single_line_equiv_1a(): + return """ + (define (problem rover) + (:domain rover-single) + (:objects + site1 site2 site3 site4 site5 site6 - waypoint + store1 store2 - store + rgb - mode + camera1 camera2 camera3 - camera + objective1 objective2 objective3 - objective + ) + (:init + (can_traverse site1 site2) + (can_traverse site2 site3) + (can_traverse site3 site4) + (can_traverse site4 site5) + (can_traverse site5 site6) + (visible site1 site2) + (visible site2 site3) + (visible site3 site4) + (visible site4 site5) + (visible site5 site6) + (visible site6 site6) + (at_rover site1) + (available) + (at_lander site6) + (at_rock_sample site6) + (empty store1) + (full store2) + (supports camera1 rgb) + (supports camera2 rgb) + (supports camera3 rgb) + (visible_from objective1 site5) + (channel_free) + ) + (:goal + (and + (at_rover site6) + (communicated_rock_data site6) + (communicated_image_data objective1 rgb) + ) + ) + )""" + + +@pytest.fixture +def rover_single_line_equiv_1b(): + return """ + (define (problem rover) + (:domain rover-single) + (:objects + site1 site2 site3 site4 site5 site6 - waypoint + store1 store2 - store + rgb - mode + camera1 camera2 camera3 - camera + objective1 objective2 objective3 - objective + ) + (:init + (can_traverse site1 site2) + (can_traverse site2 site3) + (can_traverse site3 site4) + (can_traverse site4 site5) + (can_traverse site5 site6) + (visible site1 site2) + (visible site2 site3) + (visible site3 site4) + (visible site4 site5) + (visible site5 site6) + (visible site6 site6) + (at_rover site1) + (available) + (at_lander site6) + (at_rock_sample site6) + (empty store1) + (full store2) + (supports camera1 rgb) + (supports camera2 rgb) + (supports camera3 rgb) + (visible_from objective1 site5) + (channel_free) + ) + (:goal + (and + (communicated_rock_data site6) + (communicated_image_data objective1 rgb) + ) + ) + )""" diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py index 1bb3f57..99d3aa1 100644 --- a/tests/test_evaluate.py +++ b/tests/test_evaluate.py @@ -9,6 +9,11 @@ blocksworld_missing_ontables, blocksworld_fully_specified, blocksworld_invalid_1, + rover_single_line_equiv, + rover_single_line_equiva, + rover_single_line_equiv_1, + rover_single_line_equiv_1a, + rover_single_line_equiv_1b, ) @@ -281,6 +286,47 @@ def test_evaluate_inequivalent( False, ) + def test_rover_single_equivalent( + self, + subtests, + rover_single_line_equiv, + rover_single_line_equiva, + rover_single_line_equiv_1, + rover_single_line_equiv_1a, + rover_single_line_equiv_1b, + ): + """ + Test if the evaluation of PDDL problem descriptions is correct. + """ + def print_diff(desc1, desc2): + full_1 = planetarium.oracle.fully_specify(planetarium.builder.build(desc1)) + full_2 = planetarium.oracle.fully_specify(planetarium.builder.build(desc2)) + print([p for p in full_1.goal().predicates if p not in full_2.goal().predicates]) + print([p for p in full_2.goal().predicates if p not in full_1.goal().predicates]) + def print_diff_init(desc1, desc2): + full_1 = planetarium.oracle.fully_specify(planetarium.builder.build(desc1)) + full_2 = planetarium.oracle.fully_specify(planetarium.builder.build(desc2)) + print([p for p in full_1.init().predicates if p not in full_2.init().predicates]) + print([p for p in full_2.init().predicates if p not in full_1.init().predicates]) + + with subtests.test("rover_single_line_equiv equals rover_single_line_equiva"): + print_diff(rover_single_line_equiv, rover_single_line_equiva) + assert all( + planetarium.evaluate( + rover_single_line_equiv, + rover_single_line_equiva, + ) + ) + + descs = { + "rover_single_line_equiv_1": rover_single_line_equiv_1, + "rover_single_line_equiv_1a": rover_single_line_equiv_1a, + "rover_single_line_equiv_1b": rover_single_line_equiv_1b, + } + for (name1, desc1), (name2, desc2) in product(descs.items(), descs.items()): + with subtests.test(f"{name1} equals {name2}"): + assert all(planetarium.evaluate(desc1, desc2)) + class TestUnsupportedDomain: """ diff --git a/tests/test_metric.py b/tests/test_metric.py index 6673a47..a8c8bdc 100644 --- a/tests/test_metric.py +++ b/tests/test_metric.py @@ -290,7 +290,12 @@ def test_blocksworld_equivalence( assert not metric.equals(p2, p1, is_placeholder=True) assert not metric.equals(p2, p1, is_placeholder=False) - def test_rover_single_eqquivalence(self, subtests, rover_single_line_fully_specified_4, rover_single_line_fully_specified_4a,): + def test_rover_single_eqquivalence( + self, + subtests, + rover_single_line_fully_specified_4, + rover_single_line_fully_specified_4a, + ): """Test the equivalence of rover single line problems.""" p1 = builder.build(rover_single_line_fully_specified_4) p2 = builder.build(rover_single_line_fully_specified_4a) @@ -306,4 +311,4 @@ def test_rover_single_eqquivalence(self, subtests, rover_single_line_fully_speci assert not metric.equals(p1, p2, is_placeholder=True) assert not metric.equals(p1, p2, is_placeholder=False) assert not metric.equals(p2, p1, is_placeholder=True) - assert not metric.equals(p2, p1, is_placeholder=False) \ No newline at end of file + assert not metric.equals(p2, p1, is_placeholder=False) diff --git a/tests/test_oracle.py b/tests/test_oracle.py index 2c2e534..5a987f0 100644 --- a/tests/test_oracle.py +++ b/tests/test_oracle.py @@ -203,16 +203,27 @@ def test_fully_specified( """ descs = [ ("rover_single_line_fully_specified", rover_single_line_fully_specified), - ("rover_single_line_fully_specified_1", rover_single_line_fully_specified_1), - ("rover_single_line_fully_specified_2", rover_single_line_fully_specified_2), - ("rover_single_line_fully_specified_3", rover_single_line_fully_specified_3), - ("rover_single_line_fully_specified_4", rover_single_line_fully_specified_4), + ( + "rover_single_line_fully_specified_1", + rover_single_line_fully_specified_1, + ), + ( + "rover_single_line_fully_specified_2", + rover_single_line_fully_specified_2, + ), + ( + "rover_single_line_fully_specified_3", + rover_single_line_fully_specified_3, + ), + ( + "rover_single_line_fully_specified_4", + rover_single_line_fully_specified_4, + ), ] for name, desc in descs: with subtests.test(name): problem = builder.build(desc) full = oracle.fully_specify(problem) - print([pred for pred in full.goal().predicates if pred not in problem.goal().predicates]) assert full == problem, name assert oracle.fully_specify(full) == full, name