diff --git a/scripts/benchmarking.py b/scripts/benchmarking.py index f46831b..404fbeb 100644 --- a/scripts/benchmarking.py +++ b/scripts/benchmarking.py @@ -4,8 +4,8 @@ First requires the test dat files to be created by unzipping gis_osm_roads_free_1.zip and running load_osm_network.py -cd D:/Github/Python/wayfarer -#./wayfarer-venv/Scripts/activate.ps1 +C:\VirtualEnvs\wayfarer\Scripts\activate.ps1 +cd D:/Github/wayfarer pytest -v scripts/benchmarking.py @@ -13,6 +13,8 @@ import pytest import copy +import marshal +import networkx from wayfarer import loader, functions @@ -63,9 +65,44 @@ def clone_network(net): copy.deepcopy(net) +def marshal_network(net): + marshal.loads(marshal.dumps(net)) + + def test_cloning_network(benchmark): """ 971 ms seconds to clone - much slower than simply re-reading from disk """ net = loader.load_network_from_file("./data/dublin.pickle") benchmark(clone_network, net) + + +def xtest_marshal_network(benchmark): + """ + ValueError: unmarshallable object - it is not possible to + use marshal.dumps on the networkx network + """ + net = loader.load_network_from_file("./data/dublin.pickle") + benchmark(marshal_network, net) + + +def test_get_edges_from_nodes_all_lengths(benchmark): + """ + pytest -v scripts/benchmarking.py -k "test_get_edges_from_nodes_all_lengths" + + with copy.deepcopy - min 7.4000 + with marshal - min 4.9000 + + When running path solving with over 100,000 iterations: + marshal: 588 seconds + copy.deepcopy: 660 seconds + + See discussions at https://stackoverflow.com/a/55157627/179520 + """ + net = networkx.MultiGraph() + + net.add_edge(1, 2, key="A", **{"EDGE_ID": "A", "LEN_": 50}) + net.add_edge(1, 2, key="B", **{"EDGE_ID": "B", "LEN_": 100}) + + node_list = [1, 2] + benchmark(functions.get_edges_from_nodes, net, node_list, shortest_path_only=False) diff --git a/wayfarer/functions.py b/wayfarer/functions.py index fe4e9a4..22ff12c 100644 --- a/wayfarer/functions.py +++ b/wayfarer/functions.py @@ -4,7 +4,7 @@ from __future__ import annotations import itertools -import copy +import marshal import logging from collections import OrderedDict import networkx @@ -325,7 +325,8 @@ def get_all_paths_from_nodes(net, node_list, with_direction_flag=False): for key, attributes in edges.items(): # make a copy so client programs can modify without # affecting the original edge dict - atts_copy = copy.deepcopy(attributes) + # atts_copy = copy.deepcopy(attributes) + atts_copy = marshal.loads(marshal.dumps(attributes)) edge = Edge(start_node=u, end_node=v, key=key, attributes=atts_copy) if with_direction_flag: add_direction_flag( @@ -403,13 +404,13 @@ def get_edges_from_nodes( if edges: if shortest_path_only: key, attributes = get_shortest_edge(edges, length_field) - atts_copy = copy.deepcopy(attributes) + atts_copy = marshal.loads(marshal.dumps(attributes)) edge = Edge(start_node=u, end_node=v, key=key, attributes=atts_copy) node_edges.append(edge) else: # get all edges between the nodes, ignoring the length_field for key, attributes in edges.items(): - atts_copy = copy.deepcopy(attributes) + atts_copy = marshal.loads(marshal.dumps(attributes)) node_edges.append( Edge(start_node=u, end_node=v, key=key, attributes=atts_copy) )