From 516f1da1f458fc5a4472f388f9f27ced9a582994 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 17:53:23 +0100 Subject: [PATCH 1/5] dont use old spalloc --- .../interface_functions/spalloc_allocator.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py index ae2085d69..13ce66fb8 100644 --- a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py +++ b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py @@ -22,7 +22,8 @@ from spinn_utilities.log import FormatAdapter from spinn_utilities.overrides import overrides from spinn_utilities.typing.coords import XY -from spinn_utilities.config_holder import get_config_int, get_config_str +from spinn_utilities.config_holder import ( + get_config_int, get_config_str, set_config) from spalloc_client import Job # type: ignore[import] from spalloc_client.states import JobState # type: ignore[import] @@ -312,13 +313,12 @@ def spalloc_allocator( if n_boards - n_boards_float < 0.5: n_boards += 1 - if is_server_address(spalloc_server): - host, connections, mac = _allocate_job_new( - spalloc_server, n_boards, bearer_token, group, collab, - int(nmpi_job) if nmpi_job is not None else None, - nmpi_user) - else: - host, connections, mac = _allocate_job_old(spalloc_server, n_boards) + if not is_server_address(spalloc_server): + set_config("Machine", "spalloc_use_proxy", False) + host, connections, mac = _allocate_job_new( + spalloc_server, n_boards, bearer_token, group, collab, + int(nmpi_job) if nmpi_job is not None else None, + nmpi_user) return (host, _MACHINE_VERSION, None, False, False, connections, mac) From c80596a5b30cfec22c6772d1923a1bb832b30b90 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 18:00:16 +0100 Subject: [PATCH 2/5] remove old spalloc code --- .../interface_functions/spalloc_allocator.py | 150 +----------------- .../utilities/utility_calls.py | 32 ---- 2 files changed, 1 insertion(+), 181 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py index 13ce66fb8..854df45af 100644 --- a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py +++ b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py @@ -39,7 +39,6 @@ MachineAllocationController) from spinn_front_end_common.data import FecDataView from spinn_front_end_common.interface.provenance import ProvenanceWriter -from spinn_front_end_common.utilities.utility_calls import parse_old_spalloc logger = FormatAdapter(logging.getLogger(__name__)) _MACHINE_VERSION = 5 # Spalloc only ever works with v5 boards @@ -198,73 +197,6 @@ def make_report(self, filename: str): report.write(f"Job: {self._job}") -class _OldSpallocJobController(MachineAllocationController): - __slots__ = ( - # the spalloc job object - "_job", - # the current job's old state - "_state" - ) - - def __init__(self, job: Job, host: str): - """ - :param ~spalloc.job.Job job: - """ - if job is None: - raise TypeError("must have a real job") - self._job = job - self._state = job.state - super().__init__("SpallocJobController", host) - - @overrides(MachineAllocationController.extend_allocation) - def extend_allocation(self, new_total_run_time: float): - # Does Nothing in this allocator - machines are held until exit - pass - - @overrides(MachineAllocationController.close) - def close(self) -> None: - super().close() - self._job.destroy() - - @property - def power(self) -> bool: - """ - :rtype: bool - """ - return self._job.power - - def set_power(self, power: bool): - """ - :param bool power: - """ - self._job.set_power(power) - if power: - self._job.wait_until_ready() - - @overrides(MachineAllocationController.where_is_machine) - def where_is_machine( - self, chip_x: int, chip_y: int) -> Tuple[int, int, int]: - return self._job.where_is_machine(chip_y=chip_y, chip_x=chip_x) - - @overrides(MachineAllocationController._wait) - def _wait(self) -> bool: - try: - if self._state != JobState.destroyed: - self._state = self._job.wait_for_state_change(self._state) - except TypeError: - pass - except Exception as e: # pylint: disable=broad-except - if not self._exited: - raise e - return self._state != JobState.destroyed - - @overrides(MachineAllocationController._teardown) - def _teardown(self) -> None: - if not self._exited: - self._job.close() - super()._teardown() - - _MACHINE_VERSION = 5 @@ -314,7 +246,7 @@ def spalloc_allocator( n_boards += 1 if not is_server_address(spalloc_server): - set_config("Machine", "spalloc_use_proxy", False) + set_config("Machine", "spalloc_use_proxy", "False") host, connections, mac = _allocate_job_new( spalloc_server, n_boards, bearer_token, group, collab, int(nmpi_job) if nmpi_job is not None else None, @@ -374,83 +306,3 @@ def _allocate_job_new( stack.pop_all() assert root is not None, "no root of ready board" return (root, connections, allocation_controller) - - -def _allocate_job_old(spalloc_server: str, n_boards: int) -> Tuple[ - str, Dict[XY, str], MachineAllocationController]: - """ - Request a machine from an old-style spalloc server that will fit the - requested number of boards. - - :param str spalloc_server: - The server from which the machine should be requested - :param int n_boards: The number of boards required - :rtype: tuple(str, dict(tuple(int,int),str), MachineAllocationController) - """ - host, port, user = parse_old_spalloc( - spalloc_server, get_config_int("Machine", "spalloc_port"), - get_config_str("Machine", "spalloc_user")) - spalloc_kwargs = { - 'hostname': host, - 'port': port, - 'owner': user - } - spalloc_machine = get_config_str_or_none("Machine", "spalloc_machine") - - if spalloc_machine is not None: - spalloc_kwargs['machine'] = spalloc_machine - - job, hostname, scamp_connection_data = _launch_checked_job_old( - n_boards, spalloc_kwargs) - machine_allocation_controller = _OldSpallocJobController(job, hostname) - return (hostname, scamp_connection_data, machine_allocation_controller) - - -def _launch_checked_job_old(n_boards: int, spalloc_kwargs: dict) -> Tuple[ - Job, str, Dict[XY, str]]: - """ - :rtype: tuple(~.Job, str, dict(tuple(int,int),str)) - """ - logger.info(f"Requesting job with {n_boards} boards") - avoid_boards = get_config_str_list("Machine", "spalloc_avoid_boards") - avoid_jobs = [] - try: - while True: - job = Job(n_boards, **spalloc_kwargs) - try: - job.wait_until_ready() - # get param from jobs before starting, so that hanging doesn't - # occur - hostname = job.hostname - except Exception as ex: - job.destroy(str(ex)) - raise - connections = job.connections - if len(connections) < n_boards: - logger.warning( - "boards: {}", - str(connections).replace("{", "[").replace("}", "]")) - raise ValueError("Not enough connections detected") - if logger.isEnabledFor(logging.DEBUG): - logger.debug("boards: {}", - str(connections).replace("{", "[").replace( - "}", "]")) - with ProvenanceWriter() as db: - db.insert_board_provenance(connections) - if hostname not in avoid_boards: - break - avoid_jobs.append(job) - logger.warning( - f"Asking for new job as {hostname} " - f"as in the spalloc_avoid_boards list") - finally: - if avoid_boards: - for key in list(connections.keys()): - if connections[key] in avoid_boards: - logger.warning( - f"Removing connection info for {connections[key]} " - f"as in the spalloc avoid_boards list") - del connections[key] - for avoid_job in avoid_jobs: - avoid_job.destroy("Asked to avoid by cfg") - return job, hostname, connections diff --git a/spinn_front_end_common/utilities/utility_calls.py b/spinn_front_end_common/utilities/utility_calls.py index 4b8ed6ba2..d28c2b071 100644 --- a/spinn_front_end_common/utilities/utility_calls.py +++ b/spinn_front_end_common/utilities/utility_calls.py @@ -106,38 +106,6 @@ def get_report_writer( return io.TextIOWrapper(io.FileIO(name, "w")) -def parse_old_spalloc( - spalloc_server: str, spalloc_port: int, - spalloc_user: str) -> Tuple[str, int, str]: - """ - Parse a URL to the old-style service. This may take the form: - - spalloc://user@spalloc.host.example.com:22244 - - The leading ``spalloc://`` is the mandatory part (as is the actual host - name). If the port and user are omitted, the defaults given in the other - arguments are used (or default defaults). - - A bare hostname can be used instead. If that's the case (i.e., there's no - ``spalloc://`` prefix) then the port and user are definitely used. - - :param str spalloc_server: Hostname or URL - :param int spalloc_port: Default port - :param str spalloc_user: Default user - :return: hostname, port, username - :rtype: tuple(str,int,str) - """ - if spalloc_port is None or spalloc_port == "": - spalloc_port = 22244 - if spalloc_user is None or spalloc_user == "": - spalloc_user = "unknown user" - parsed = urlparse(spalloc_server, "spalloc") - if parsed.netloc == "" or parsed.hostname is None: - return spalloc_server, spalloc_port, spalloc_user - return parsed.hostname, (parsed.port or spalloc_port), \ - (parsed.username or spalloc_user) - - def retarget_tag( connection: Union[SpallocEIEIOListener, SpallocEIEIOConnection, SCAMPConnection], x: int, y: int, tag: int, From 1bce94b076ecc99c3a38552275847e77d18b1641 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Jun 2024 18:06:18 +0100 Subject: [PATCH 3/5] remove unused imports --- .../interface/interface_functions/spalloc_allocator.py | 7 ++----- spinn_front_end_common/utilities/utility_calls.py | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py index 854df45af..67e80105c 100644 --- a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py +++ b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py @@ -18,15 +18,12 @@ from typing import ContextManager, Dict, Tuple, Optional, Union, cast from spinn_utilities.config_holder import ( - get_config_bool, get_config_str_or_none, get_config_str_list) + get_config_bool, get_config_str_or_none) from spinn_utilities.log import FormatAdapter from spinn_utilities.overrides import overrides from spinn_utilities.typing.coords import XY from spinn_utilities.config_holder import ( - get_config_int, get_config_str, set_config) - -from spalloc_client import Job # type: ignore[import] -from spalloc_client.states import JobState # type: ignore[import] + get_config_str, set_config) from spinnman.connections.udp_packet_connections import ( SCAMPConnection, EIEIOConnection) diff --git a/spinn_front_end_common/utilities/utility_calls.py b/spinn_front_end_common/utilities/utility_calls.py index d28c2b071..257433ac2 100644 --- a/spinn_front_end_common/utilities/utility_calls.py +++ b/spinn_front_end_common/utilities/utility_calls.py @@ -19,8 +19,7 @@ import io import os import threading -from typing import (Optional, Union, TextIO, Tuple, TypeVar) -from urllib.parse import urlparse +from typing import (Optional, Union, TextIO, TypeVar) from spinn_utilities.config_holder import get_config_bool from spinn_machine import Chip from spinnman.connections.udp_packet_connections import SCAMPConnection From 3727e702174509aa4de236aa2ebe1b623f87c966 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Fri, 28 Jun 2024 06:22:52 +0100 Subject: [PATCH 4/5] hack user --- .../interface/interface_functions/spalloc_allocator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py index 67e80105c..301f75ae3 100644 --- a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py +++ b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py @@ -243,7 +243,10 @@ def spalloc_allocator( n_boards += 1 if not is_server_address(spalloc_server): - set_config("Machine", "spalloc_use_proxy", "False") + if spalloc_server.lower() in ["spinnaker.cs.man.ac.uk", "10.11.192.11"]: + set_config("Machine", "spalloc_use_proxy", "False") + set_config("Machine", "spalloc_server", + "https://hack:hack@spinnaker.cs.man.ac.uk/spalloc/") host, connections, mac = _allocate_job_new( spalloc_server, n_boards, bearer_token, group, collab, int(nmpi_job) if nmpi_job is not None else None, From 474f01f685a08b2d9c5d140956e0bfdbbb6e31a6 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Fri, 28 Jun 2024 06:56:50 +0100 Subject: [PATCH 5/5] set spalloc_server --- .../interface/interface_functions/spalloc_allocator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py index 301f75ae3..3d51f9445 100644 --- a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py +++ b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py @@ -245,8 +245,8 @@ def spalloc_allocator( if not is_server_address(spalloc_server): if spalloc_server.lower() in ["spinnaker.cs.man.ac.uk", "10.11.192.11"]: set_config("Machine", "spalloc_use_proxy", "False") - set_config("Machine", "spalloc_server", - "https://hack:hack@spinnaker.cs.man.ac.uk/spalloc/") + spalloc_server = \ + "https://hack:hack@spinnaker.cs.man.ac.uk/spalloc/" host, connections, mac = _allocate_job_new( spalloc_server, n_boards, bearer_token, group, collab, int(nmpi_job) if nmpi_job is not None else None,