binary = False, python_version = '3.8'
control_model_handle_tag_tuple = (PosixPath('/tmp/pytest-of-runner/pytest-0/test_control_truss_python_sys_0/control_truss'), <truss.truss_handle.truss_handle.TrussHandle object at 0x7f2e441d5250>, 'test-docker-custom-model-control-tag:0.0.1')

    @pytest.mark.integration
    @pytest.mark.parametrize(
        "binary, python_version",
        [
            (binary, python_version)
            for binary in [True, False]
            for python_version in SUPPORTED_PYTHON_VERSIONS
        ],
    )
    def test_control_truss_python_sys_req_patch(
        binary, python_version, control_model_handle_tag_tuple
    ):
        _, th, tag = control_model_handle_tag_tuple
        th.update_python_version(python_version)
    
        def predict_with_python_requirement_added(req: str):
            th.add_python_requirement(req)
            return th.docker_predict([1], tag=tag, binary=binary)
    
        def predict_with_python_requirement_removed(req):
            th.remove_python_requirement(req)
            return th.docker_predict([1], tag=tag, binary=binary)
    
        def predict_with_system_requirement_added(pkg):
            th.add_system_package(pkg)
            return th.docker_predict([1], tag=tag, binary=binary)
    
        def predict_with_system_requirement_removed(pkg):
            th.remove_system_package(pkg)
            return th.docker_predict([1], tag=tag, binary=binary)
    
        with ensure_kill_all():
>           th.docker_predict([1], tag=tag, binary=binary)

truss/tests/test_control_truss_patching.py:160: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
truss/truss_handle/decorators.py:7: in wrapper
    return func(*args, **kwargs)
truss/truss_handle/truss_handle.py:424: in docker_predict
    resp = TrussHandle._wait_for_predict(model_base_url, request, binary)
.venv/lib/python3.9/site-packages/tenacity/__init__.py:336: in wrapped_f
    return copy(f, *args, **kw)
.venv/lib/python3.9/site-packages/tenacity/__init__.py:475: in __call__
    do = self.iter(retry_state=retry_state)
.venv/lib/python3.9/site-packages/tenacity/__init__.py:376: in iter
    result = action(retry_state)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

rs = <RetryCallState 139836689400016: attempt #20; slept for 19.0; last result: returned <Response [503]>>

    def exc_check(rs: "RetryCallState") -> None:
        fut = t.cast(Future, rs.outcome)
        retry_exc = self.retry_error_cls(fut)
        if self.reraise:
            raise retry_exc.reraise()
>       raise retry_exc from fut.exception()
E       tenacity.RetryError: RetryError[<Future at 0x7f2e441c9250 state=finished returned Response>]

.venv/lib/python3.9/site-packages/tenacity/__init__.py:419: RetryError

binary = True, python_version = '3.8'
control_model_handle_tag_tuple = (PosixPath('/tmp/pytest-of-runner/pytest-0/test_control_truss_patch_ignor3/control_truss'), <truss.truss_handle.truss_handle.TrussHandle object at 0x7f2e414c5eb0>, 'test-docker-custom-model-control-tag:0.0.1')

    @pytest.mark.integration
    @pytest.mark.parametrize(
        "binary, python_version",
        [
            (binary, python_version)
            for binary in [True, False]
            for python_version in SUPPORTED_PYTHON_VERSIONS
        ],
    )
    def test_control_truss_patch_ignored_changes(
        binary, python_version, control_model_handle_tag_tuple
    ):
        custom_model_control, th, tag = control_model_handle_tag_tuple
        th.update_python_version(python_version)
    
        def predict_with_ignored_changes():
            top_pycache_path = custom_model_control / "__pycache__"
            top_pycache_path.mkdir()
            (top_pycache_path / "bla.pyc").touch()
            model_pycache_path = custom_model_control / "model" / "__pycache__"
            model_pycache_path.mkdir()
            (model_pycache_path / "foo.pyc").touch()
            return th.docker_predict([1], tag=tag, binary=binary)
    
        with ensure_kill_all():
>           th.docker_predict([1], tag=tag, binary=binary)

truss/tests/test_control_truss_patching.py:209: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
truss/truss_handle/decorators.py:7: in wrapper
    return func(*args, **kwargs)
truss/truss_handle/truss_handle.py:424: in docker_predict
    resp = TrussHandle._wait_for_predict(model_base_url, request, binary)
.venv/lib/python3.9/site-packages/tenacity/__init__.py:336: in wrapped_f
    return copy(f, *args, **kw)
.venv/lib/python3.9/site-packages/tenacity/__init__.py:475: in __call__
    do = self.iter(retry_state=retry_state)
.venv/lib/python3.9/site-packages/tenacity/__init__.py:376: in iter
    result = action(retry_state)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

rs = <RetryCallState 139836642525344: attempt #20; slept for 19.0; last result: returned <Response [503]>>

    def exc_check(rs: "RetryCallState") -> None:
        fut = t.cast(Future, rs.outcome)
        retry_exc = self.retry_error_cls(fut)
        if self.reraise:
            raise retry_exc.reraise()
>       raise retry_exc from fut.exception()
E       tenacity.RetryError: RetryError[<Future at 0x7f2e41478d60 state=finished returned Response>]

.venv/lib/python3.9/site-packages/tenacity/__init__.py:419: RetryError

binary = False, python_version = '3.8'
control_model_handle_tag_tuple = (PosixPath('/tmp/pytest-of-runner/pytest-0/test_control_truss_patch_ignor7/control_truss'), <truss.truss_handle.truss_handle.TrussHandle object at 0x7f2e409678e0>, 'test-docker-custom-model-control-tag:0.0.1')

    @pytest.mark.integration
    @pytest.mark.parametrize(
        "binary, python_version",
        [
            (binary, python_version)
            for binary in [True, False]
            for python_version in SUPPORTED_PYTHON_VERSIONS
        ],
    )
    def test_control_truss_patch_ignored_changes(
        binary, python_version, control_model_handle_tag_tuple
    ):
        custom_model_control, th, tag = control_model_handle_tag_tuple
        th.update_python_version(python_version)
    
        def predict_with_ignored_changes():
            top_pycache_path = custom_model_control / "__pycache__"
            top_pycache_path.mkdir()
            (top_pycache_path / "bla.pyc").touch()
            model_pycache_path = custom_model_control / "model" / "__pycache__"
            model_pycache_path.mkdir()
            (model_pycache_path / "foo.pyc").touch()
            return th.docker_predict([1], tag=tag, binary=binary)
    
        with ensure_kill_all():
>           th.docker_predict([1], tag=tag, binary=binary)

truss/tests/test_control_truss_patching.py:209: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
truss/truss_handle/decorators.py:7: in wrapper
    return func(*args, **kwargs)
truss/truss_handle/truss_handle.py:424: in docker_predict
    resp = TrussHandle._wait_for_predict(model_base_url, request, binary)
.venv/lib/python3.9/site-packages/tenacity/__init__.py:336: in wrapped_f
    return copy(f, *args, **kw)
.venv/lib/python3.9/site-packages/tenacity/__init__.py:475: in __call__
    do = self.iter(retry_state=retry_state)
.venv/lib/python3.9/site-packages/tenacity/__init__.py:376: in iter
    result = action(retry_state)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

rs = <RetryCallState 139836641876960: attempt #20; slept for 19.0; last result: returned <Response [503]>>

    def exc_check(rs: "RetryCallState") -> None:
        fut = t.cast(Future, rs.outcome)
        retry_exc = self.retry_error_cls(fut)
        if self.reraise:
            raise retry_exc.reraise()
>       raise retry_exc from fut.exception()
E       tenacity.RetryError: RetryError[<Future at 0x7f2e41519880 state=finished returned Response>]

.venv/lib/python3.9/site-packages/tenacity/__init__.py:419: RetryError

url = 'http://localhost:8090/v1/models/model'
container = python_on_whales.Container(id='5180a46bc1bb', name='exciting_hypatia')
wait_for_server_ready = True, model_server_stop_retry_override = None

    def wait_for_truss(
        url: str,
        container: str,
        wait_for_server_ready: bool = True,
        model_server_stop_retry_override=None,
    ) -> None:
        from python_on_whales.exceptions import NoSuchContainer
    
        try:
            _wait_for_docker_build(container)
            if wait_for_server_ready:
                if model_server_stop_retry_override is not None:
                    _wait_for_model_server(url, stop=model_server_stop_retry_override)
                else:
>                   _wait_for_model_server(url)

truss/truss_handle/truss_handle.py:1093: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

url = 'http://localhost:8090/v1/models/model'
stop = <tenacity.stop.stop_after_delay object at 0x7f2e46160820>

    def _wait_for_model_server(url: str, stop=stop_after_delay(120)) -> Response:  # type: ignore[return]
>       for attempt in Retrying(
            stop=stop,
            wait=wait_fixed(2),
            retry=(
                retry_if_result(lambda response: response.status_code in [502, 503])
                | retry_if_exception_type(exceptions.ConnectionError)
            ),
        ):

truss/truss_handle/truss_handle.py:1066: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Retrying object at 0x7f2e4159eb20 (stop=<tenacity.stop.stop_after_delay object at 0x7f2e46160820>, wait=<tenacity.wai...0x7f2e44323fd0>, before=<function before_nothing at 0x7f2e47c824c0>, after=<function after_nothing at 0x7f2e47c82700>)>

    def __iter__(self) -> t.Generator[AttemptManager, None, None]:
        self.begin()
    
        retry_state = RetryCallState(self, fn=None, args=(), kwargs={})
        while True:
>           do = self.iter(retry_state=retry_state)

.venv/lib/python3.9/site-packages/tenacity/__init__.py:443: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Retrying object at 0x7f2e4159eb20 (stop=<tenacity.stop.stop_after_delay object at 0x7f2e46160820>, wait=<tenacity.wai...0x7f2e44323fd0>, before=<function before_nothing at 0x7f2e47c824c0>, after=<function after_nothing at 0x7f2e47c82700>)>
retry_state = <RetryCallState 139836641636160: attempt #61; slept for 120.0; last result: failed (ConnectionError HTTPConnectionPool...ion.HTTPConnection object at 0x7f2e4159eb80>: Failed to establish a new connection: [Errno 111] Connection refused')))>

    def iter(self, retry_state: "RetryCallState") -> t.Union[DoAttempt, DoSleep, t.Any]:  # noqa
        self._begin_iter(retry_state)
        result = None
        for action in self.iter_state.actions:
>           result = action(retry_state)

.venv/lib/python3.9/site-packages/tenacity/__init__.py:376: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

rs = <RetryCallState 139836641636160: attempt #61; slept for 120.0; last result: failed (ConnectionError HTTPConnectionPool...ion.HTTPConnection object at 0x7f2e4159eb80>: Failed to establish a new connection: [Errno 111] Connection refused')))>

    def exc_check(rs: "RetryCallState") -> None:
        fut = t.cast(Future, rs.outcome)
        retry_exc = self.retry_error_cls(fut)
        if self.reraise:
            raise retry_exc.reraise()
>       raise retry_exc from fut.exception()
E       tenacity.RetryError: RetryError[<Future at 0x7f2e4159ec40 state=finished raised ConnectionError>]

.venv/lib/python3.9/site-packages/tenacity/__init__.py:419: RetryError

During handling of the above exception, another exception occurred:

config_python_version = 'py38', inspected_python_version = '3.8'

    @pytest.mark.integration
    @pytest.mark.parametrize(
        "config_python_version, inspected_python_version",
        [("py38", "3.8"), ("py39", "3.9"), ("py310", "3.10"), ("py311", "3.11")],
    )
    def test_predict_python_versions(config_python_version, inspected_python_version):
        model = """
        import sys
        class Model:
            def predict(self, data):
                version = sys.version_info
                return f"{version.major}.{version.minor}"
        """
    
        config = f"python_version: {config_python_version}"
    
        with ensure_kill_all(), _temp_truss(model, config) as tr:
>           _ = tr.docker_run(local_port=8090, detach=True, wait_for_server_ready=True)

truss/tests/test_model_inference.py:124: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
truss/truss_handle/decorators.py:7: in wrapper
    return func(*args, **kwargs)
truss/truss_handle/truss_handle.py:349: in docker_run
    raise err
truss/truss_handle/truss_handle.py:339: in docker_run
    wait_for_truss(
truss/truss_handle/truss_handle.py:1097: in wait_for_truss
    retry_err.reraise()
.venv/lib/python3.9/site-packages/tenacity/__init__.py:185: in reraise
    raise self.last_attempt.result()
/opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/concurrent/futures/_base.py:439: in result
    return self.__get_result()
/opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/concurrent/futures/_base.py:391: in __get_result
    raise self._exception
truss/truss_handle/truss_handle.py:1075: in _wait_for_model_server
    response = requests.get(url)
.venv/lib/python3.9/site-packages/requests/api.py:73: in get
    return request("get", url, params=params, **kwargs)
.venv/lib/python3.9/site-packages/requests/api.py:59: in request
    return session.request(method=method, url=url, **kwargs)
.venv/lib/python3.9/site-packages/requests/sessions.py:589: in request
    resp = self.send(prep, **send_kwargs)
.venv/lib/python3.9/site-packages/requests/sessions.py:703: in send
    r = adapter.send(request, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <requests.adapters.HTTPAdapter object at 0x7f2e4159ec10>
request = <PreparedRequest [GET]>, stream = False
timeout = Timeout(connect=None, read=None, total=None), verify = True
cert = None, proxies = OrderedDict()

    def send(
        self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
    ):
        """Sends PreparedRequest object. Returns Response object.
    
        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
        :param stream: (optional) Whether to stream the request content.
        :param timeout: (optional) How long to wait for the server to send
            data before giving up, as a float, or a :ref:`(connect timeout,
            read timeout) <timeouts>` tuple.
        :type timeout: float or tuple or urllib3 Timeout object
        :param verify: (optional) Either a boolean, in which case it controls whether
            we verify the server's TLS certificate, or a string, in which case it
            must be a path to a CA bundle to use
        :param cert: (optional) Any user-provided SSL certificate to be trusted.
        :param proxies: (optional) The proxies dictionary to apply to the request.
        :rtype: requests.Response
        """
    
        try:
            conn = self.get_connection_with_tls_context(
                request, verify, proxies=proxies, cert=cert
            )
        except LocationValueError as e:
            raise InvalidURL(e, request=request)
    
        self.cert_verify(conn, request.url, verify, cert)
        url = self.request_url(request, proxies)
        self.add_headers(
            request,
            stream=stream,
            timeout=timeout,
            verify=verify,
            cert=cert,
            proxies=proxies,
        )
    
        chunked = not (request.body is None or "Content-Length" in request.headers)
    
        if isinstance(timeout, tuple):
            try:
                connect, read = timeout
                timeout = TimeoutSauce(connect=connect, read=read)
            except ValueError:
                raise ValueError(
                    f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
                    f"or a single float to set both timeouts to the same value."
                )
        elif isinstance(timeout, TimeoutSauce):
            pass
        else:
            timeout = TimeoutSauce(connect=timeout, read=timeout)
    
        try:
            resp = conn.urlopen(
                method=request.method,
                url=url,
                body=request.body,
                headers=request.headers,
                redirect=False,
                assert_same_host=False,
                preload_content=False,
                decode_content=False,
                retries=self.max_retries,
                timeout=timeout,
                chunked=chunked,
            )
    
        except (ProtocolError, OSError) as err:
            raise ConnectionError(err, request=request)
    
        except MaxRetryError as e:
            if isinstance(e.reason, ConnectTimeoutError):
                # TODO: Remove this in 3.0.0: see #2811
                if not isinstance(e.reason, NewConnectionError):
                    raise ConnectTimeout(e, request=request)
    
            if isinstance(e.reason, ResponseError):
                raise RetryError(e, request=request)
    
            if isinstance(e.reason, _ProxyError):
                raise ProxyError(e, request=request)
    
            if isinstance(e.reason, _SSLError):
                # This branch is for urllib3 v1.22 and later.
                raise SSLError(e, request=request)
    
>           raise ConnectionError(e, request=request)
E           requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /v1/models/model (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f2e4159eb80>: Failed to establish a new connection: [Errno 111] Connection refused'))

.venv/lib/python3.9/site-packages/requests/adapters.py:700: ConnectionError