From fc1cc4fb9a3ccd1c46ab803f80c6ea31256d3d79 Mon Sep 17 00:00:00 2001 From: Jeremy Metz Date: Tue, 7 Nov 2023 20:53:36 +0100 Subject: [PATCH 1/4] Working cellpose-python test, stardist is WIP --- scripts/test-triton-models.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/scripts/test-triton-models.py b/scripts/test-triton-models.py index 4644656..53c9b34 100644 --- a/scripts/test-triton-models.py +++ b/scripts/test-triton-models.py @@ -1,7 +1,10 @@ import asyncio +import traceback +import imageio from imjoy_rpc.hypha import connect_to_server import requests + async def main(): server = await connect_to_server( {"name": "test client", "server_url": "https://hypha.bioimage.io/", "method_timeout": 30} @@ -29,4 +32,37 @@ async def main(): except: print("Model test failed: ", ret["result"]["rdf"]["name"], ret["result"]["rdf"].get("id")) + # One-off test of the cellpose model + image = imageio.v3.imread('https://static.imjoy.io/img/img02.png') + try: + ret = await triton.execute( + inputs=[image.astype('float32'), {'diameter': 30}], + model_name="cellpose-python", + decode_json=True + ) + print(ret.keys()) + print(ret) + mask = ret['mask'][0] + print("Model test passed: ", "Cellpose usinig cellpose-python") + except: + + print("Model test failed: ", "Cellpose using cellpose-python") + traceback.print_exc() + + # One-off test of the stardist model + image = imageio.v3.imread('https://static.imjoy.io/img/img02.png') + try: + ret = await triton.execute( + inputs=[image.astype('float32'), {'diameter': 30}], + model_name="stardist-python", + decode_json=True + ) + print(ret.keys()) + print(ret) + mask = ret['mask'][0] + print("Model test passed: ", "Stardist using stardist-python") + except: + print("Model test failed: ", "Cellpose using cellpose-python") + traceback.print_exc() + asyncio.run(main()) From 5411e83638881a230aeb1802f5574e6dccd0e340 Mon Sep 17 00:00:00 2001 From: Jeremy Metz Date: Fri, 10 Nov 2023 16:41:36 +0100 Subject: [PATCH 2/4] Added loop over servers in and working stardist test-triton-models.py --- scripts/test-triton-models.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/scripts/test-triton-models.py b/scripts/test-triton-models.py index 53c9b34..3ac282f 100644 --- a/scripts/test-triton-models.py +++ b/scripts/test-triton-models.py @@ -3,11 +3,24 @@ import imageio from imjoy_rpc.hypha import connect_to_server import requests +import numpy as np + + +SERVERS = [ + "https://ai.imjoy.io/", + "https://hypha.bioimage.io/", +] async def main(): + for url in SERVERS: + print(f"***** Testing {url} *****") + await test_server(url) + + +async def test_server(url): server = await connect_to_server( - {"name": "test client", "server_url": "https://hypha.bioimage.io/", "method_timeout": 30} + {"name": "test client", "server_url": url, "method_timeout": 30} ) triton = await server.get_service("triton-client") @@ -40,10 +53,8 @@ async def main(): model_name="cellpose-python", decode_json=True ) - print(ret.keys()) - print(ret) mask = ret['mask'][0] - print("Model test passed: ", "Cellpose usinig cellpose-python") + print("Model test passed: ", "Cellpose using cellpose-python") except: print("Model test failed: ", "Cellpose using cellpose-python") @@ -52,17 +63,19 @@ async def main(): # One-off test of the stardist model image = imageio.v3.imread('https://static.imjoy.io/img/img02.png') try: + # big_im = (255 * np.random.rand(1000,1000)).astype('uint16') + # nms_thresh = 100 + # prob_thresh = 0.5 ret = await triton.execute( - inputs=[image.astype('float32'), {'diameter': 30}], - model_name="stardist-python", + inputs=[image[...,0].astype('uint16'), {}], + # inputs=[big_im, {'nms_thresh' : nms_thresh, 'prob_thresh' : prob_thresh}], + model_name="stardist", decode_json=True ) - print(ret.keys()) - print(ret) mask = ret['mask'][0] - print("Model test passed: ", "Stardist using stardist-python") + print("Model test passed: ", "Stardist using stardist") except: - print("Model test failed: ", "Cellpose using cellpose-python") + print("Model test failed: ", "Stardist using stardist") traceback.print_exc() asyncio.run(main()) From c276b9edf7bd4babee3db8bd04b9c7111898b5f8 Mon Sep 17 00:00:00 2001 From: Jeremy Metz Date: Fri, 10 Nov 2023 18:24:01 +0100 Subject: [PATCH 3/4] Added meaningful assertions to cellpose and stardist --- scripts/test-triton-models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/test-triton-models.py b/scripts/test-triton-models.py index 3ac282f..d5d7fb1 100644 --- a/scripts/test-triton-models.py +++ b/scripts/test-triton-models.py @@ -47,13 +47,15 @@ async def test_server(url): # One-off test of the cellpose model image = imageio.v3.imread('https://static.imjoy.io/img/img02.png') + image = image.astype('float32') try: ret = await triton.execute( - inputs=[image.astype('float32'), {'diameter': 30}], + inputs=[image, {'diameter': 30}], model_name="cellpose-python", decode_json=True ) - mask = ret['mask'][0] + # NOTE: Input is RGB, Output is binary mask with leading singleton dimension + assert ret['mask'].shape[1:] == image.shape[:2], f"Mismatched shapes: {ret['mask'].shape} != {image.shape}" print("Model test passed: ", "Cellpose using cellpose-python") except: @@ -62,17 +64,18 @@ async def test_server(url): # One-off test of the stardist model image = imageio.v3.imread('https://static.imjoy.io/img/img02.png') + image = image[...,0].astype('uint16') try: # big_im = (255 * np.random.rand(1000,1000)).astype('uint16') # nms_thresh = 100 # prob_thresh = 0.5 ret = await triton.execute( - inputs=[image[...,0].astype('uint16'), {}], + inputs=[image, {}], # inputs=[big_im, {'nms_thresh' : nms_thresh, 'prob_thresh' : prob_thresh}], model_name="stardist", decode_json=True ) - mask = ret['mask'][0] + assert ret['mask'].shape == image.shape, f"Mismatched shapes: {ret['mask'].shape} != {image.shape}" print("Model test passed: ", "Stardist using stardist") except: print("Model test failed: ", "Stardist using stardist") From 656dd41e9fbd3504de690b30aabfea58fc82ef29 Mon Sep 17 00:00:00 2001 From: Jeremy Metz Date: Fri, 10 Nov 2023 19:42:44 +0100 Subject: [PATCH 4/4] Added slack broadcasting of errors --- scripts/test-triton-models.py | 89 +++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 19 deletions(-) diff --git a/scripts/test-triton-models.py b/scripts/test-triton-models.py index d5d7fb1..1b81155 100644 --- a/scripts/test-triton-models.py +++ b/scripts/test-triton-models.py @@ -1,26 +1,29 @@ import asyncio import traceback +import os +import sys + import imageio -from imjoy_rpc.hypha import connect_to_server import requests import numpy as np +from imjoy_rpc.hypha import connect_to_server SERVERS = [ - "https://ai.imjoy.io/", - "https://hypha.bioimage.io/", + "https://ai.imjoy.io/", + "https://hypha.bioimage.io/", ] async def main(): for url in SERVERS: print(f"***** Testing {url} *****") - await test_server(url) + errors = await test_server(url) -async def test_server(url): +async def test_server(server_url): server = await connect_to_server( - {"name": "test client", "server_url": url, "method_timeout": 30} + {"name": "test client", "server_url": server_url, "method_timeout": 30} ) triton = await server.get_service("triton-client") @@ -30,6 +33,7 @@ async def test_server(url): resp = requests.get(url=url) data = resp.json() collection = data["collection"] + errors = [] for model_info in collection: model_id = model_info["id"] @@ -38,33 +42,52 @@ async def test_server(url): ret = await triton.execute( inputs=[kwargs], model_name="bioengine-model-runner", - serialization="imjoy" + serialization="imjoy", ) assert "result" in ret and "rdf" in ret["result"] - print("Model test passed: ", ret["result"]["rdf"]["name"], ret["result"]["rdf"].get("id")) + print( + "Model test passed: ", + ret["result"]["rdf"]["name"], + ret["result"]["rdf"].get("id"), + ) except: - print("Model test failed: ", ret["result"]["rdf"]["name"], ret["result"]["rdf"].get("id")) + print( + "Model test failed: ", + ret["result"]["rdf"]["name"], + ret["result"]["rdf"].get("id"), + ) + errors.append( + " ".join( + "Model test failed: ", + ret["result"]["rdf"]["name"], + ret["result"]["rdf"].get("id"), + ) + ) + errors.append(traceback.format_exc()) # One-off test of the cellpose model - image = imageio.v3.imread('https://static.imjoy.io/img/img02.png') - image = image.astype('float32') + image = imageio.v3.imread("https://static.imjoy.io/img/img02.png") + image = image.astype("float32") try: ret = await triton.execute( - inputs=[image, {'diameter': 30}], + inputs=[image, {"diameter": 30}], model_name="cellpose-python", - decode_json=True + decode_json=True, ) # NOTE: Input is RGB, Output is binary mask with leading singleton dimension - assert ret['mask'].shape[1:] == image.shape[:2], f"Mismatched shapes: {ret['mask'].shape} != {image.shape}" + assert ( + ret["mask"].shape[1:] == image.shape[:1] + ), f"Mismatched shapes: {ret['mask'].shape} != {image.shape}" print("Model test passed: ", "Cellpose using cellpose-python") except: - print("Model test failed: ", "Cellpose using cellpose-python") traceback.print_exc() + errors.append("Model test failed: Cellpose using cellpose-python") + errors.append(traceback.format_exc()) # One-off test of the stardist model - image = imageio.v3.imread('https://static.imjoy.io/img/img02.png') - image = image[...,0].astype('uint16') + image = imageio.v3.imread("https://static.imjoy.io/img/img02.png") + image = image[..., 0].astype("uint16") try: # big_im = (255 * np.random.rand(1000,1000)).astype('uint16') # nms_thresh = 100 @@ -73,12 +96,40 @@ async def test_server(url): inputs=[image, {}], # inputs=[big_im, {'nms_thresh' : nms_thresh, 'prob_thresh' : prob_thresh}], model_name="stardist", - decode_json=True + decode_json=True, ) - assert ret['mask'].shape == image.shape, f"Mismatched shapes: {ret['mask'].shape} != {image.shape}" + assert ( + ret["mask"].shape == image.shape + ), f"Mismatched shapes: {ret['mask'].shape} != {image.shape}" print("Model test passed: ", "Stardist using stardist") except: print("Model test failed: ", "Stardist using stardist") traceback.print_exc() + errors.append("Model test failed: Stardist using stardist") + errors.append(traceback.format_exc()) + + if errors: + broadcast_errors(errors, server_url) + + +def broadcast_errors(errors, server_url): + try: + url = os.getenv("AICELL_LAB_SLACK_WEBHOOK_URL") + if url is None: + print("AICELL_LAB_SLACK_WEBHOOK_URL not set, can not send errors") + return + headers = {"Content-type": "application/json"} + requests.post( + url, + headers=headers, + json={"text": f"Error(s) occurred in test-triton-models.py on {server_url}:"}, + ) + for error in errors: + data = {"text": error} + requests.post(url, headers=headers, json=data) + requests.post(url, headers=headers, json={"text": "End of errors"}) + except: + print("Could not send error report to AiCell-Lab 😢") + traceback.print_exc() asyncio.run(main())