Merge pull request #130 from arturgontijo/gprc_error

[repo] gRPC context error and Daemon update.

.circleci/config.yml

@@ -15,7 +15,7 @@ jobs:
           service-dir: "services/cntk-image-recon"
           mainnet-port: "7004"
           testnet-port: "7054"
-          run-command: "source '/cntk/activate-cntk';python3 run_image_recon_service.py --ssl"
+          run-command: "source '/cntk/activate-cntk';python3 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/cntk-image-recon/Dockerfile"
           docker-no-cache: true
@@ -37,7 +37,7 @@ jobs:
           service-dir: "services/deepfakes-faceswap"
           mainnet-port: "7060"
           testnet-port: "7061-7062"
-          run-command: "python3 run_deepfakes_faceswap_service.py --ssl"
+          run-command: "python3 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/deepfakes-faceswap/Dockerfile"
           docker-no-cache: true
@@ -59,7 +59,7 @@ jobs:
           service-dir: "services/deoldify-colorizer"
           mainnet-port: "7084"
           testnet-port: "7085"
-          run-command: "python3 run_colorize_service.py --ssl"
+          run-command: "python3 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/deoldify-colorizer/Dockerfile"
           docker-no-cache: true
@@ -81,7 +81,7 @@ jobs:
           service-dir: "services/i3d-video-action-recognition"
           mainnet-port: "7005"
           testnet-port: "7055"
-          run-command: "python3 run_video_action_recon_service.py --ssl"
+          run-command: "python3 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/i3d-video-action-recognition/Dockerfile"
           docker-no-cache: true
@@ -103,7 +103,7 @@ jobs:
           service-dir: "services/places365-scene-recognition"
           mainnet-port: "7019"
           testnet-port: "7079"
-          run-command: "python3 run_scene_recognition_service.py --ssl"
+          run-command: "python3 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/places365-scene-recognition/Dockerfile"
           docker-no-cache: true
@@ -125,7 +125,7 @@ jobs:
           service-dir: "services/pneumonia-diagnosis"
           mainnet-port: "7072"
           testnet-port: "7062"
-          run-command: "python3 run_pneumonia_diagnosis_service.py --ssl"
+          run-command: "python3 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/pneumonia-diagnosis/Dockerfile"
           docker-no-cache: true
@@ -147,7 +147,7 @@ jobs:
           service-dir: "services/real-time-voice-cloning"
           mainnet-port: "7064"
           testnet-port: "7065"
-          run-command: "python3 run_voice_cloning_service.py --ssl"
+          run-command: "python3 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/real-time-voice-cloning/Dockerfile"
           docker-no-cache: true
@@ -169,7 +169,7 @@ jobs:
           service-dir: "services/s2vt-video-captioning"
           mainnet-port: "7006"
           testnet-port: "7056"
-          run-command: "python run_video_cap_service.py --ssl"
+          run-command: "python run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/s2vt-video-captioning/Dockerfile"
           docker-no-cache: true
@@ -191,7 +191,7 @@ jobs:
           service-dir: "services/sound-spleeter"
           mainnet-port: "7066"
           testnet-port: "7067"
-          run-command: "python3 run_sound_spleeter_service.py --ssl"
+          run-command: "python3 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/sound-spleeter/Dockerfile"
           docker-no-cache: true
@@ -213,7 +213,7 @@ jobs:
           service-dir: "services/yolov3-object-detection"
           mainnet-port: "7007"
           testnet-port: "7057"
-          run-command: "python3 run_object_detection_service.py --ssl"
+          run-command: "python3 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/yolov3-object-detection/Dockerfile"
           docker-no-cache: true
@@ -235,7 +235,7 @@ jobs:
           service-dir: "services/zeta36-chess-alpha-zero"
           mainnet-port: "7008"
           testnet-port: "7058"
-          run-command: "python3.6 run_alpha_zero_service.py --ssl"
+          run-command: "python3.6 run_service.py --ssl"
           docker-test-files: "Dockerfile, requirements.txt"
           docker-file-path: "services/zeta36-chess-alpha-zero/Dockerfile"
           docker-no-cache: true
@@ -254,47 +254,47 @@ jobs:
         - ssh-deployment/test:
               ssh-host: ${SSH_HOST_CNTK_IMAGE_RECON}
               ssh-user: ${SSH_USER_CNTK_IMAGE_RECON}
-              test-command: "sleep 10;source '/cntk/activate-cntk';python3 test_image_recon_service.py auto"
+              test-command: "sleep 10;source '/cntk/activate-cntk';python3 test_service.py auto"
               docker-container: "CNTK_IMAGE_RECON"
         - ssh-deployment/test:
               ssh-host: ${SSH_HOST_DEOLDIFY_COLORIZER}
               ssh-user: ${SSH_USER_DEOLDIFY_COLORIZER}
-              test-command: "sleep 10;python3 test_colorize_service.py auto"
+              test-command: "sleep 10;python3 test_service.py auto"
               docker-container: "DEOLDIFY_COLORIZER"
         - ssh-deployment/test:
               ssh-host: ${SSH_HOST_I3D_VIDEO_ACTION_RECON}
               ssh-user: ${SSH_USER_I3D_VIDEO_ACTION_RECON}
-              test-command: "sleep 10;python3 test_video_action_recon_service.py auto"
+              test-command: "sleep 10;python3 test_service.py auto"
               docker-container: "I3D_VIDEO_ACTION_RECON"
         - ssh-deployment/test:
               ssh-host: ${SSH_HOST_PLACES365_SCENE_RECOGNITION}
               ssh-user: ${SSH_USER_PLACES365_SCENE_RECOGNITION}
-              test-command: "sleep 10;python3 test_scene_recognition_service.py auto"
+              test-command: "sleep 10;python3 test_service.py auto"
               docker-container: "PLACES365_SCENE_RECOGNITION"
         - ssh-deployment/test:
               ssh-host: ${SSH_HOST_VOICE_CLONING}
               ssh-user: ${SSH_USER_VOICE_CLONING}
-              test-command: "sleep 10;python3 test_voice_cloning_service.py auto"
+              test-command: "sleep 10;python3 test_service.py auto"
               docker-container: "VOICE_CLONING"
         - ssh-deployment/test:
               ssh-host: ${SSH_HOST_S2VT_VIDEO_CAPTIONING}
               ssh-user: ${SSH_USER_S2VT_VIDEO_CAPTIONING}
-              test-command: "sleep 10;python test_video_cap_service.py auto"
+              test-command: "sleep 10;python test_service.py auto"
               docker-container: "S2VT_VIDEO_CAPTIONING"
         - ssh-deployment/test:
               ssh-host: ${SSH_HOST_SOUND_SPLEETER}
               ssh-user: ${SSH_USER_SOUND_SPLEETER}
-              test-command: "sleep 10;python3 test_sound_spleeter_service.py auto"
+              test-command: "sleep 10;python3 test_service.py auto"
               docker-container: "SOUND_SPLEETER"
         - ssh-deployment/test:
               ssh-host: ${SSH_HOST_YOLOv3_OBJECT_DETECTION}
               ssh-user: ${SSH_USER_YOLOv3_OBJECT_DETECTION}
-              test-command: "sleep 10;python3 test_object_detection_service.py auto"
+              test-command: "sleep 10;python3 test_service.py auto"
               docker-container: "YOLOv3_OBJECT_DETECTION"
         - ssh-deployment/test:
               ssh-host: ${SSH_HOST_CHESS_ALPHA_ZERO}
               ssh-user: ${SSH_USER_CHESS_ALPHA_ZERO}
-              test-command: "sleep 10;python3.6 test_alpha_zero_service.py auto"
+              test-command: "sleep 10;python3.6 test_service.py auto"
               docker-container: "CHESS_ALPHA_ZERO"
 
 workflows:

services/cntk-image-recon/Dockerfile

@@ -20,7 +20,7 @@ RUN apt-get update && \
     curl
 
 # Installing SNET Daemon
-RUN SNETD_GIT_VERSION=`curl -s https://api.github.com/repos/singnet/snet-daemon/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")' || echo "v3.1.3"` && \
+RUN SNETD_GIT_VERSION=`curl -s https://api.github.com/repos/singnet/snet-daemon/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")' || echo "v3.1.6"` && \
     SNETD_VERSION=${snetd_version:-${SNETD_GIT_VERSION}} && \
     cd /tmp && \
     wget https://github.com/singnet/snet-daemon/releases/download/${SNETD_VERSION}/snet-daemon-${SNETD_VERSION}-linux-amd64.tar.gz && \

services/cntk-image-recon/README.md

@@ -45,7 +45,6 @@ Create the `SNET Daemon`'s config JSON file (`snetd.config.json`).
    "DAEMON_END_POINT": "DAEMON_HOST:DAEMON_PORT",
    "IPFS_END_POINT": "http://ipfs.singularitynet.io:80",
    "BLOCKCHAIN_NETWORK_SELECTED": "BLOCKCHAIN_NETWORK",
-   "PASSTHROUGH_ENABLED": true,
    "PASSTHROUGH_ENDPOINT": "http://SERVICE_GRPC_HOST:SERVICE_GRPC_PORT",  
    "ORGANIZATION_ID": "ORGANIZATION_ID",
    "SERVICE_ID": "SERVICE_ID",
@@ -66,7 +65,6 @@ $ cat snetd.config.json
    "DAEMON_END_POINT": "0.0.0.0:7054",
    "IPFS_END_POINT": "http://ipfs.singularitynet.io:80",
    "BLOCKCHAIN_NETWORK_SELECTED": "ropsten",
-   "PASSTHROUGH_ENABLED": true,
    "PASSTHROUGH_ENDPOINT": "http://localhost:7003",
    "ORGANIZATION_ID": "snet",
    "SERVICE_ID": "cntk-image-recon",
@@ -91,7 +89,7 @@ $ sh buildproto.sh
 ```
 Start the service and `SNET Daemon`:
 ```
-$ python3 run_image_recon_service.py
+$ python3 run_service.py
 ```
 
 ### Calling the service:
@@ -104,15 +102,15 @@ Inputs:
 Local (testing purpose):
 
 ```
-$ python3 test_image_recon_service.py 
+$ python3 test_service.py 
 Endpoint (localhost:7003): 
 Method (flowers|dogs): flowers
 Model (ResNet152): <ENTER>
 Image (Link): https://www.fiftyflowers.com/site_files/FiftyFlowers/Image/Product/Mini-Black-Eye-bloom-350_c7d02e72.jpg
 1.8751
 {1: '98.93%: sunflower', 2: '00.64%: black-eyed susan', 3: '00.16%: barbeton daisy', 4: '00.14%: oxeye daisy', 5: '00.03%: daffodil'}
 
-$ python3 test_image_recon_service.py 
+$ python3 test_service.py 
 Endpoint (localhost:7003): 
 Method (flowers|dogs): dogs
 Model (ResNet152): <ENTER>

services/cntk-image-recon/requirements.txt

@@ -1,5 +1,5 @@
 grpcio>=1.14.2
 grpcio-tools>=1.14.1
 requests==2.20.0
-Pillow==6.2.0
-cntk==2.5
+Pillow==6.2.2
+cntk-gpu==2.7

services/cntk-image-recon/service/image_recon.py

@@ -87,7 +87,7 @@ def image_recognition(method, model, map_names, img_path, image_dims):
     except Exception as e:
         log.error(e)
         traceback.print_exc()
-        return {"delta_time": "Fail", "top_5": "Fail"}
+        return {"delta_time": "Fail", "top_5": "Fail", "error": str(e)}
 
 
 def generate_uid():

services/cntk-image-recon/service/image_recon_service.py

@@ -38,7 +38,7 @@ def __init__(self):
     # The method that will be exposed to the snet-cli call command.
     # request: incoming data
     # context: object that provides RPC-specific information (timeout, etc).
-    def flowers(self, request, _):
+    def flowers(self, request, context):
         manager = multiprocessing.Manager()
         return_dict = manager.dict()
         p = multiprocessing.Process(target=mp_image_recognition, args=("flowers",
@@ -50,13 +50,17 @@ def flowers(self, request, _):
         p.join()
 
         response = return_dict.get("response", None)
-        if not response:
-            return Output(delta_time="Fail", top_5="Fail")
+        if not response or "error" in response:
+            error_msg = response.get("error", None) if response else None
+            log.error(error_msg)
+            context.set_details(error_msg)
+            context.set_code(grpc.StatusCode.INTERNAL)
+            return Output()
 
         log.debug("flowers({},{})=OK".format(self.model, self.img_path))
         return Output(delta_time=response["delta_time"], top_5=str(response["top_5"]))
 
-    def dogs(self, request, _):
+    def dogs(self, request, context):
         manager = multiprocessing.Manager()
         return_dict = manager.dict()
         p = multiprocessing.Process(target=mp_image_recognition, args=("dogs",
@@ -68,7 +72,11 @@ def dogs(self, request, _):
         p.join()
 
         response = return_dict.get("response", None)
-        if not response:
+        if not response or "error" in response:
+            error_msg = response.get("error", "None") if response else "None"
+            log.error(error_msg)
+            context.set_details(error_msg)
+            context.set_code(grpc.StatusCode.INTERNAL)
             return Output(delta_time="Fail", top_5="Fail")
 
         log.debug("flowers({},{})=OK".format(self.model, self.img_path))

services/cntk-image-recon/snetd_configs/snetd.mainnet.json

@@ -1,7 +1,6 @@
 {
   "daemon_end_point": "0.0.0.0:7004",
   "ipfs_end_point": "http://ipfs.singularitynet.io:80",
-  "passthrough_enabled": true,
   "passthrough_endpoint": "http://localhost:7003",
   "blockchain_network_selected": "main",
   "organization_id": "snet",

services/cntk-image-recon/snetd_configs/snetd.ropsten

@@ -1,7 +1,6 @@
 {
   "daemon_end_point": "0.0.0.0:7054",
   "ipfs_end_point": "http://ipfs.singularitynet.io:80",
-  "passthrough_enabled": true,
   "passthrough_endpoint": "http://localhost:7003",
   "blockchain_network_selected": "ropsten",
   "organization_id": "snet",

services/deepfakes-faceswap/Dockerfile

@@ -25,7 +25,7 @@ RUN apt-get install -y libglib2.0-0 libsm6 libxext6 libxrender-dev
 RUN apt-get install -y python3 python3-pip ffmpeg
 
 # Installing SNET Daemon
-RUN SNETD_GIT_VERSION=`curl -s https://api.github.com/repos/singnet/snet-daemon/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")' || echo "v3.1.3"` && \
+RUN SNETD_GIT_VERSION=`curl -s https://api.github.com/repos/singnet/snet-daemon/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")' || echo "v3.1.6"` && \
     SNETD_VERSION=${snetd_version:-${SNETD_GIT_VERSION}} && \
     cd /tmp && \
     wget https://github.com/singnet/snet-daemon/releases/download/${SNETD_VERSION}/snet-daemon-${SNETD_VERSION}-linux-amd64.tar.gz && \

services/deepfakes-faceswap/README.md

@@ -36,7 +36,6 @@ Create the `SNET Daemon`'s config JSON file (`snetd.config.json`).
    "DAEMON_END_POINT": "DAEMON_HOST:DAEMON_PORT",
    "IPFS_END_POINT": "http://ipfs.singularitynet.io:80",
    "BLOCKCHAIN_NETWORK_SELECTED": "BLOCKCHAIN_NETWORK",
-   "PASSTHROUGH_ENABLED": true,
    "PASSTHROUGH_ENDPOINT": "http://SERVICE_GRPC_HOST:SERVICE_GRPC_PORT",  
    "ORGANIZATION_ID": "ORGANIZATION_ID",
    "SERVICE_ID": "SERVICE_ID",
@@ -57,7 +56,6 @@ $ cat snetd.config.json
    "DAEMON_END_POINT": "0.0.0.0:7005",
    "IPFS_END_POINT": "http://ipfs.singularitynet.io:80",
    "BLOCKCHAIN_NETWORK_SELECTED": "ropsten",
-   "PASSTHROUGH_ENABLED": true,
    "PASSTHROUGH_ENDPOINT": "http://localhost:7003",
    "ORGANIZATION_ID": "snet",
    "SERVICE_ID": "deepfakes-faceswap",
@@ -82,7 +80,7 @@ $ sh buildproto.sh
 ```
 Start the service and `SNET Daemon`:
 ```
-$ python3 run_deepfakes_faceswap_service.py
+$ python3 run_service.py
 ```
 
 ### Calling the service:
@@ -96,7 +94,7 @@ Inputs:
 Local (testing purpose):
 
 ```
-$ python3 test_deepfakes_faceswap_service.py
+$ python3 test_service.py
 Endpoint (localhost:7003): 
 Method (faceswap): 
 UID:
@@ -108,7 +106,7 @@ uid: "http://52.38.111.172:7006/dashboard?uid=705493e1b8594380e247"
 ```
 
 ```
-$ python3 test_deepfakes_faceswap_service.py
+$ python3 test_service.py
 Endpoint (localhost:7003): 
 Method (faceswap): 
 UID: 705493e1b8594380e247

services/deepfakes-faceswap/snetd_configs/snetd.mainnet.json

@@ -1,7 +1,6 @@
 {
   "daemon_end_point": "0.0.0.0:7060",
   "ipfs_end_point": "http://ipfs.singularitynet.io:80",
-  "passthrough_enabled": true,
   "passthrough_endpoint": "http://localhost:7003",
   "blockchain_network_selected": "main",
   "organization_id": "snet",

services/deepfakes-faceswap/snetd_configs/snetd.ropsten

@@ -1,7 +1,6 @@
 {
   "daemon_end_point": "0.0.0.0:7061",
   "ipfs_end_point": "http://ipfs.singularitynet.io:80",
-  "passthrough_enabled": true,
   "passthrough_endpoint": "http://localhost:7003",
   "blockchain_network_selected": "ropsten",
   "organization_id": "snet",

services/deoldify-colorizer/README.md

@@ -45,7 +45,6 @@ Create the `SNET Daemon`'s config JSON file (`snetd.config.json`).
    "DAEMON_END_POINT": "DAEMON_HOST:DAEMON_PORT",
    "IPFS_END_POINT": "http://ipfs.singularitynet.io:80",
    "BLOCKCHAIN_NETWORK_SELECTED": "BLOCKCHAIN_NETWORK",
-   "PASSTHROUGH_ENABLED": true,
    "PASSTHROUGH_ENDPOINT": "http://SERVICE_GRPC_HOST:SERVICE_GRPC_PORT",  
    "ORGANIZATION_ID": "ORGANIZATION_ID",
    "SERVICE_ID": "SERVICE_ID",
@@ -66,7 +65,6 @@ $ cat snetd.config.json
    "DAEMON_END_POINT": "0.0.0.0:7085",
    "IPFS_END_POINT": "http://ipfs.singularitynet.io:80",
    "BLOCKCHAIN_NETWORK_SELECTED": "ropsten",
-   "PASSTHROUGH_ENABLED": true,
    "PASSTHROUGH_ENDPOINT": "http://localhost:7003",
    "ORGANIZATION_ID": "snet",
    "SERVICE_ID": "deoldify-colorizer",
@@ -91,7 +89,7 @@ $ sh buildproto.sh
 ```
 Start the service and `SNET Daemon`:
 ```
-$ python3 run_colorize_service.py
+$ python3 run_service.py
 ```
 
 ### Calling the service:
@@ -103,7 +101,7 @@ Inputs:
 Local (testing purpose):
 
 ```
-$ python3 test_colorize_service.py 
+$ python3 test_service.py 
 Endpoint (localhost:7003): 
 Method (colorize): colorize
 Image (Link): https://snet-models.s3.amazonaws.com/bh/Colorize/jucelino.jpg

services/deoldify-colorizer/service/colorization.py

@@ -87,5 +87,5 @@ def colorize(self):
         except Exception as e:
             log.error(e)
             traceback.print_exc()
-            self.response["img_colorized"] = "Fail"
+            self.response = {"img_colorized": "Fail", "error": str(e)}
             return self.response