Merge pull request #1 from belajarqywok/development

Update: CI/CD Pipeline, Unit testing, and Load testing using Locust

.github/workflows/development.yaml

@@ -0,0 +1,122 @@
+name: Development Pipeline
+
+on:
+  push:
+    branches:
+      - development
+    tags:
+      - '*'
+
+permissions:
+  contents: read
+
+env:
+  host: "172.17.0.1"
+
+jobs:
+  # Unit Testing
+  unit_testing:
+    name: Unit Testing
+    runs-on: ubuntu-latest
+    environment: testing
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Unit Testing
+      run: |
+        pip3 install nose2 nose2[coverage_plugin] scikit-learn \
+          numpy keras tensorflow keras_preprocessing
+
+        nose2 --start-dir tests \
+          --verbose \
+          --pretty-assert \
+          --with-coverage
+
+  # Performance and Load Testing
+  performance_and_load_testing:
+    name: Performance and Load Testing
+    runs-on: ubuntu-latest
+    environment: testing
+    needs: unit_testing
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Build Docker Image
+      run: |
+        docker build --tag ml_service:1.0 \
+          --file deployment/development.dockerfile .
+
+    - name: Deploy Database (PostgreSQL) and Service
+      run: |
+        docker container create \
+          --name pgserver \
+          -p 5432:5432 \
+          -e POSTGRES_USER=postgres \
+          -e POSTGRES_PASSWORD=admin \
+          -e POSTGRES_DB=coba_capstone \
+          postgres:11
+
+        docker container create --name ml_service \
+          -p 80:80 \
+          -e VERSION=v1 \
+          -e JWT_ACCESS_TOKEN_SECRET=AHBiadbiaeud92uedb9ub9wue92 \
+          -e JWT_REFRESH_TOKEN_SECRET=eu92uidbsAHBiadbiauedb9ub9wue92eu \
+          -e JWT_ALGORITHM=HS512 \
+          -e JWT_ACCESS_TOKEN_EXPIRE=1 \
+          -e JWT_REFRESH_TOKEN_EXPIRE=7 \
+          -e POSTGRES_HOST=$host \
+          -e POSTGRES_PORT=5432 \
+          -e POSTGRES_USER=postgres \
+          -e POSTGRES_PASS=admin \
+          -e POSTGRES_DB=coba_capstone \
+          ml_service:1.0
+
+        docker container start pgserver
+        sleep 5
+        docker container start ml_service
+        sleep 5
+        docker container ls
+        sleep 5
+
+    - name: liveness checking
+      run: |
+        while true; do
+          if docker logs ml_service 2>&1 | grep -q "Application startup complete."; then
+            echo "Application startup is complete. Stopping the loop."
+            break
+          fi
+          sleep 5
+          clear
+        done
+        curl http://$host/liveness_check
+
+    - name: Performance & Load Testing
+      run: |
+        pip3 install locust
+        locust --headless \
+          --run-time 60s \
+          --users 400 \
+          --spawn-rate 50 \
+          -H http://$host \
+          --locustfile locust/configuration.py
+
+  # Vulnerability Scanning
+  vulnerability_scanning:
+    name: Vulnerability Scanning
+    runs-on: ubuntu-latest
+    environment: testing
+    needs: performance_and_load_testing
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Vulnerability Scanning
+      run: |
+        pip3 install pip-audit
+        pip-audit --requirement ./linux.requirements.txt
+    

Makefile

@@ -14,4 +14,6 @@ postgres:
 jaeger-test:
 	uvicorn --host 0.0.0.0 --port 3000 --reload --workers 10 jaeger_test:app
 
+load_test:
+	locust --headless --run-time 10s --users 100 --spawn-rate 10 -H http://192.168.137.1:3000 --locustfile locust/configuration.py
 

locust/configuration.py

@@ -0,0 +1,11 @@
+from locust import HttpUser, task
+
+class main_routes(HttpUser):
+    @task
+    def main_route(self):
+        self.client.get("/")
+
+class ml_routes(HttpUser):
+    @task
+    def liveness_check_route(self):
+        self.client.get("/liveness_check")

src/services/ml_services/nlp_service.py

@@ -26,4 +26,5 @@ def predict_question(self, question: str) -> str:
         # Inverse Transform Label Encoder
         return self.__NLP_UTILS.label_encoder.inverse_transform(
             [argmax(predict)]
-        )[0]
+        )[0]
+

src/utilities/nlp_utils.py

@@ -36,7 +36,6 @@ def clean_stopwords(self, sentence: str) -> str:
 
         return temp.strip()
 
-
     # Model
     @property
     def model(self) -> Any:

tests/test_ml_services.py

@@ -0,0 +1,16 @@
+import unittest
+from src.services.ml_services.nlp_service import nlp_service
+
+class ml_service_tests(unittest.TestCase):
+    # NLP Service Test
+    def test_nlp_service(self) -> None:
+        # Question
+        question: str = "mental"
+
+        # NLP Service
+        service = nlp_service()
+
+        # Answer
+        answer: str = service.predict_question(question)
+
+        self.assertTrue(question in answer.lower())