traefik reverse proxy service added

ilkersigirci · Jan 1, 2025 · c8968f2 · c8968f2
1 parent 3bfc359
commit c8968f2
Show file tree

Hide file tree

Showing 10 changed files with 168 additions and 68 deletions.
diff --git a/.env.example b/.env.example
@@ -1,22 +1,21 @@
 CHAINLIT_APP_ROOT=DUMMY_PATH/configs/chainlit
 CHAINLIT_AUTH_SECRET=your-secret-here
-CROSS_ENCODER_HOST=http://localhost:8001
-EMBEDDING_HOST=http://localhost:8002
+EMBEDDING_HOST=http://hf_embedding.localhost
 EMBEDDING_MODEL_NAME=your-embedding-model
 ENABLE_OPENAI_API=false
 ENABLE_STARTER_QUESTIONS=true
 HF_TOKEN=your-hf-token
-LANGFUSE_HOST=http://localhost:3000
+LANGFUSE_HOST=http://langfuse.localhost
 LANGFUSE_PUBLIC_KEY=your-public-key
 LANGFUSE_SECRET_KEY=your-secret-key
 LIBRARY_BASE_PATH=DUMMY_PATH
-MODEL_API_BASE=http://localhost:8000
+MODEL_API_BASE=http://llamacpp.localhost
 MODEL_NAME=qwen2-0_5b-instruct-fp16.gguf
 # OPENAI_API_KEY=None
 SQLALCHEMY_DB_TYPE=sqlite
 TIMEOUT_LIMIT=30
-WHISPER_API_URL=http://localhost:8003
+WHISPER_API_BASE=http://whisper.localhost
 WHISPER_MODEL_NAME=Systran/faster-distil-whisper-large-v3
 
 ##### DEPLOYMENT #####
-WHISPER_API_PORT=8003
+DOMAIN_NAME=localhost
diff --git a/Makefile b/Makefile
@@ -204,6 +204,9 @@ docker-build: ## Build docker image
 init-db: ## Initialize the database
 	uv run src/podflix/db/init_db.py
 
+create-ssl-cert: ## Create a self-signed SSL certificate for localhost development
+	bash scripts/create_ssl_cert.sh
+
 download-hf-model: ## Download the huggingface model
 	uv run src/podflix/utils/hf_related.py
 

diff --git a/README.md b/README.md
@@ -30,13 +30,15 @@ POSTGRES_USER=your_username
 
 ## Healthchecks
 
+- Assuming `DOMAIN_NAME=localhost`
+
 ### Openai like model api
 
 - Request with system message assuming `MODEL_NAME=qwen2-0_5b-instruct-fp16.gguf`
 
 ```bash
 curl --request POST \
-    --url http://0.0.0.0:8000/v1/chat/completions \
+    --url https://llamacpp.localhost/v1/chat/completions \
     --header "Content-Type: application/json" \
     --data '{
   "model": "qwen2-0_5b-instruct-fp16.gguf",
@@ -58,7 +60,7 @@ curl --request POST \
 
 ```bash
 curl --request POST \
- --url http://0.0.0.0:8000/v1/chat/completions \
+ --url https://llamacpp.localhost/v1/chat/completions \
     --header "Content-Type: application/json" \
     --data '{
   "model": "qwen2-0_5b-instruct-fp16.gguf",

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,6 +1,8 @@
 name: podflix
 
 networks:
+  t2_proxy:
+    external: true
   podflix-network:
     name: podflix-network
     driver: bridge
@@ -18,6 +20,60 @@ x-deploy: &gpu-deploy
             capabilities: [gpu]
 
 services:
+  ########## TRAEFIK ##########
+  traefik:
+    image: traefik:v3.2.3
+    container_name: traefik
+    restart: ${RESTART_POLICY:-always}
+    security_opt:
+      - no-new-privileges:true
+    command:
+      - --api.dashboard=true
+      - --api.insecure=true
+      - --providers.docker=true
+      - --entrypoints.web.address=:80
+      - --entrypoints.websecure.address=:443
+      - --entryPoints.traefik.address=:8080
+      # Certificates related
+      - --providers.docker.exposedbydefault=false
+      - --entrypoints.websecure.http.tls=true
+      - --entrypoints.websecure.http.tls.certificates.certFile=/etc/certs/local-dev.crt
+      - --entrypoints.websecure.http.tls.certificates.keyFile=/etc/certs/local-dev.key
+    networks:
+      - podflix-network
+    ports:
+      - "8008:80"
+      - "4443:443"
+      - "5080:8080" # Traefik dashboard
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - "./deployment/certs:/etc/certs:ro"
+    healthcheck:
+      test: "traefik healthcheck --ping"
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    labels:
+      - "traefik.enable=true"
+      # HTTP-to-HTTPS Redirect
+      - "traefik.http.routers.http-catchall.entrypoints=http"
+      - "traefik.http.routers.http-catchall.rule=HostRegexp(`{host:.+}`)"
+      - "traefik.http.routers.http-catchall.middlewares=redirect-to-https"
+      - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
+      # HTTP Routers
+      - "traefik.http.routers.traefik-rtr.entrypoints=https"
+      - "traefik.http.routers.traefik-rtr.rule=Host(`traefik.$DOMAIN_NAME`)"
+      - "traefik.http.routers.traefik-rtr.tls=true" # Some people had 404s without this
+      - "traefik.http.routers.traefik-rtr.tls.domains[0].main=$DOMAIN_NAME"
+      - "traefik.http.routers.traefik-rtr.tls.domains[0].sans=*.$DOMAIN_NAME"
+      ## Services - API
+      - "traefik.http.routers.traefik-rtr.service=api@internal"
+      ## Healthcheck/ping
+      # - "traefik.http.routers.ping.rule=Host(`traefik.$DOMAIN_NAME`) && Path(`/ping`)"
+      #- "traefik.http.routers.ping.tls=true"
+      #- "traefik.http.routers.ping.service=ping@internal"
+
+  ########## PODFLIX ##########
   podflix-dev:
     image: podflix-dev:latest
     container_name: podflix-dev
@@ -26,9 +82,8 @@ services:
       dockerfile: docker/Dockerfile
       target: development
     networks:
+      - t2_proxy
       - podflix-network
-    # ports:
-    #   - 8080:8080
     command: ["tail", "-f", "/dev/null"] # NOTE: For testing the container
     restart: "no"
     develop:
@@ -53,9 +108,8 @@ services:
       dockerfile: docker/Dockerfile
       target: production
     networks:
+      - t2_proxy
       - podflix-network
-    # ports:
-    #   - 8080:8080
     command: ["tail", "-f", "/dev/null"] # NOTE: For testing the container
     restart: "no"
 
@@ -65,6 +119,7 @@ services:
     container_name: langfuse-db
     restart: ${RESTART_POLICY:-unless-stopped}
     networks:
+      - t2_proxy
       - podflix-network
     environment:
       - POSTGRES_USER=postgres
@@ -84,12 +139,13 @@ services:
     container_name: langfuse-server
     restart: ${RESTART_POLICY:-unless-stopped}
     depends_on:
+      # podflix-traefik:
+      #   condition: service_healthy
       langfuse-db:
         condition: service_healthy
     networks:
+      - t2_proxy
       - podflix-network
-    ports:
-      - ${LANGFUSE_PORT:-3000}:3000
     environment:
       - DATABASE_URL=postgresql://postgres:postgres@langfuse-db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
@@ -104,23 +160,14 @@ services:
       # - LANGFUSE_DEFAULT_PROJECT_ROLE=ADMIN
       # - AUTH_DISABLE_SIGNUP=true
       # - AUTH_DISABLE_USERNAME_PASSWORD=true
-
-  ########## TRAEFIK ##########
-  podflix-traefik:
-    image: traefik:v3.1.4
-    container_name: podflix-traefik
-    restart: ${RESTART_POLICY:-always}
-    command:
-      - "--api.insecure=true"
-      - "--providers.docker=true"
-      - "--entrypoints.web.address=:80"
-    ports:
-      - "8008:80"
-      - "5001:8080" # Traefik dashboard
-    volumes:
-      - "/var/run/docker.sock:/var/run/docker.sock:ro"
-    networks:
-      - podflix-network
+    labels:
+      - "traefik.enable=true"
+      ## HTTP Routers
+      - "traefik.http.routers.langfuse-rtr.entrypoints=https"
+      - "traefik.http.routers.langfuse-rtr.rule=Host(`langfuse.$DOMAIN_NAME`)"
+      ## HTTP Services
+      - "traefik.http.routers.langfuse-rtr.service=langfuse-svc"
+      - "traefik.http.services.langfuse-svc.loadbalancer.server.port=3000"
 
   ############ WHISPER API #############
   faster-whisper-server:
@@ -129,9 +176,8 @@ services:
     restart: "no"
     <<: *gpu-deploy
     networks:
+      - t2_proxy
       - podflix-network
-    ports:
-      - $WHISPER_API_PORT:8000
     environment:
       - HOST=0.0.0.0
       - PORT=8000
@@ -145,6 +191,14 @@ services:
       # - WHISPER__COMPUTE_TYPE=bfloat16
     volumes:
       - ./deployment/volumes/huggingface:/root/.cache/huggingface
+    labels:
+      - "traefik.enable=true"
+      ## HTTP Routers
+      - "traefik.http.routers.whisper-rtr.entrypoints=https"
+      - "traefik.http.routers.whisper-rtr.rule=Host(`whisper.$DOMAIN_NAME`)"
+      ## HTTP Services
+      - "traefik.http.routers.whisper-rtr.service=whisper-svc"
+      - "traefik.http.services.whisper-svc.loadbalancer.server.port=8000"
 
   # whisper-cpp:
   #   container_name: whisper-cpp
@@ -156,8 +210,6 @@ services:
   #   # command: ["./server", "--host", "0.0.0.0", "--port", "80"]
   #   networks:
   #     - podflix-network
-  #   ports:
-  #     - $WHISPER_API_PORT:80
   #   volumes:
   #     - ./deployment/models/whisperfile:/models
   #     - ./notebooks/resources:/resources
@@ -171,9 +223,8 @@ services:
     restart: ${RESTART_POLICY:-no}
     <<: *gpu-deploy
     networks:
+      - t2_proxy
       - podflix-network
-    ports:
-      - ${MODEL_API_PORT:-8000}:8000
     environment:
       LLAMA_ARG_HOST: 0.0.0.0
       LLAMA_ARG_PORT: 8000
@@ -193,6 +244,14 @@ services:
       interval: 10s
       timeout: 5s
       retries: 3
+    labels:
+      - "traefik.enable=true"
+      ## HTTP Routers
+      - "traefik.http.routers.llama-cpp-rtr.entrypoints=https"
+      - "traefik.http.routers.llama-cpp-rtr.rule=Host(`llamacpp.$DOMAIN_NAME`)"
+      ## HTTP Services
+      - "traefik.http.routers.llama-cpp-rtr.service=llama-cpp-svc"
+      - "traefik.http.services.llama-cpp-svc.loadbalancer.server.port=8000"
 
   # vllm:
   #   image: vllm/vllm-openai:${VLLM_VERSION:-v0.6.6}
@@ -201,8 +260,6 @@ services:
   #   <<: *gpu-deploy
   #   networks:
   #     - podflix-network
-  #   ports:
-  #     - ${MODEL_API_PORT:-8000}:8000
   #   ipc: host
   #   command:
   #     - "--model"
@@ -240,12 +297,11 @@ services:
     restart: ${RESTART_POLICY:-unless-stopped}
     command: ["--model-id", $EMBEDDING_MODEL_NAME, "--revision", $EMBEDDING_MODEL_REVISION, --hostname, "0.0.0.0", "--port", "80"]
     <<: *gpu-deploy
-    depends_on:
-      - podflix-traefik
+    # depends_on:
+    #   - podflix-traefik
     networks:
+      - t2_proxy
       - podflix-network
-    ports:
-      - $EMBEDDING_MODEL_PORT:80
     environment:
       - HF_HOME=/root/.cache/huggingface
       - HF_HUB_OFFLINE=1
@@ -260,23 +316,24 @@ services:
       retries: 3
     labels:
       - "traefik.enable=true"
-      - "traefik.http.services.hf_embedding.loadbalancer.server.port=80"
-      - "traefik.http.routers.hf_embedding.rule=PathPrefix(`/hf_embedding`)"
-      - "traefik.http.routers.hf_embedding.middlewares=hf_embedding-strip-prefix"
-      - "traefik.http.middlewares.hf_embedding-strip-prefix.stripprefix.prefixes=/hf_embedding"
+      ## HTTP Routers
+      - "traefik.http.routers.hf_embedding-rtr.entrypoints=https"
+      - "traefik.http.routers.hf_embedding-rtr.rule=Host(`hf_embedding.$DOMAIN_NAME`)"
+      ## HTTP Services
+      - "traefik.http.routers.hf_embedding-rtr.service=hf_embedding-svc"
+      - "traefik.http.services.hf_embedding-svc.loadbalancer.server.port=80"
 
   hf_rerank:
     image: ghcr.io/huggingface/text-embeddings-inference:1.6
     container_name: hf_rerank
     restart: ${RESTART_POLICY:-unless-stopped}
     command: ["--model-id", $RERANK_MODEL_NAME, "--revision", $RERANK_MODEL_REVISION, --hostname, "0.0.0.0", "--port", "80"]
     <<: *gpu-deploy
-    depends_on:
-      - podflix-traefik
+    # depends_on:
+    #   - podflix-traefik
     networks:
+      - t2_proxy
       - podflix-network
-    ports:
-      - $RERANK_MODEL_PORT:80
     environment:
       - HF_HOME=/root/.cache/huggingface
       - HF_HUB_OFFLINE=1
@@ -291,7 +348,9 @@ services:
       retries: 3
     labels:
       - "traefik.enable=true"
-      - "traefik.http.services.hf_rerank.loadbalancer.server.port=80"
-      - "traefik.http.routers.hf_rerank.rule=PathPrefix(`/hf_rerank`)"
-      - "traefik.http.routers.hf_rerank.middlewares=hf_rerank-strip-prefix"
-      - "traefik.http.middlewares.hf_rerank-strip-prefix.stripprefix.prefixes=/hf_rerank"
+      ## HTTP Routers
+      - "traefik.http.routers.hf_rerank-rtr.entrypoints=https"
+      - "traefik.http.routers.hf_rerank-rtr.rule=Host(`hf_rerank.$DOMAIN_NAME`)"
+      ## HTTP Services
+      - "traefik.http.routers.hf_rerank-rtr.service=hf_rerank-svc"
+      - "traefik.http.services.hf_rerank-svc.loadbalancer.server.port=80"
diff --git a/notebooks/test_graph.ipynb b/notebooks/test_graph.ipynb
@@ -58,16 +58,38 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from pprint import pprint\n",
+    "from pprint import pprint  # noqa: F401\n",
+    "\n",
+    "from langchain.schema.runnable.config import RunnableConfig\n",
+    "\n",
+    "graph_runnable_config = RunnableConfig(\n",
+    "    callbacks=[],\n",
+    "    recursion_limit=10,\n",
+    ")\n",
+    "\n",
+    "streamable_node_names = [\n",
+    "    \"mock_answer\",\n",
+    "]\n",
+    "\n",
     "\n",
     "async for event in compiled_graph.astream_events(\n",
     "    inputs,\n",
+    "    config=graph_runnable_config,\n",
     "    version=\"v2\",\n",
     "):\n",
     "    event_kind = event[\"event\"]\n",
     "    langgraph_node = event[\"metadata\"].get(\"langgraph_node\", None)\n",
     "\n",
-    "    pprint(event)"
+    "    # pprint(event)\n",
+    "\n",
+    "    if event_kind == \"on_chat_model_stream\":\n",
+    "        if langgraph_node not in streamable_node_names:\n",
+    "            continue\n",
+    "\n",
+    "        ai_message_content = event[\"data\"][\"chunk\"].content\n",
+    "\n",
+    "        if ai_message_content:\n",
+    "            print(ai_message_content)"
    ]
   }
  ],