reorganizing stuff

gen-mind · May 14, 2024 · d5a8224 · d5a8224
1 parent 6f07083
commit d5a8224
Show file tree

Hide file tree

Showing 18 changed files with 168 additions and 238 deletions.
diff --git a/.devcontainer.json b/.devcontainer.json
diff --git a/.gitignore b/.gitignore
@@ -29,3 +29,6 @@ __pycache__/
 # Ignore model directories
 ai/embedder/models/
 
+# ignore data directory
+data
+
diff --git a/ai/chunking/chunking_service.py → ai/chunker/chunker.py b/ai/chunking/chunking_service.py → ai/chunker/chunker.py
diff --git a/ai/chunking/requirements.txt → ai/chunker/requirements.txt b/ai/chunking/requirements.txt → ai/chunker/requirements.txt
diff --git a/ai/chunking/test_publisher.py → ai/chunker/test_publisher.py b/ai/chunking/test_publisher.py → ai/chunker/test_publisher.py
diff --git a/ai/embedder/embedd_server.py → ai/embedder/embedder.py b/ai/embedder/embedd_server.py → ai/embedder/embedder.py
diff --git a/config/chunker.env b/config/chunker.env
diff --git a/config/embedder.env b/config/embedder.env
diff --git a/docker-compose-pulsar.yaml b/docker-compose-pulsar.yaml
diff --git a/docker-compose.yaml b/docker-compose.yaml
diff --git a/docker/docker-compose-cockroach.yaml b/docker/docker-compose-cockroach.yaml
@@ -0,0 +1,17 @@
+version: '3.8'  # Use Docker Compose file version 3.8
+
+networks:
+  dev-network:
+    driver: bridge
+
+services:
+  cockroach:
+    image: cockroachdb/cockroach:latest-v23.2
+    ports:
+      - "26257:26257"
+      - "28080:8080"
+    command: start-single-node --insecure
+    volumes:
+      - "../data/cockroach:/cockroach/cockroach-data"
+    networks:
+      - dev-network
diff --git a/docker-compose-milvus.yml → docker/docker-compose-milvus-minio.yml b/docker-compose-milvus.yml → docker/docker-compose-milvus-minio.yml
@@ -1,4 +1,8 @@
-version: '3.5'
+version: '3.8'  # Use Docker Compose file version 3.8
+
+networks:
+  dev-network:
+    driver: bridge
 
 services:
   etcd:
@@ -9,8 +13,10 @@ services:
       - ETCD_AUTO_COMPACTION_RETENTION=1000
       - ETCD_QUOTA_BACKEND_BYTES=4294967296
     volumes:
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+      - "../data/etcd:/etcd"
     command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    networks:
+      - dev-network
 
   minio:
     container_name: milvus-minio
@@ -19,13 +25,15 @@ services:
       MINIO_ACCESS_KEY: minioadmin
       MINIO_SECRET_KEY: minioadmin
     volumes:
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
+      - "../data/minio:/minio_data"
     command: minio server /minio_data
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
       timeout: 20s
       retries: 3
+    networks:
+      - dev-network
 
   standalone:
     container_name: milvus-standalone
@@ -35,13 +43,23 @@ services:
       ETCD_ENDPOINTS: etcd:2379
       MINIO_ADDRESS: minio:9000
     volumes:
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+      - "../data/milvus:/var/lib/milvus"
     ports:
       - "19530:19530"
     depends_on:
       - "etcd"
       - "minio"
-
-networks:
-  default:
-    name: milvus
+    networks:
+      - dev-network
+
+  milvus-admin:
+    image: milvusdb/milvus-insight:latest
+    environment:
+      HOST_URL: http://localhost:8000
+      MILVUS_URL: host.docker.internal:19530
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    ports:
+      - "8000:3000"
+    networks:
+      - dev-network
diff --git a/docker/docker-compose-nats.yaml b/docker/docker-compose-nats.yaml
@@ -0,0 +1,31 @@
+version: '3.8'  # Use Docker Compose file version 3.8
+
+networks:
+  dev-network:
+    driver: bridge
+
+services:
+  nats:
+    image: nats:latest  # Use the latest NATS image
+    container_name: nats
+    ports:
+      - "4222:4222"  # Client connections to NATS
+      - "8222:8222"  # HTTP monitoring port for NATS
+      - "6222:6222"  # Cluster port for NATS
+    volumes:
+      - nats-storage:/data/nats
+    command:
+      - "--name=nats"
+      - "--http_port=8222"
+      - "--js"
+      - "--sd=/data/nats"
+    networks:
+      - dev-network
+
+volumes:
+  nats-storage:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ../data/nats
diff --git a/ai/embedder/docker-compose-embedder.yml → docker/docker-compose-services-ai.yaml b/ai/embedder/docker-compose-embedder.yml → docker/docker-compose-services-ai.yaml
@@ -4,16 +4,6 @@ networks:
     driver: bridge
 
 services:
-  nats-server:
-    image: nats:latest  # Use the latest NATS image
-    container_name: nats-server
-    ports:
-      - "4222:4222"  # Client connections to NATS
-      - "8222:8222"  # HTTP monitoring port for NATS
-      - "6222:6222"  # Cluster port for NATS
-    networks:
-      - dev-net
-
   #embedder  
   embedder:
     # build: .
@@ -22,7 +12,9 @@ services:
     ports:
       - "50051:50051" 
     volumes:
-      - /models
+      - ../data//models
+    env_file:
+      - ../config/embedder.env
     networks:
       - dev-net
     # depends_on:
@@ -32,4 +24,4 @@ services:
       #PULSAR_CONNECTION_STRING: ${PULSAR_CONNECTION_STRING}
       PULSAR_CONNECTION_STRING: "pulsar://broker:6650"
       #PULSAR_CONNECTION_STRING: "pulsar://127.0.0.1:6650"
-    #command: python ./embedd_service.py
+