feat(datasource): add vectordb datasource (IGinX-THU#530)

workflows中未配置和执行vectordb的tpc-h测试 DB-CE测试中跳过了vectordb在ubuntu-latest和macos-latest平台上的测试
jzl18thu · Jan 27, 2025 · 7f7fa94 · 7f7fa94
1 parent 14b1672
commit 7f7fa94
Show file tree

Hide file tree

Showing 54 changed files with 5,997 additions and 19 deletions.
diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml
@@ -200,6 +200,20 @@ runs:
               paths: ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-*/conf/config.properties
               statements: s|^#storageEngineList=127.0.0.1#3306#relational#engine=mysql.*$|storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false|g
 
+        - if: inputs.DB-name == 'VectorDB'
+          name: Change VectorDB Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-*/conf/config.properties
+              statements: s/^#storageEngineList=127.0.0.1#19530/storageEngineList=127.0.0.1#19530/g
+
+        - if: inputs.DB-name == 'VectorDB'
+          name: Change VectorDB Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-*/conf/config.properties
+              statements: s/^#storageEngineList=192.168.106.2#19530/storageEngineList=192.168.106.2#19530/g
+
         - if: inputs.DB-name == 'FileSystem'
           name: Modify IGinX Config
           uses: ./.github/actions/edit

diff --git a/.github/actions/dbLog/action.yml b/.github/actions/dbLog/action.yml
@@ -21,3 +21,21 @@ runs:
           shell: bash
           run: |
               "${GITHUB_WORKSPACE}/.github/actions/dbLog/dbLog.sh" ${{ inputs.DB-name }}
+
+        - if: ${{ inputs.DB-name == 'VectorDB' && runner.os != 'Windows' }}
+          name: Show Milvus Log
+          working-directory: ${{ github.workspace }}
+          shell: bash
+          run: |
+              docker-compose -f 19530/docker-compose-19530.yml logs | grep -v INFO |grep -v '"info"'
+              ls volumes19530
+              du -sh volumes19530
+              du -sh volumes19530/etcd
+              du -sh volumes19530/milvus
+
+        - if: ${{ inputs.DB-name == 'VectorDB' && runner.os == 'Windows' }}
+          name: Show Milvus Log on Windows
+          working-directory: ${{ github.workspace }}
+          shell: wsl-run {0}
+          run: |
+              docker-compose -f 19530/docker-compose-19530.yml logs | grep -v INFO |grep -v '"info"'
diff --git a/.github/actions/dbRunner/action.yml b/.github/actions/dbRunner/action.yml
@@ -29,6 +29,10 @@ inputs:
         description: "Redis ports"
         required: false
         default: 6379
+    VectorDB-ports:
+        description: "VectorDB ports"
+        required: false
+        default: 19530
 
 runs:
     using: "composite" # Mandatory parameter
@@ -127,3 +131,12 @@ runs:
           uses: ./.github/actions/service/mysql
           with:
               ports: ${{ inputs.MySQL-ports }}
+
+        - if: inputs.DB-name=='VectorDB'
+          name: Run Milvus DB
+          uses: ./.github/actions/service/vectordb
+          with:
+              stop: false
+              clean: false
+              start: true
+              ports: ${{ inputs.VectorDB-ports }}
diff --git a/.github/actions/dockerCompose/action.yml b/.github/actions/dockerCompose/action.yml
@@ -0,0 +1,51 @@
+name: "docker-compose-setup"
+description: "environment docker-compose setup"
+
+runs:
+    using: "composite"
+    steps:
+        - if: runner.os == 'macOS'
+          name: Install Docker on MacOS
+          shell: bash
+          run: |
+              brew update
+              brew install docker qemu
+              brew install colima
+              LIMACTL_PATH=$(brew --prefix)/bin/limactl
+              sudo curl -L -o $LIMACTL_PATH https://github.com/mikekazakov/lima-nohvf/raw/master/limactl && sudo chmod +x $LIMACTL_PATH
+              colima start --network-address --arch arm64 --vm-type=qemu
+              docker --version
+              colima status
+              brew install docker-compose
+              docker-compose --version
+
+        - if: runner.os == 'Windows'
+          uses: vedantmgoyal9/setup-wsl2@main
+
+        - if: runner.os == 'Windows'
+          run: apt update && apt upgrade -y
+          shell: wsl-run {0}
+
+        - if: runner.os == 'Windows'
+          name: Install Docker Compose
+          shell: wsl-run {0}
+          run: |
+              sudo apt install apt-transport-https ca-certificates curl software-properties-common -y
+              curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+              sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable"
+              sudo apt update
+              sudo apt install docker-ce docker-ce-cli containerd.io -y
+              sudo service docker start
+              docker --version
+              DOCKER_COMPOSE_VERSION=v2.30.1
+              sudo curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+              sudo chmod +x /usr/local/bin/docker-compose
+              docker-compose --version
+
+        - if: runner.os == 'Linux'
+          name: Install Docker Compose
+          shell: bash
+          run: |
+              DOCKER_COMPOSE_VERSION=v2.30.1
+              sudo curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+              sudo chmod +x /usr/local/bin/docker-compose
diff --git a/.github/actions/service/vectordb/action.yml b/.github/actions/service/vectordb/action.yml
@@ -0,0 +1,83 @@
+name: "milvus"
+description: "stop, clean and start milvus in order"
+inputs:
+    stop:
+        description: "whether to stop"
+        required: true
+    clean:
+        description: "whether to clean"
+        required: true
+    start:
+        description: "whether to start"
+        required: true
+    ports:
+        description: "ports to run on"
+        required: false
+        default: "19530"
+
+runs:
+    using: "composite"
+    steps:
+        - if: fromJSON(inputs.stop) || fromJSON(inputs.clean)
+          name: unimplemented
+          shell: bash
+          run: exit 1
+
+        - name: Setup docker compose
+          uses: ./.github/actions/dockerCompose
+
+        - if: fromJSON(inputs.start)
+          name: Create Config for milvus docker compose
+          shell: bash
+          working-directory: ${{ github.workspace }}
+          run: |
+              for port in ${{ inputs.ports }}; do
+                mkdir -p $port
+                mkdir -p volumes$port
+                mkdir -p volumes$port/etcd
+                mkdir -p volumes$port/minio
+                mkdir -p volumes$port/milvus
+                cp -f docker/milvus/docker-compose.yml $port/docker-compose-$port.yml
+                sed -i"" -e "s/19530:19530/$port:19530/g" $port/docker-compose-$port.yml
+                sed -i"" -e "s/\/volumes/\/volumes$port/g" $port/docker-compose-$port.yml
+                sed -i"" -e "s/name: milvus/name: milvus$port/g" $port/docker-compose-$port.yml
+              done
+              echo "docker-compose.yml created successfully"
+
+        - if: fromJSON(inputs.start) && runner.os != 'Windows'
+          name: Start milvus
+          shell: bash
+          working-directory: ${{ github.workspace }}
+          run: |
+              for port in ${{ inputs.ports }}; do
+                sudo chown -R $(whoami) $port/
+                sudo nohup docker-compose -f $port/docker-compose-$port.yml up -d
+              done
+              sleep 10
+              docker ps -a
+
+        - if: fromJSON(inputs.start) && runner.os == 'Windows'
+          name: Start milvus on Windows
+          shell: wsl-run {0}
+          working-directory: ${{ github.workspace }}
+          run: |
+              for port in ${{ inputs.ports }}; do
+                sudo chown -R $(whoami) $port/
+                sudo nohup docker-compose -f $port/docker-compose-$port.yml up -d
+              done
+              sleep 30
+              docker ps -a
+
+        - if: runner.os == 'macOS'
+          name: replace host
+          shell: bash
+          working-directory: ${{ github.workspace }}
+          run: |
+              MILVUS_HOST="192.168.106.2"
+              echo "MILVUS_HOST=${MILVUS_HOST}"
+              sed -i"" -e "s/127.0.0.1/${MILVUS_HOST}/g" test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java
+              sed -i"" -e "s/Session(\"${MILVUS_HOST}/Session(\"127.0.0.1/g" test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java
+              sed -i"" -e "s/127.0.0.1/${MILVUS_HOST}/g" test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/vectordb/VectorDBHistoryDataGenerator.java
+              sed -i"" -e "s/127.0.0.1#19530/${MILVUS_HOST}#19530/g" test/src/test/resources/testConfig.properties
+              sed -i"" -e "s/127.0.0.1#19530/${MILVUS_HOST}#19530/g" conf/config.properties
+              sleep 60
diff --git a/.github/scripts/dataSources/restart/milvus.sh b/.github/scripts/dataSources/restart/milvus.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# IGinX - the polystore system with high performance
+# Copyright (C) Tsinghua University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+port=$1
+
+pwd
+
+sudo nohup docker-compose -f $port/docker-compose-$port.yml up -d
+
+echo "waiting for the server to start..."
+sleep 10
+docker ps
+
+if netstat -an | grep -q ".*:$port.*LISTEN"; then
+  echo "Port $port is open."
+else
+  echo "Port $port is not open."
+fi
diff --git a/.github/scripts/dataSources/restart/milvus_macos.sh b/.github/scripts/dataSources/restart/milvus_macos.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+# IGinX - the polystore system with high performance
+# Copyright (C) Tsinghua University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+port=$1
+
+pwd
+
+docker-compose -f $port/docker-compose-$port.yml up -d
+
+sleep 70
+
+docker ps
diff --git a/.github/scripts/dataSources/restart/milvus_windows.sh b/.github/scripts/dataSources/restart/milvus_windows.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+# IGinX - the polystore system with high performance
+# Copyright (C) Tsinghua University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+set -e
+port=$1
+
+pwd
+
+wsl docker-compose -f $port/docker-compose-$port.yml up -d
+
+sleep 30
+
+docker ps
diff --git a/.github/scripts/dataSources/shutdown/milvus.sh b/.github/scripts/dataSources/shutdown/milvus.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# IGinX - the polystore system with high performance
+# Copyright (C) Tsinghua University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+port=$1
+
+pwd
+
+docker-compose -f $port/docker-compose-$port.yml down
+
+docker ps
+
diff --git a/.github/scripts/dataSources/shutdown/milvus_macos.sh b/.github/scripts/dataSources/shutdown/milvus_macos.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# IGinX - the polystore system with high performance
+# Copyright (C) Tsinghua University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+port=$1
+
+pwd
+
+docker-compose -f $port/docker-compose-$port.yml down
+
+docker ps
diff --git a/.github/scripts/dataSources/shutdown/milvus_windows.sh b/.github/scripts/dataSources/shutdown/milvus_windows.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# IGinX - the polystore system with high performance
+# Copyright (C) Tsinghua University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+set -e
+port=$1
+
+pwd
+
+wsl docker-compose -f $port/docker-compose-$port.yml down
+
+sleep 10
+
+wsl docker ps
diff --git a/.github/scripts/iginx/iginx_macos.sh b/.github/scripts/iginx/iginx_macos.sh
@@ -37,7 +37,7 @@ sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &"
 sh -c "sleep 3"
 
 log_file="iginx-$1.log"
-timeout=30
+timeout=60
 interval=2
 
 elapsed_time=0