From 28cfbe71417db02ab31b3c492ab1c306e63b8ded Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Thu, 8 Aug 2024 15:01:25 +0800 Subject: [PATCH 1/8] update dockerfile --- .github/workflows/docker-cd.yaml | 4 ++-- python/xorbits/deploy/docker/Dockerfile.base | 23 ++++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-cd.yaml b/.github/workflows/docker-cd.yaml index a696d820b..2e2339219 100644 --- a/.github/workflows/docker-cd.yaml +++ b/.github/workflows/docker-cd.yaml @@ -2,7 +2,7 @@ name: Xorbits CD for DockerHub on: schedule: - - cron: '0 18 * * *' + - cron: '05 7 * * *' push: tags: - '*' @@ -33,7 +33,7 @@ jobs: - name: Build and push Docker image shell: bash - if: ${{ github.repository == 'xorbitsai/xorbits' }} + # if: ${{ github.repository == 'xorbitsai/xorbits' }} env: DOCKER_ORG: ${{ secrets.DOCKERHUB_USERNAME }} PY_VERSION: ${{ matrix.python-version }} diff --git a/python/xorbits/deploy/docker/Dockerfile.base b/python/xorbits/deploy/docker/Dockerfile.base index a2fcf0368..b46e810d1 100644 --- a/python/xorbits/deploy/docker/Dockerfile.base +++ b/python/xorbits/deploy/docker/Dockerfile.base @@ -6,35 +6,34 @@ FROM base AS py3.9-base SHELL ["/bin/bash", "-c"] ARG PYTHON_VERSION=3.9 RUN if [ "$PYTHON_VERSION" == "3.9" ] ; then \ - /opt/conda/bin/conda install --freeze-installed -c conda-forge python=3.9 "numpy>=1.14.0,<1.23.0" scipy\>=1.9.2 pandas\>=1.5.0 && \ - /opt/conda/bin/conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi + conda install python=3.9 && \ + conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi FROM base AS py3.10-base SHELL ["/bin/bash", "-c"] ARG PYTHON_VERSION=3.9 RUN if [ "$PYTHON_VERSION" == "3.10" ] ; then \ - /opt/conda/bin/conda install --freeze-installed -c conda-forge python=3.10 numpy\>=1.14.0 scipy\>=1.9.2 pandas\>=1.5.0 && \ - /opt/conda/bin/conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi + conda install python=3.10 && \ + conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi FROM base AS py3.11-base SHELL ["/bin/bash", "-c"] ARG PYTHON_VERSION=3.9 RUN if [ "$PYTHON_VERSION" == "3.11" ] ; then \ - /opt/conda/bin/conda install --freeze-installed -c conda-forge \ - python=3.11 numpy\>=1.14.0 pandas\>=1.5.0 cython\>=0.29.33 scipy\>=1.9.2 \ - mkl numexpr psutil scikit-learn sqlalchemy tornado lz4 ; fi + conda install python=3.11 && \ + conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi FROM py${PYTHON_VERSION}-base AS final -RUN /opt/conda/bin/conda install \ +RUN conda install \ cython \ mkl \ - numexpr\<2.8.5 \ + numexpr \ psutil \ scikit-learn \ sqlalchemy \ tornado \ lz4 \ - && /opt/conda/bin/conda install -c conda-forge \ + && conda install -c conda-forge \ pyarrow\>=1.0 \ python-kubernetes \ jax \ @@ -42,6 +41,8 @@ RUN /opt/conda/bin/conda install \ uvloop \ libnuma \ && pip install -U pip \ + && pip install -U "numpy>=1.14.0,<2.0" \ + && pip install scipy\>=1.9.2 pandas\>=1.5.0 \ && pip install -U \ xoscar \ cloudpickle \ @@ -51,7 +52,7 @@ RUN /opt/conda/bin/conda install \ s3fs \ pyopenssl>=23.0.0 \ datasets \ - && /opt/conda/bin/conda clean --all -f -y + && conda clean --all -f -y ENV NVM_DIR /usr/local/nvm ENV NODE_VERSION 14.21.1 From 9e097e2f2c5b616fa181e87663cc2000a5e85888 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Fri, 9 Aug 2024 09:13:42 +0800 Subject: [PATCH 2/8] pandas DataFrame._data -> .mgr --- .github/workflows/python.yaml | 1 - python/xorbits/_mars/_utils.pyx | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 77d3266f8..548fdf2cf 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -62,7 +62,6 @@ jobs: run: cd python/xorbits/web/ui && ./node_modules/.bin/prettier --check . build_test_job: - if: github.repository == 'xorbitsai/xorbits' runs-on: ${{ matrix.os }} needs: lint env: diff --git a/python/xorbits/_mars/_utils.pyx b/python/xorbits/_mars/_utils.pyx index 878d19e01..3564ce27b 100644 --- a/python/xorbits/_mars/_utils.pyx +++ b/python/xorbits/_mars/_utils.pyx @@ -201,7 +201,7 @@ cdef list tokenize_pandas_series(ob): cdef list tokenize_pandas_dataframe(ob): - l = [block.values for block in ob._data.blocks] + l = [block.values for block in ob._mgr.blocks] l.extend([ob.columns, ob.index]) return iterative_tokenize(l) From c109a0ab013addb5f00076c8ba5437d094bb848e Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Fri, 9 Aug 2024 09:29:22 +0800 Subject: [PATCH 3/8] pandas DataFrame._data -> .mgr --- .github/workflows/python.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 548fdf2cf..81b0eca5b 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -4,8 +4,6 @@ on: push: branches: - '*' - pull_request: - types: ['opened', 'reopened', 'synchronize'] concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 508833fb453653dd8aec3388adafc14c2cb2b100 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Fri, 9 Aug 2024 21:48:30 +0800 Subject: [PATCH 4/8] clean ci --- .github/workflows/docker-cd.yaml | 4 ++-- .github/workflows/python.yaml | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-cd.yaml b/.github/workflows/docker-cd.yaml index 2e2339219..a696d820b 100644 --- a/.github/workflows/docker-cd.yaml +++ b/.github/workflows/docker-cd.yaml @@ -2,7 +2,7 @@ name: Xorbits CD for DockerHub on: schedule: - - cron: '05 7 * * *' + - cron: '0 18 * * *' push: tags: - '*' @@ -33,7 +33,7 @@ jobs: - name: Build and push Docker image shell: bash - # if: ${{ github.repository == 'xorbitsai/xorbits' }} + if: ${{ github.repository == 'xorbitsai/xorbits' }} env: DOCKER_ORG: ${{ secrets.DOCKERHUB_USERNAME }} PY_VERSION: ${{ matrix.python-version }} diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 81b0eca5b..2f1ff12bb 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -4,6 +4,8 @@ on: push: branches: - '*' + pull_request: + types: ['opened', 'reopened', 'synchronize'] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -135,8 +137,7 @@ jobs: PYTHON: ${{ matrix.python-version }} if: ${{ matrix.module != 'gpu' }} run: | - pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar" - pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky "numexpr<2.8.5" + pip install xoscar "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr if [[ "$MODULE" == "xorbits" ]]; then pip install openpyxl @@ -245,7 +246,6 @@ jobs: - name: Install on JuiceFsCI if: ${{ matrix.module == 'kubernetes-juicefs' }} run: | - pip install -U xoscar python setup.py build_ext -i working-directory: ./python @@ -264,7 +264,6 @@ jobs: - name: Install on GPU if: ${{ matrix.module == 'gpu' }} run: | - pip install -U xoscar python setup.py build_ext -i working-directory: ./python From f48e766977fa0576fe47bc46e80f750c15dc79be Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Fri, 9 Aug 2024 21:55:49 +0800 Subject: [PATCH 5/8] clean ci --- .github/workflows/python.yaml | 3 +++ python/xorbits/deploy/docker/Dockerfile.base | 23 ++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 2f1ff12bb..db2dfa13a 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -62,6 +62,7 @@ jobs: run: cd python/xorbits/web/ui && ./node_modules/.bin/prettier --check . build_test_job: + if: github.repository == 'xorbitsai/xorbits' runs-on: ${{ matrix.os }} needs: lint env: @@ -246,6 +247,7 @@ jobs: - name: Install on JuiceFsCI if: ${{ matrix.module == 'kubernetes-juicefs' }} run: | + pip install -U xoscar python setup.py build_ext -i working-directory: ./python @@ -264,6 +266,7 @@ jobs: - name: Install on GPU if: ${{ matrix.module == 'gpu' }} run: | + pip install -U xoscar python setup.py build_ext -i working-directory: ./python diff --git a/python/xorbits/deploy/docker/Dockerfile.base b/python/xorbits/deploy/docker/Dockerfile.base index b46e810d1..a2fcf0368 100644 --- a/python/xorbits/deploy/docker/Dockerfile.base +++ b/python/xorbits/deploy/docker/Dockerfile.base @@ -6,34 +6,35 @@ FROM base AS py3.9-base SHELL ["/bin/bash", "-c"] ARG PYTHON_VERSION=3.9 RUN if [ "$PYTHON_VERSION" == "3.9" ] ; then \ - conda install python=3.9 && \ - conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi + /opt/conda/bin/conda install --freeze-installed -c conda-forge python=3.9 "numpy>=1.14.0,<1.23.0" scipy\>=1.9.2 pandas\>=1.5.0 && \ + /opt/conda/bin/conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi FROM base AS py3.10-base SHELL ["/bin/bash", "-c"] ARG PYTHON_VERSION=3.9 RUN if [ "$PYTHON_VERSION" == "3.10" ] ; then \ - conda install python=3.10 && \ - conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi + /opt/conda/bin/conda install --freeze-installed -c conda-forge python=3.10 numpy\>=1.14.0 scipy\>=1.9.2 pandas\>=1.5.0 && \ + /opt/conda/bin/conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi FROM base AS py3.11-base SHELL ["/bin/bash", "-c"] ARG PYTHON_VERSION=3.9 RUN if [ "$PYTHON_VERSION" == "3.11" ] ; then \ - conda install python=3.11 && \ - conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py ; fi + /opt/conda/bin/conda install --freeze-installed -c conda-forge \ + python=3.11 numpy\>=1.14.0 pandas\>=1.5.0 cython\>=0.29.33 scipy\>=1.9.2 \ + mkl numexpr psutil scikit-learn sqlalchemy tornado lz4 ; fi FROM py${PYTHON_VERSION}-base AS final -RUN conda install \ +RUN /opt/conda/bin/conda install \ cython \ mkl \ - numexpr \ + numexpr\<2.8.5 \ psutil \ scikit-learn \ sqlalchemy \ tornado \ lz4 \ - && conda install -c conda-forge \ + && /opt/conda/bin/conda install -c conda-forge \ pyarrow\>=1.0 \ python-kubernetes \ jax \ @@ -41,8 +42,6 @@ RUN conda install \ uvloop \ libnuma \ && pip install -U pip \ - && pip install -U "numpy>=1.14.0,<2.0" \ - && pip install scipy\>=1.9.2 pandas\>=1.5.0 \ && pip install -U \ xoscar \ cloudpickle \ @@ -52,7 +51,7 @@ RUN conda install \ s3fs \ pyopenssl>=23.0.0 \ datasets \ - && conda clean --all -f -y + && /opt/conda/bin/conda clean --all -f -y ENV NVM_DIR /usr/local/nvm ENV NODE_VERSION 14.21.1 From 13a51c4bba9db3b7b92cc70839128ed6e1ac1423 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Fri, 9 Aug 2024 22:14:57 +0800 Subject: [PATCH 6/8] ci --- .github/workflows/python.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index db2dfa13a..4675f78ea 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -138,7 +138,8 @@ jobs: PYTHON: ${{ matrix.python-version }} if: ${{ matrix.module != 'gpu' }} run: | - pip install xoscar "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr + pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar" + pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr if [[ "$MODULE" == "xorbits" ]]; then pip install openpyxl From e53e8ce31e1893c897c625291e8deb145645abd8 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Sat, 17 Aug 2024 11:51:00 +0800 Subject: [PATCH 7/8] CI test execution --- CI/test_basic_execution.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CI/test_basic_execution.py b/CI/test_basic_execution.py index 06aa62fd5..64edb4fbf 100644 --- a/CI/test_basic_execution.py +++ b/CI/test_basic_execution.py @@ -7,10 +7,8 @@ def test_basic_cases(): - with warnings.catch_warnings(): - # TODO: change warnings.simplefilter("error") to "module" - # "error" help us find the deprecated APIs - warnings.simplefilter("module") + with warnings.catch_warnings(): # "error" help us find the deprecated APIs + warnings.simplefilter("error") import xorbits import xorbits.pandas as pd import xorbits.numpy as np From afccf7cfd928c60950da30c04d7ce35f5b0b5f55 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Sat, 17 Aug 2024 13:10:46 +0800 Subject: [PATCH 8/8] CI test execution --- CI/test_basic_execution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CI/test_basic_execution.py b/CI/test_basic_execution.py index 64edb4fbf..fc5cacce9 100644 --- a/CI/test_basic_execution.py +++ b/CI/test_basic_execution.py @@ -7,7 +7,8 @@ def test_basic_cases(): - with warnings.catch_warnings(): # "error" help us find the deprecated APIs + with warnings.catch_warnings(): + # "error" help us find the deprecated APIs warnings.simplefilter("error") import xorbits import xorbits.pandas as pd