From 4748977381b16071e23079be69ca61cb309f2cfe Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:07:08 +1100 Subject: [PATCH 01/39] Change back to rust 1.74.1 for testing --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a541091e3a2b..7c8cd40670eb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install stable - rustup default stable + rustup toolchain install 1.74.1 + rustup default 1.74.1 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 18908d5e5e558d9425abc873dc050b581b66e9f6 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:57:56 +1100 Subject: [PATCH 02/39] Run only tpcds planning for faster feedback --- .github/workflows/rust.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7c8cd40670eb..a832d278bd7f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -296,10 +296,7 @@ jobs: - name: Run tests (excluding doctests) shell: bash run: | - export PATH=$PATH:$HOME/d/protoc/bin - cargo test --lib --tests --bins --features avro,json,backtrace - cd datafusion-cli - cargo test --lib --tests --bins --all-features + cargo test -p datafusion --test tpcds_planning env: # do not produce debug symbols to keep memory usage down RUSTFLAGS: "-C debuginfo=0" From c70faaedff6174fdb818227ef0532dcfdf3e2eda Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:06:20 +1100 Subject: [PATCH 03/39] test --- .github/workflows/rust.yml | 820 ++++++++++++++++++------------------- 1 file changed, 410 insertions(+), 410 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a832d278bd7f..dd3bc98216f5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -40,234 +40,234 @@ on: jobs: # Check crate compiles - linux-build-lib: - name: cargo check - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable + # linux-build-lib: + # name: cargo check + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ./target/ - ./datafusion-cli/target/ - # this key equals the ones on `linux-build-lib` for re-use - key: cargo-cache-benchmark-${{ hashFiles('datafusion/**/Cargo.toml', 'benchmarks/Cargo.toml', 'datafusion-cli/Cargo.toml') }} + # - name: Cache Cargo + # uses: actions/cache@v3 + # with: + # path: | + # ~/.cargo/bin/ + # ~/.cargo/registry/index/ + # ~/.cargo/registry/cache/ + # ~/.cargo/git/db/ + # ./target/ + # ./datafusion-cli/target/ + # # this key equals the ones on `linux-build-lib` for re-use + # key: cargo-cache-benchmark-${{ hashFiles('datafusion/**/Cargo.toml', 'benchmarks/Cargo.toml', 'datafusion-cli/Cargo.toml') }} - - name: Check workspace without default features - run: cargo check --no-default-features -p datafusion + # - name: Check workspace without default features + # run: cargo check --no-default-features -p datafusion - - name: Check datafusion-common without default features - run: cargo check --tests --no-default-features -p datafusion-common + # - name: Check datafusion-common without default features + # run: cargo check --tests --no-default-features -p datafusion-common - - name: Check workspace in debug mode - run: cargo check + # - name: Check workspace in debug mode + # run: cargo check - - name: Check workspace with all features - run: cargo check --workspace --benches --features avro,json - - name: Check Cargo.lock for datafusion-cli - run: | - # If this test fails, try running `cargo update` in the `datafusion-cli` directory - # and check in the updated Cargo.lock file. - cargo check --manifest-path datafusion-cli/Cargo.toml --locked + # - name: Check workspace with all features + # run: cargo check --workspace --benches --features avro,json + # - name: Check Cargo.lock for datafusion-cli + # run: | + # # If this test fails, try running `cargo update` in the `datafusion-cli` directory + # # and check in the updated Cargo.lock file. + # cargo check --manifest-path datafusion-cli/Cargo.toml --locked - # test the crate - linux-test: - name: cargo test (amd64) - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run tests (excluding doctests) - run: cargo test --lib --tests --bins --features avro,json,backtrace - - name: Verify Working Directory Clean - run: git diff --exit-code + # # test the crate + # linux-test: + # name: cargo test (amd64) + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Run tests (excluding doctests) + # run: cargo test --lib --tests --bins --features avro,json,backtrace + # - name: Verify Working Directory Clean + # run: git diff --exit-code - linux-test-datafusion-cli: - name: cargo test datafusion-cli (amd64) - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run tests (excluding doctests) - run: | - cd datafusion-cli - cargo test --lib --tests --bins --all-features - - name: Verify Working Directory Clean - run: git diff --exit-code + # linux-test-datafusion-cli: + # name: cargo test datafusion-cli (amd64) + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Run tests (excluding doctests) + # run: | + # cd datafusion-cli + # cargo test --lib --tests --bins --all-features + # - name: Verify Working Directory Clean + # run: git diff --exit-code - linux-test-example: - name: cargo examples (amd64) - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run examples - run: | - # test datafusion-sql examples - cargo run --example sql - # test datafusion-examples - ci/scripts/rust_example.sh - - name: Verify Working Directory Clean - run: git diff --exit-code + # linux-test-example: + # name: cargo examples (amd64) + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Run examples + # run: | + # # test datafusion-sql examples + # cargo run --example sql + # # test datafusion-examples + # ci/scripts/rust_example.sh + # - name: Verify Working Directory Clean + # run: git diff --exit-code - # Run `cargo test doc` (test documentation examples) - linux-test-doc: - name: cargo test doc (amd64) - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run doctests - run: | - cargo test --doc --features avro,json - cd datafusion-cli - cargo test --doc --all-features - - name: Verify Working Directory Clean - run: git diff --exit-code + # # Run `cargo test doc` (test documentation examples) + # linux-test-doc: + # name: cargo test doc (amd64) + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Run doctests + # run: | + # cargo test --doc --features avro,json + # cd datafusion-cli + # cargo test --doc --all-features + # - name: Verify Working Directory Clean + # run: git diff --exit-code - # Run `cargo doc` to ensure the rustdoc is clean - linux-rustdoc: - name: cargo doc - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run cargo doc - run: | - export RUSTDOCFLAGS="-D warnings -A rustdoc::private-intra-doc-links" - cargo doc --document-private-items --no-deps --workspace - cd datafusion-cli - cargo doc --document-private-items --no-deps + # # Run `cargo doc` to ensure the rustdoc is clean + # linux-rustdoc: + # name: cargo doc + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Run cargo doc + # run: | + # export RUSTDOCFLAGS="-D warnings -A rustdoc::private-intra-doc-links" + # cargo doc --document-private-items --no-deps --workspace + # cd datafusion-cli + # cargo doc --document-private-items --no-deps - linux-wasm-pack: - name: build with wasm-pack - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Install wasm-pack - run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - name: Build with wasm-pack - working-directory: ./datafusion/wasmtest - run: wasm-pack build --dev + # linux-wasm-pack: + # name: build with wasm-pack + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Install wasm-pack + # run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + # - name: Build with wasm-pack + # working-directory: ./datafusion/wasmtest + # run: wasm-pack build --dev - # verify that the benchmark queries return the correct results - verify-benchmark-results: - name: verify benchmark results (amd64) - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Generate benchmark data and expected query results - run: | - mkdir -p datafusion/sqllogictest/test_files/tpch/data - git clone https://github.com/databricks/tpch-dbgen.git - cd tpch-dbgen - make - ./dbgen -f -s 0.1 - mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data - - name: Verify that benchmark queries return expected results - run: | - export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` - # use release build for plan verificaton because debug build causes stack overflow - cargo test plan_q --package datafusion-benchmarks --profile release-nonlto --features=ci -- --test-threads=1 - INCLUDE_TPCH=true cargo test --test sqllogictests - - name: Verify Working Directory Clean - run: git diff --exit-code + # # verify that the benchmark queries return the correct results + # verify-benchmark-results: + # name: verify benchmark results (amd64) + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Generate benchmark data and expected query results + # run: | + # mkdir -p datafusion/sqllogictest/test_files/tpch/data + # git clone https://github.com/databricks/tpch-dbgen.git + # cd tpch-dbgen + # make + # ./dbgen -f -s 0.1 + # mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data + # - name: Verify that benchmark queries return expected results + # run: | + # export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` + # # use release build for plan verificaton because debug build causes stack overflow + # cargo test plan_q --package datafusion-benchmarks --profile release-nonlto --features=ci -- --test-threads=1 + # INCLUDE_TPCH=true cargo test --test sqllogictests + # - name: Verify Working Directory Clean + # run: git diff --exit-code - sqllogictest-postgres: - name: "Run sqllogictest with Postgres runner" - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: db_test - POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=C --lc-ctype=C - ports: - - 5432/tcp - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - - name: Run sqllogictest - run: PG_COMPAT=true PG_URI="postgresql://postgres:postgres@localhost:$POSTGRES_PORT/db_test" cargo test --features=postgres --test sqllogictests - env: - POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + # sqllogictest-postgres: + # name: "Run sqllogictest with Postgres runner" + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # services: + # postgres: + # image: postgres:15 + # env: + # POSTGRES_PASSWORD: postgres + # POSTGRES_DB: db_test + # POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=C --lc-ctype=C + # ports: + # - 5432/tcp + # options: >- + # --health-cmd pg_isready + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup toolchain + # run: | + # rustup toolchain install stable + # rustup default stable + # - name: Run sqllogictest + # run: PG_COMPAT=true PG_URI="postgresql://postgres:postgres@localhost:$POSTGRES_PORT/db_test" cargo test --features=postgres --test sqllogictests + # env: + # POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} windows: name: cargo test (win64) @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install 1.74.1 - rustup default 1.74.1 + rustup toolchain install nightly-yyyy-mm-dd + rustup default nightly-2023-10-16 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash @@ -302,96 +302,96 @@ jobs: RUSTFLAGS: "-C debuginfo=0" RUST_BACKTRACE: "1" - macos: - name: cargo test (mac) - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install protobuf compiler - shell: bash - run: | - mkdir -p $HOME/d/protoc - cd $HOME/d/protoc - export PROTO_ZIP="protoc-21.4-osx-x86_64.zip" - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP - unzip $PROTO_ZIP - echo "$HOME/d/protoc/bin" >> $GITHUB_PATH - export PATH=$PATH:$HOME/d/protoc/bin - protoc --version - # TODO: this won't cache anything, which is expensive. Setup this action - # with a OS-dependent path. - - name: Setup Rust toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt - - name: Run tests (excluding doctests) - shell: bash - run: | - cargo test --lib --tests --bins --features avro,json,backtrace - cd datafusion-cli - cargo test --lib --tests --bins --all-features - env: - # do not produce debug symbols to keep memory usage down - RUSTFLAGS: "-C debuginfo=0" - RUST_BACKTRACE: "1" + # macos: + # name: cargo test (mac) + # runs-on: macos-latest + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Install protobuf compiler + # shell: bash + # run: | + # mkdir -p $HOME/d/protoc + # cd $HOME/d/protoc + # export PROTO_ZIP="protoc-21.4-osx-x86_64.zip" + # curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + # unzip $PROTO_ZIP + # echo "$HOME/d/protoc/bin" >> $GITHUB_PATH + # export PATH=$PATH:$HOME/d/protoc/bin + # protoc --version + # # TODO: this won't cache anything, which is expensive. Setup this action + # # with a OS-dependent path. + # - name: Setup Rust toolchain + # run: | + # rustup toolchain install stable + # rustup default stable + # rustup component add rustfmt + # - name: Run tests (excluding doctests) + # shell: bash + # run: | + # cargo test --lib --tests --bins --features avro,json,backtrace + # cd datafusion-cli + # cargo test --lib --tests --bins --all-features + # env: + # # do not produce debug symbols to keep memory usage down + # RUSTFLAGS: "-C debuginfo=0" + # RUST_BACKTRACE: "1" - test-datafusion-pyarrow: - name: cargo test pyarrow (amd64) - needs: [ linux-build-lib ] - runs-on: ubuntu-20.04 - container: - image: amd64/rust:bullseye # Workaround https://github.com/actions/setup-python/issues/721 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: actions/setup-python@v5 - with: - python-version: "3.8" - - name: Install PyArrow - run: | - echo "LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV - python -m pip install pyarrow - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run datafusion-common tests - run: cargo test -p datafusion-common --features=pyarrow + # test-datafusion-pyarrow: + # name: cargo test pyarrow (amd64) + # needs: [ linux-build-lib ] + # runs-on: ubuntu-20.04 + # container: + # image: amd64/rust:bullseye # Workaround https://github.com/actions/setup-python/issues/721 + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - uses: actions/setup-python@v5 + # with: + # python-version: "3.8" + # - name: Install PyArrow + # run: | + # echo "LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV + # python -m pip install pyarrow + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Run datafusion-common tests + # run: cargo test -p datafusion-common --features=pyarrow - vendor: - name: Verify Vendored Code - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Run gen - run: ./regen.sh - working-directory: ./datafusion/proto - - name: Verify workspace clean (if this fails, run ./datafusion/proto/regen.sh and check in results) - run: git diff --exit-code + # vendor: + # name: Verify Vendored Code + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # - name: Run gen + # run: ./regen.sh + # working-directory: ./datafusion/proto + # - name: Verify workspace clean (if this fails, run ./datafusion/proto/regen.sh and check in results) + # run: git diff --exit-code - check-fmt: - name: Check cargo fmt - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run - run: | - echo '' > datafusion/proto/src/generated/datafusion.rs - ci/scripts/rust_fmt.sh + # check-fmt: + # name: Check cargo fmt + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Run + # run: | + # echo '' > datafusion/proto/src/generated/datafusion.rs + # ci/scripts/rust_fmt.sh # Coverage job disabled due to # https://github.com/apache/arrow-datafusion/issues/3678 @@ -435,114 +435,114 @@ jobs: # continue-on-error: true # run: bash <(curl -s https://codecov.io/bash) - clippy: - name: clippy - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Install Clippy - run: rustup component add clippy - - name: Run clippy - run: ci/scripts/rust_clippy.sh + # clippy: + # name: clippy + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Install Clippy + # run: rustup component add clippy + # - name: Run clippy + # run: ci/scripts/rust_clippy.sh - # Check answers are correct when hash values collide - hash-collisions: - name: cargo test hash collisions (amd64) - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run tests - run: | - cd datafusion - cargo test --lib --tests --features=force_hash_collisions,avro + # # Check answers are correct when hash values collide + # hash-collisions: + # name: cargo test hash collisions (amd64) + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Run tests + # run: | + # cd datafusion + # cargo test --lib --tests --features=force_hash_collisions,avro - cargo-toml-formatting-checks: - name: check Cargo.toml formatting - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Install cargo-tomlfmt - run: which cargo-tomlfmt || cargo install cargo-tomlfmt + # cargo-toml-formatting-checks: + # name: check Cargo.toml formatting + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - name: Install cargo-tomlfmt + # run: which cargo-tomlfmt || cargo install cargo-tomlfmt - - name: Check Cargo.toml formatting - run: | - # if you encounter an error, try running 'cargo tomlfmt -p path/to/Cargo.toml' to fix the formatting automatically. - # If the error still persists, you need to manually edit the Cargo.toml file, which introduces formatting violation. - # - # ignore ./Cargo.toml because putting workspaces in multi-line lists make it easy to read - ci/scripts/rust_toml_fmt.sh + # - name: Check Cargo.toml formatting + # run: | + # # if you encounter an error, try running 'cargo tomlfmt -p path/to/Cargo.toml' to fix the formatting automatically. + # # If the error still persists, you need to manually edit the Cargo.toml file, which introduces formatting violation. + # # + # # ignore ./Cargo.toml because putting workspaces in multi-line lists make it easy to read + # ci/scripts/rust_toml_fmt.sh - config-docs-check: - name: check configs.md is up-to-date - needs: [ linux-build-lib ] - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - uses: actions/setup-node@v4 - with: - node-version: "20" - - name: Check if configs.md has been modified - run: | - # If you encounter an error, run './dev/update_config_docs.sh' and commit - ./dev/update_config_docs.sh - git diff --exit-code + # config-docs-check: + # name: check configs.md is up-to-date + # needs: [ linux-build-lib ] + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # with: + # rust-version: stable + # - uses: actions/setup-node@v4 + # with: + # node-version: "20" + # - name: Check if configs.md has been modified + # run: | + # # If you encounter an error, run './dev/update_config_docs.sh' and commit + # ./dev/update_config_docs.sh + # git diff --exit-code - # Verify MSRV for the crates which are directly used by other projects. - msrv: - name: Verify MSRV - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Install cargo-msrv - run: cargo install cargo-msrv - - name: Check datafusion - working-directory: datafusion/core - run: cargo msrv verify - - name: Check datafusion-substrait - working-directory: datafusion/substrait - run: cargo msrv verify - - name: Check datafusion-proto - working-directory: datafusion/proto - run: cargo msrv verify - - name: Check datafusion-cli - working-directory: datafusion-cli - run: cargo msrv verify + # # Verify MSRV for the crates which are directly used by other projects. + # msrv: + # name: Verify MSRV + # runs-on: ubuntu-latest + # container: + # image: amd64/rust + # steps: + # - uses: actions/checkout@v4 + # - name: Setup Rust toolchain + # uses: ./.github/actions/setup-builder + # - name: Install cargo-msrv + # run: cargo install cargo-msrv + # - name: Check datafusion + # working-directory: datafusion/core + # run: cargo msrv verify + # - name: Check datafusion-substrait + # working-directory: datafusion/substrait + # run: cargo msrv verify + # - name: Check datafusion-proto + # working-directory: datafusion/proto + # run: cargo msrv verify + # - name: Check datafusion-cli + # working-directory: datafusion-cli + # run: cargo msrv verify From 2b006c5e58e4173de25028802328cdc1a7f62140 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:09:05 +1100 Subject: [PATCH 04/39] test --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index dd3bc98216f5..a00d80c402c0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,7 +290,7 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-yyyy-mm-dd + rustup toolchain install nightly-2023-10-16 rustup default nightly-2023-10-16 rustup component add rustfmt - name: Run tests (excluding doctests) From a358c5aae2f35daa329dbdd8763fda246433e60d Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:14:56 +1100 Subject: [PATCH 05/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a00d80c402c0..d5360fe5c23d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-16 - rustup default nightly-2023-10-16 + rustup toolchain install nightly-2023-10-20 + rustup default nightly-2023-10-20 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From f17e064711e49b57cdb9fb3569d81dd1a0fe41c6 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:20:47 +1100 Subject: [PATCH 06/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d5360fe5c23d..e6c7c44cd8a1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-20 - rustup default nightly-2023-10-20 + rustup toolchain install nightly-2023-11-20 + rustup default nightly-2023-11-20 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From efe74c60e75948cb9332a693fe3b158122e636b4 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:26:47 +1100 Subject: [PATCH 07/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e6c7c44cd8a1..899798601933 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-11-20 - rustup default nightly-2023-11-20 + rustup toolchain install 1.75.0 + rustup default 1.75.0 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 9efc1d57912d32c3dd1d05301fde9ce0cd683415 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:34:09 +1100 Subject: [PATCH 08/39] test --- .github/workflows/rust.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 899798601933..9d35631e3ebb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -296,7 +296,8 @@ jobs: - name: Run tests (excluding doctests) shell: bash run: | - cargo test -p datafusion --test tpcds_planning + export PATH=$PATH:$HOME/d/protoc/bin + cargo test --lib --tests --bins --features avro,json,backtrace env: # do not produce debug symbols to keep memory usage down RUSTFLAGS: "-C debuginfo=0" From 5931a34335c3c14e24c00a0e77eaf842aa36e2cd Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:53:10 +1100 Subject: [PATCH 09/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9d35631e3ebb..bbbc6f645aa2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install 1.75.0 - rustup default 1.75.0 + rustup toolchain install nightly-2023-10-16 + rustup default nightly-2023-10-16 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 3da2aa5e1dcbb5ea43225ccde78b6cbb3df9b1d2 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 16:12:16 +1100 Subject: [PATCH 10/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bbbc6f645aa2..d4708c20651c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-16 - rustup default nightly-2023-10-16 + rustup toolchain install nightly-2023-10-01 + rustup default nightly-2023-10-01 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 48d62a12358e2685a8ea07e55b6b29859f716d4d Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 16:31:14 +1100 Subject: [PATCH 11/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d4708c20651c..96bea8ab2839 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-01 - rustup default nightly-2023-10-01 + rustup toolchain install nightly-2023-09-20 + rustup default nightly-2023-09-20 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From b213b55105befdc672877660f5d5bc40febec6ba Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 16:35:11 +1100 Subject: [PATCH 12/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 96bea8ab2839..7affd39e212e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-09-20 - rustup default nightly-2023-09-20 + rustup toolchain install nightly-2023-10-25 + rustup default nightly-2023-10-25 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From b8b445aa61b03a4150db7bc57a64f9d79c1a10e5 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 16:53:00 +1100 Subject: [PATCH 13/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7affd39e212e..e2bf13ca770d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-25 - rustup default nightly-2023-10-25 + rustup toolchain install nightly-2023-11-15 + rustup default nightly-2023-11-15 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From ef96467c38c977c14d1271746b1d651345127aaa Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 17:16:45 +1100 Subject: [PATCH 14/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e2bf13ca770d..e644b58bf0f0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-11-15 - rustup default nightly-2023-11-15 + rustup toolchain install nightly-2023-11-05 + rustup default nightly-2023-11-05 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 632081e1b2126e25e0cb2bd9000c8e58a54d2697 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 17:38:58 +1100 Subject: [PATCH 15/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e644b58bf0f0..350eb0b6c846 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-11-05 - rustup default nightly-2023-11-05 + rustup toolchain install nightly-2023-11-01 + rustup default nightly-2023-11-01 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From bfb94bce1d81aeeab41dafbefeef581164d1f345 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 18:00:31 +1100 Subject: [PATCH 16/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 350eb0b6c846..c2b8beeffe7d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-11-01 - rustup default nightly-2023-11-01 + rustup toolchain install nightly-2023-10-28 + rustup default nightly-2023-10-28 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 8926c54168ffbf225e3c9deb6e47b323b8998c05 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 18:18:33 +1100 Subject: [PATCH 17/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c2b8beeffe7d..8448645a4db2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-28 - rustup default nightly-2023-10-28 + rustup toolchain install nightly-2023-10-30 + rustup default nightly-2023-10-30 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From c85e67c5a77d5166bd11e9b31db3b9189c4d0325 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 18:41:02 +1100 Subject: [PATCH 18/39] test --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8448645a4db2..e056c8b3b8b3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-30 - rustup default nightly-2023-10-30 + rustup toolchain install nightly-2023-10-29 + rustup default nightly-2023-10-29 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 2e99e2151e1bb34dd16c40c364a8e1363346d422 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Wed, 3 Jan 2024 19:05:47 +1100 Subject: [PATCH 19/39] bad --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e056c8b3b8b3..8448645a4db2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-29 - rustup default nightly-2023-10-29 + rustup toolchain install nightly-2023-10-30 + rustup default nightly-2023-10-30 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From f67bfce0e667713d5990f630b5d21fc8ca1ef06b Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:34:28 +1100 Subject: [PATCH 20/39] isolate test --- .github/workflows/rust.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8448645a4db2..07d1e4081edd 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,14 +290,15 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-30 - rustup default nightly-2023-10-30 + rustup toolchain install nightly-2023-10-29 + rustup default nightly-2023-10-29 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash run: | export PATH=$PATH:$HOME/d/protoc/bin - cargo test --lib --tests --bins --features avro,json,backtrace + # cargo test --lib --tests --bins --features avro,json,backtrace + cargo test -p datafusion --test tpcds_planning --features backtrace env: # do not produce debug symbols to keep memory usage down RUSTFLAGS: "-C debuginfo=0" From 4ef39d05f2a3d971e2cf07ab6955f2ea78187f06 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:48:56 +1100 Subject: [PATCH 21/39] isolate test bad --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 07d1e4081edd..a0450471d58e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-29 - rustup default nightly-2023-10-29 + rustup toolchain install nightly-2023-10-30 + rustup default nightly-2023-10-30 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 9ea64d4b6d391321783653877db10f056d1991a7 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 10:11:30 +1100 Subject: [PATCH 22/39] isolate test bad narrow to physical q44 --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a0450471d58e..a859b502132d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -298,7 +298,7 @@ jobs: run: | export PATH=$PATH:$HOME/d/protoc/bin # cargo test --lib --tests --bins --features avro,json,backtrace - cargo test -p datafusion --test tpcds_planning --features backtrace + cargo test -p datafusion --test tpcds_planning --features backtrace tpcds_physical_q44 env: # do not produce debug symbols to keep memory usage down RUSTFLAGS: "-C debuginfo=0" From b36c2044152a5c1823826ac83ab631d81ea762fe Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 10:27:46 +1100 Subject: [PATCH 23/39] isolate test good narrow to physical q44 --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a859b502132d..47f71241c63c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-30 - rustup default nightly-2023-10-30 + rustup toolchain install nightly-2023-10-29 + rustup default nightly-2023-10-29 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From b1f125b61a7977011b68f3237477a155156648b6 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 10:35:31 +1100 Subject: [PATCH 24/39] physical q44 bad no backtrace --- .github/workflows/rust.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 47f71241c63c..ca59120d64a5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,15 +290,15 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-29 - rustup default nightly-2023-10-29 + rustup toolchain install nightly-2023-10-30 + rustup default nightly-2023-10-30 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash run: | export PATH=$PATH:$HOME/d/protoc/bin # cargo test --lib --tests --bins --features avro,json,backtrace - cargo test -p datafusion --test tpcds_planning --features backtrace tpcds_physical_q44 + cargo test -p datafusion --test tpcds_planning tpcds_physical_q44 env: # do not produce debug symbols to keep memory usage down RUSTFLAGS: "-C debuginfo=0" From c55a09c3175e92b3d7fd14825fc70940074be430 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 10:40:06 +1100 Subject: [PATCH 25/39] Trigger From 5e1fcddecabf4a126d04d5c2da511c340575d231 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 10:45:51 +1100 Subject: [PATCH 26/39] physical q44 good no backtrace --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ca59120d64a5..db0adf49e5ef 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-30 - rustup default nightly-2023-10-30 + rustup toolchain install nightly-2023-10-29 + rustup default nightly-2023-10-29 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 7ed65d21679a226da788327a37305fe048f23009 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 11:20:09 +1100 Subject: [PATCH 27/39] physical q44 bad with backtrace with stdout --- .github/workflows/rust.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index db0adf49e5ef..b1c7afae7339 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,15 +290,14 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-29 - rustup default nightly-2023-10-29 + rustup toolchain install nightly-2023-10-30 + rustup default nightly-2023-10-30 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash run: | export PATH=$PATH:$HOME/d/protoc/bin - # cargo test --lib --tests --bins --features avro,json,backtrace - cargo test -p datafusion --test tpcds_planning tpcds_physical_q44 + cargo test -p datafusion --test tpcds_planning tpcds_physical_q44 --features backtrace env: # do not produce debug symbols to keep memory usage down RUSTFLAGS: "-C debuginfo=0" From e29e092248e3e6664225bfd61755d551798ff7c6 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 12:19:32 +1100 Subject: [PATCH 28/39] physical q44 1.76.0 with backtrace --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b1c7afae7339..80e8e8029885 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-30 - rustup default nightly-2023-10-30 + rustup toolchain install nightly-2023-12-21 + rustup default nightly-2023-12-21 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From f6d493e5f49db4adb4c0b158b0bf8c2fbbabe379 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 12:44:03 +1100 Subject: [PATCH 29/39] physical q44 latest nightly with backtrace --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 80e8e8029885..301bf6747570 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,8 +290,8 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-12-21 - rustup default nightly-2023-12-21 + rustup toolchain install nightly-2024-01-01 + rustup default nightly-2024-01-01 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash From 0c7a6f88bccacafc764526ef3d9c37f6c589fcc2 Mon Sep 17 00:00:00 2001 From: Jefffrey <22608443+Jefffrey@users.noreply.github.com> Date: Thu, 4 Jan 2024 13:38:24 +1100 Subject: [PATCH 30/39] physical q44 bad with backtrace with stdout --- .github/workflows/rust.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 301bf6747570..a8e7bbbcfb83 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,14 +290,14 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2024-01-01 - rustup default nightly-2024-01-01 + rustup toolchain install nightly-2023-10-30 + rustup default nightly-2023-10-30 rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash run: | export PATH=$PATH:$HOME/d/protoc/bin - cargo test -p datafusion --test tpcds_planning tpcds_physical_q44 --features backtrace + cargo test -p datafusion --test tpcds_planning tpcds_physical_q44 --features backtrace -- --nocapture --exact env: # do not produce debug symbols to keep memory usage down RUSTFLAGS: "-C debuginfo=0" From 951063b67d0d9f97d01f77eb88019678e326f651 Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sat, 6 Jan 2024 14:57:25 +1100 Subject: [PATCH 31/39] try with debuginfo=line-tables-only --- .github/workflows/rust.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a8e7bbbcfb83..65f344fabc44 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -290,17 +290,17 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | - rustup toolchain install nightly-2023-10-30 - rustup default nightly-2023-10-30 + rustup toolchain install stable + rustup default stable rustup component add rustfmt - name: Run tests (excluding doctests) shell: bash run: | export PATH=$PATH:$HOME/d/protoc/bin - cargo test -p datafusion --test tpcds_planning tpcds_physical_q44 --features backtrace -- --nocapture --exact + cargo test -p datafusion --test tpcds_planning tpcds_physical_q44 --features backtrace env: # do not produce debug symbols to keep memory usage down - RUSTFLAGS: "-C debuginfo=0" + RUSTFLAGS: "-C debuginfo=line-tables-only" RUST_BACKTRACE: "1" # macos: From b93d7fca434b77ca2596c430ed1ad61ff6abe701 Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sat, 6 Jan 2024 15:10:12 +1100 Subject: [PATCH 32/39] Reenable other checks --- .github/workflows/rust.yml | 892 ++++++++++++++++++------------------- 1 file changed, 446 insertions(+), 446 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 65f344fabc44..e7c7d4560ce8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -40,234 +40,234 @@ on: jobs: # Check crate compiles - # linux-build-lib: - # name: cargo check - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable + linux-build-lib: + name: cargo check + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable - # - name: Cache Cargo - # uses: actions/cache@v3 - # with: - # path: | - # ~/.cargo/bin/ - # ~/.cargo/registry/index/ - # ~/.cargo/registry/cache/ - # ~/.cargo/git/db/ - # ./target/ - # ./datafusion-cli/target/ - # # this key equals the ones on `linux-build-lib` for re-use - # key: cargo-cache-benchmark-${{ hashFiles('datafusion/**/Cargo.toml', 'benchmarks/Cargo.toml', 'datafusion-cli/Cargo.toml') }} + - name: Cache Cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ./target/ + ./datafusion-cli/target/ + # this key equals the ones on `linux-build-lib` for re-use + key: cargo-cache-benchmark-${{ hashFiles('datafusion/**/Cargo.toml', 'benchmarks/Cargo.toml', 'datafusion-cli/Cargo.toml') }} - # - name: Check workspace without default features - # run: cargo check --no-default-features -p datafusion + - name: Check workspace without default features + run: cargo check --no-default-features -p datafusion - # - name: Check datafusion-common without default features - # run: cargo check --tests --no-default-features -p datafusion-common + - name: Check datafusion-common without default features + run: cargo check --tests --no-default-features -p datafusion-common - # - name: Check workspace in debug mode - # run: cargo check + - name: Check workspace in debug mode + run: cargo check - # - name: Check workspace with all features - # run: cargo check --workspace --benches --features avro,json - # - name: Check Cargo.lock for datafusion-cli - # run: | - # # If this test fails, try running `cargo update` in the `datafusion-cli` directory - # # and check in the updated Cargo.lock file. - # cargo check --manifest-path datafusion-cli/Cargo.toml --locked + - name: Check workspace with all features + run: cargo check --workspace --benches --features avro,json + - name: Check Cargo.lock for datafusion-cli + run: | + # If this test fails, try running `cargo update` in the `datafusion-cli` directory + # and check in the updated Cargo.lock file. + cargo check --manifest-path datafusion-cli/Cargo.toml --locked - # # test the crate - # linux-test: - # name: cargo test (amd64) - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Run tests (excluding doctests) - # run: cargo test --lib --tests --bins --features avro,json,backtrace - # - name: Verify Working Directory Clean - # run: git diff --exit-code + # test the crate + linux-test: + name: cargo test (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Run tests (excluding doctests) + run: cargo test --lib --tests --bins --features avro,json,backtrace + - name: Verify Working Directory Clean + run: git diff --exit-code - # linux-test-datafusion-cli: - # name: cargo test datafusion-cli (amd64) - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Run tests (excluding doctests) - # run: | - # cd datafusion-cli - # cargo test --lib --tests --bins --all-features - # - name: Verify Working Directory Clean - # run: git diff --exit-code + linux-test-datafusion-cli: + name: cargo test datafusion-cli (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Run tests (excluding doctests) + run: | + cd datafusion-cli + cargo test --lib --tests --bins --all-features + - name: Verify Working Directory Clean + run: git diff --exit-code - # linux-test-example: - # name: cargo examples (amd64) - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Run examples - # run: | - # # test datafusion-sql examples - # cargo run --example sql - # # test datafusion-examples - # ci/scripts/rust_example.sh - # - name: Verify Working Directory Clean - # run: git diff --exit-code + linux-test-example: + name: cargo examples (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Run examples + run: | + # test datafusion-sql examples + cargo run --example sql + # test datafusion-examples + ci/scripts/rust_example.sh + - name: Verify Working Directory Clean + run: git diff --exit-code - # # Run `cargo test doc` (test documentation examples) - # linux-test-doc: - # name: cargo test doc (amd64) - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Run doctests - # run: | - # cargo test --doc --features avro,json - # cd datafusion-cli - # cargo test --doc --all-features - # - name: Verify Working Directory Clean - # run: git diff --exit-code + # Run `cargo test doc` (test documentation examples) + linux-test-doc: + name: cargo test doc (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Run doctests + run: | + cargo test --doc --features avro,json + cd datafusion-cli + cargo test --doc --all-features + - name: Verify Working Directory Clean + run: git diff --exit-code - # # Run `cargo doc` to ensure the rustdoc is clean - # linux-rustdoc: - # name: cargo doc - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Run cargo doc - # run: | - # export RUSTDOCFLAGS="-D warnings -A rustdoc::private-intra-doc-links" - # cargo doc --document-private-items --no-deps --workspace - # cd datafusion-cli - # cargo doc --document-private-items --no-deps + # Run `cargo doc` to ensure the rustdoc is clean + linux-rustdoc: + name: cargo doc + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Run cargo doc + run: | + export RUSTDOCFLAGS="-D warnings -A rustdoc::private-intra-doc-links" + cargo doc --document-private-items --no-deps --workspace + cd datafusion-cli + cargo doc --document-private-items --no-deps - # linux-wasm-pack: - # name: build with wasm-pack - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Install wasm-pack - # run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - # - name: Build with wasm-pack - # working-directory: ./datafusion/wasmtest - # run: wasm-pack build --dev + linux-wasm-pack: + name: build with wasm-pack + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: Build with wasm-pack + working-directory: ./datafusion/wasmtest + run: wasm-pack build --dev - # # verify that the benchmark queries return the correct results - # verify-benchmark-results: - # name: verify benchmark results (amd64) - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Generate benchmark data and expected query results - # run: | - # mkdir -p datafusion/sqllogictest/test_files/tpch/data - # git clone https://github.com/databricks/tpch-dbgen.git - # cd tpch-dbgen - # make - # ./dbgen -f -s 0.1 - # mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data - # - name: Verify that benchmark queries return expected results - # run: | - # export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` - # # use release build for plan verificaton because debug build causes stack overflow - # cargo test plan_q --package datafusion-benchmarks --profile release-nonlto --features=ci -- --test-threads=1 - # INCLUDE_TPCH=true cargo test --test sqllogictests - # - name: Verify Working Directory Clean - # run: git diff --exit-code + # verify that the benchmark queries return the correct results + verify-benchmark-results: + name: verify benchmark results (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Generate benchmark data and expected query results + run: | + mkdir -p datafusion/sqllogictest/test_files/tpch/data + git clone https://github.com/databricks/tpch-dbgen.git + cd tpch-dbgen + make + ./dbgen -f -s 0.1 + mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data + - name: Verify that benchmark queries return expected results + run: | + export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` + # use release build for plan verificaton because debug build causes stack overflow + cargo test plan_q --package datafusion-benchmarks --profile release-nonlto --features=ci -- --test-threads=1 + INCLUDE_TPCH=true cargo test --test sqllogictests + - name: Verify Working Directory Clean + run: git diff --exit-code - # sqllogictest-postgres: - # name: "Run sqllogictest with Postgres runner" - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # services: - # postgres: - # image: postgres:15 - # env: - # POSTGRES_PASSWORD: postgres - # POSTGRES_DB: db_test - # POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=C --lc-ctype=C - # ports: - # - 5432/tcp - # options: >- - # --health-cmd pg_isready - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup toolchain - # run: | - # rustup toolchain install stable - # rustup default stable - # - name: Run sqllogictest - # run: PG_COMPAT=true PG_URI="postgresql://postgres:postgres@localhost:$POSTGRES_PORT/db_test" cargo test --features=postgres --test sqllogictests - # env: - # POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + sqllogictest-postgres: + name: "Run sqllogictest with Postgres runner" + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: db_test + POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=C --lc-ctype=C + ports: + - 5432/tcp + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup toolchain + run: | + rustup toolchain install stable + rustup default stable + - name: Run sqllogictest + run: PG_COMPAT=true PG_URI="postgresql://postgres:postgres@localhost:$POSTGRES_PORT/db_test" cargo test --features=postgres --test sqllogictests + env: + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} windows: name: cargo test (win64) @@ -303,247 +303,247 @@ jobs: RUSTFLAGS: "-C debuginfo=line-tables-only" RUST_BACKTRACE: "1" - # macos: - # name: cargo test (mac) - # runs-on: macos-latest - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Install protobuf compiler - # shell: bash - # run: | - # mkdir -p $HOME/d/protoc - # cd $HOME/d/protoc - # export PROTO_ZIP="protoc-21.4-osx-x86_64.zip" - # curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP - # unzip $PROTO_ZIP - # echo "$HOME/d/protoc/bin" >> $GITHUB_PATH - # export PATH=$PATH:$HOME/d/protoc/bin - # protoc --version - # # TODO: this won't cache anything, which is expensive. Setup this action - # # with a OS-dependent path. - # - name: Setup Rust toolchain - # run: | - # rustup toolchain install stable - # rustup default stable - # rustup component add rustfmt - # - name: Run tests (excluding doctests) - # shell: bash - # run: | - # cargo test --lib --tests --bins --features avro,json,backtrace - # cd datafusion-cli - # cargo test --lib --tests --bins --all-features - # env: - # # do not produce debug symbols to keep memory usage down - # RUSTFLAGS: "-C debuginfo=0" - # RUST_BACKTRACE: "1" + macos: + name: cargo test (mac) + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install protobuf compiler + shell: bash + run: | + mkdir -p $HOME/d/protoc + cd $HOME/d/protoc + export PROTO_ZIP="protoc-21.4-osx-x86_64.zip" + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + unzip $PROTO_ZIP + echo "$HOME/d/protoc/bin" >> $GITHUB_PATH + export PATH=$PATH:$HOME/d/protoc/bin + protoc --version + # TODO: this won't cache anything, which is expensive. Setup this action + # with a OS-dependent path. + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable + rustup component add rustfmt + - name: Run tests (excluding doctests) + shell: bash + run: | + cargo test --lib --tests --bins --features avro,json,backtrace + cd datafusion-cli + cargo test --lib --tests --bins --all-features + env: + # do not produce debug symbols to keep memory usage down + RUSTFLAGS: "-C debuginfo=0" + RUST_BACKTRACE: "1" - # test-datafusion-pyarrow: - # name: cargo test pyarrow (amd64) - # needs: [ linux-build-lib ] - # runs-on: ubuntu-20.04 - # container: - # image: amd64/rust:bullseye # Workaround https://github.com/actions/setup-python/issues/721 - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - uses: actions/setup-python@v5 - # with: - # python-version: "3.8" - # - name: Install PyArrow - # run: | - # echo "LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV - # python -m pip install pyarrow - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Run datafusion-common tests - # run: cargo test -p datafusion-common --features=pyarrow + test-datafusion-pyarrow: + name: cargo test pyarrow (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-20.04 + container: + image: amd64/rust:bullseye # Workaround https://github.com/actions/setup-python/issues/721 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-python@v5 + with: + python-version: "3.8" + - name: Install PyArrow + run: | + echo "LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV + python -m pip install pyarrow + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Run datafusion-common tests + run: cargo test -p datafusion-common --features=pyarrow - # vendor: - # name: Verify Vendored Code - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # - name: Run gen - # run: ./regen.sh - # working-directory: ./datafusion/proto - # - name: Verify workspace clean (if this fails, run ./datafusion/proto/regen.sh and check in results) - # run: git diff --exit-code + vendor: + name: Verify Vendored Code + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + - name: Run gen + run: ./regen.sh + working-directory: ./datafusion/proto + - name: Verify workspace clean (if this fails, run ./datafusion/proto/regen.sh and check in results) + run: git diff --exit-code - # check-fmt: - # name: Check cargo fmt - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Run - # run: | - # echo '' > datafusion/proto/src/generated/datafusion.rs - # ci/scripts/rust_fmt.sh + check-fmt: + name: Check cargo fmt + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Run + run: | + echo '' > datafusion/proto/src/generated/datafusion.rs + ci/scripts/rust_fmt.sh # Coverage job disabled due to # https://github.com/apache/arrow-datafusion/issues/3678 - # coverage: - # name: coverage - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Install protobuf compiler - # shell: bash - # run: | - # mkdir -p $HOME/d/protoc - # cd $HOME/d/protoc - # export PROTO_ZIP="protoc-21.4-linux-x86_64.zip" - # curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP - # unzip $PROTO_ZIP - # export PATH=$PATH:$HOME/d/protoc/bin - # protoc --version - # - name: Setup Rust toolchain - # run: | - # rustup toolchain install stable - # rustup default stable - # rustup component add rustfmt clippy - # - name: Cache Cargo - # uses: actions/cache@v3 - # with: - # path: /home/runner/.cargo - # # this key is not equal because the user is different than on a container (runner vs github) - # key: cargo-coverage-cache3- - # - name: Run coverage - # run: | - # export PATH=$PATH:$HOME/d/protoc/bin - # rustup toolchain install stable - # rustup default stable - # cargo install --version 0.20.1 cargo-tarpaulin - # cargo tarpaulin --all --out Xml - # - name: Report coverage - # continue-on-error: true - # run: bash <(curl -s https://codecov.io/bash) + coverage: + name: coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install protobuf compiler + shell: bash + run: | + mkdir -p $HOME/d/protoc + cd $HOME/d/protoc + export PROTO_ZIP="protoc-21.4-linux-x86_64.zip" + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + unzip $PROTO_ZIP + export PATH=$PATH:$HOME/d/protoc/bin + protoc --version + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable + rustup component add rustfmt clippy + - name: Cache Cargo + uses: actions/cache@v3 + with: + path: /home/runner/.cargo + # this key is not equal because the user is different than on a container (runner vs github) + key: cargo-coverage-cache3- + - name: Run coverage + run: | + export PATH=$PATH:$HOME/d/protoc/bin + rustup toolchain install stable + rustup default stable + cargo install --version 0.20.1 cargo-tarpaulin + cargo tarpaulin --all --out Xml + - name: Report coverage + continue-on-error: true + run: bash <(curl -s https://codecov.io/bash) - # clippy: - # name: clippy - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Install Clippy - # run: rustup component add clippy - # - name: Run clippy - # run: ci/scripts/rust_clippy.sh + clippy: + name: clippy + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Install Clippy + run: rustup component add clippy + - name: Run clippy + run: ci/scripts/rust_clippy.sh - # # Check answers are correct when hash values collide - # hash-collisions: - # name: cargo test hash collisions (amd64) - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Run tests - # run: | - # cd datafusion - # cargo test --lib --tests --features=force_hash_collisions,avro + # Check answers are correct when hash values collide + hash-collisions: + name: cargo test hash collisions (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Run tests + run: | + cd datafusion + cargo test --lib --tests --features=force_hash_collisions,avro - # cargo-toml-formatting-checks: - # name: check Cargo.toml formatting - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - name: Install cargo-tomlfmt - # run: which cargo-tomlfmt || cargo install cargo-tomlfmt + cargo-toml-formatting-checks: + name: check Cargo.toml formatting + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Install cargo-tomlfmt + run: which cargo-tomlfmt || cargo install cargo-tomlfmt - # - name: Check Cargo.toml formatting - # run: | - # # if you encounter an error, try running 'cargo tomlfmt -p path/to/Cargo.toml' to fix the formatting automatically. - # # If the error still persists, you need to manually edit the Cargo.toml file, which introduces formatting violation. - # # - # # ignore ./Cargo.toml because putting workspaces in multi-line lists make it easy to read - # ci/scripts/rust_toml_fmt.sh + - name: Check Cargo.toml formatting + run: | + # if you encounter an error, try running 'cargo tomlfmt -p path/to/Cargo.toml' to fix the formatting automatically. + # If the error still persists, you need to manually edit the Cargo.toml file, which introduces formatting violation. + # + # ignore ./Cargo.toml because putting workspaces in multi-line lists make it easy to read + ci/scripts/rust_toml_fmt.sh - # config-docs-check: - # name: check configs.md is up-to-date - # needs: [ linux-build-lib ] - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # with: - # rust-version: stable - # - uses: actions/setup-node@v4 - # with: - # node-version: "20" - # - name: Check if configs.md has been modified - # run: | - # # If you encounter an error, run './dev/update_config_docs.sh' and commit - # ./dev/update_config_docs.sh - # git diff --exit-code + config-docs-check: + name: check configs.md is up-to-date + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Check if configs.md has been modified + run: | + # If you encounter an error, run './dev/update_config_docs.sh' and commit + ./dev/update_config_docs.sh + git diff --exit-code - # # Verify MSRV for the crates which are directly used by other projects. - # msrv: - # name: Verify MSRV - # runs-on: ubuntu-latest - # container: - # image: amd64/rust - # steps: - # - uses: actions/checkout@v4 - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-builder - # - name: Install cargo-msrv - # run: cargo install cargo-msrv - # - name: Check datafusion - # working-directory: datafusion/core - # run: cargo msrv verify - # - name: Check datafusion-substrait - # working-directory: datafusion/substrait - # run: cargo msrv verify - # - name: Check datafusion-proto - # working-directory: datafusion/proto - # run: cargo msrv verify - # - name: Check datafusion-cli - # working-directory: datafusion-cli - # run: cargo msrv verify + # Verify MSRV for the crates which are directly used by other projects. + msrv: + name: Verify MSRV + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + - name: Install cargo-msrv + run: cargo install cargo-msrv + - name: Check datafusion + working-directory: datafusion/core + run: cargo msrv verify + - name: Check datafusion-substrait + working-directory: datafusion/substrait + run: cargo msrv verify + - name: Check datafusion-proto + working-directory: datafusion/proto + run: cargo msrv verify + - name: Check datafusion-cli + working-directory: datafusion-cli + run: cargo msrv verify From e196b5b05236d904aa19e551f4eda480a7f3c615 Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sat, 6 Jan 2024 15:12:22 +1100 Subject: [PATCH 33/39] Revert win CI check to running all tests --- .github/workflows/rust.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e7c7d4560ce8..c3270cb95339 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -297,7 +297,9 @@ jobs: shell: bash run: | export PATH=$PATH:$HOME/d/protoc/bin - cargo test -p datafusion --test tpcds_planning tpcds_physical_q44 --features backtrace + cargo test --lib --tests --bins --features avro,json,backtrace + cd datafusion-cli + cargo test --lib --tests --bins --all-features env: # do not produce debug symbols to keep memory usage down RUSTFLAGS: "-C debuginfo=line-tables-only" From c928ed6dc875b7879eec721ca7d06f7bf86aa4f9 Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sat, 6 Jan 2024 15:13:17 +1100 Subject: [PATCH 34/39] Disable coverage CI --- .github/workflows/rust.yml | 76 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c3270cb95339..c95bf182f72b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -399,44 +399,44 @@ jobs: # Coverage job disabled due to # https://github.com/apache/arrow-datafusion/issues/3678 - coverage: - name: coverage - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install protobuf compiler - shell: bash - run: | - mkdir -p $HOME/d/protoc - cd $HOME/d/protoc - export PROTO_ZIP="protoc-21.4-linux-x86_64.zip" - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP - unzip $PROTO_ZIP - export PATH=$PATH:$HOME/d/protoc/bin - protoc --version - - name: Setup Rust toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt clippy - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /home/runner/.cargo - # this key is not equal because the user is different than on a container (runner vs github) - key: cargo-coverage-cache3- - - name: Run coverage - run: | - export PATH=$PATH:$HOME/d/protoc/bin - rustup toolchain install stable - rustup default stable - cargo install --version 0.20.1 cargo-tarpaulin - cargo tarpaulin --all --out Xml - - name: Report coverage - continue-on-error: true - run: bash <(curl -s https://codecov.io/bash) + # coverage: + # name: coverage + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Install protobuf compiler + # shell: bash + # run: | + # mkdir -p $HOME/d/protoc + # cd $HOME/d/protoc + # export PROTO_ZIP="protoc-21.4-linux-x86_64.zip" + # curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + # unzip $PROTO_ZIP + # export PATH=$PATH:$HOME/d/protoc/bin + # protoc --version + # - name: Setup Rust toolchain + # run: | + # rustup toolchain install stable + # rustup default stable + # rustup component add rustfmt clippy + # - name: Cache Cargo + # uses: actions/cache@v3 + # with: + # path: /home/runner/.cargo + # # this key is not equal because the user is different than on a container (runner vs github) + # key: cargo-coverage-cache3- + # - name: Run coverage + # run: | + # export PATH=$PATH:$HOME/d/protoc/bin + # rustup toolchain install stable + # rustup default stable + # cargo install --version 0.20.1 cargo-tarpaulin + # cargo tarpaulin --all --out Xml + # - name: Report coverage + # continue-on-error: true + # run: bash <(curl -s https://codecov.io/bash) clippy: name: clippy From a50b0893519d74413c4b478c4e2e908983db2b5f Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sat, 6 Jan 2024 16:32:44 +1100 Subject: [PATCH 35/39] Change only debuginfo --- .github/workflows/rust.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c95bf182f72b..34fba68a7164 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -99,6 +99,14 @@ jobs: rust-version: stable - name: Run tests (excluding doctests) run: cargo test --lib --tests --bins --features avro,json,backtrace + env: + # do not produce debug symbols to keep memory usage down + # hardcoding other profile params to avoid profile override values + # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings + RUSTFLAGS: "-C debuginfo=line-tables-only -C opt-level=0 -C incremental=false -C codegen-units=256" + RUST_BACKTRACE: "1" + # avoid rust stack overflows on tpc-ds tests + RUST_MINSTACK: "3000000" - name: Verify Working Directory Clean run: git diff --exit-code @@ -290,6 +298,7 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | + rustup update stable rustup toolchain install stable rustup default stable rustup component add rustfmt @@ -302,9 +311,13 @@ jobs: cargo test --lib --tests --bins --all-features env: # do not produce debug symbols to keep memory usage down - RUSTFLAGS: "-C debuginfo=line-tables-only" + # use higher optimization level to overcome Windows rust slowness for tpc-ds + # and speed builds: https://github.com/apache/arrow-datafusion/issues/8696 + # Cargo profile docs https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings + RUSTFLAGS: "-C debuginfo=line-tables-only -C opt-level=1 -C target-feature=+crt-static -C incremental=false -C codegen-units=256" RUST_BACKTRACE: "1" - + # avoid rust stack overflows on tpc-ds tests + RUST_MINSTACK: "3000000" macos: name: cargo test (mac) runs-on: macos-latest @@ -327,6 +340,7 @@ jobs: # with a OS-dependent path. - name: Setup Rust toolchain run: | + rustup update stable rustup toolchain install stable rustup default stable rustup component add rustfmt @@ -338,8 +352,12 @@ jobs: cargo test --lib --tests --bins --all-features env: # do not produce debug symbols to keep memory usage down - RUSTFLAGS: "-C debuginfo=0" + # hardcoding other profile params to avoid profile override values + # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings + RUSTFLAGS: "-C debuginfo=line-tables-only -C opt-level=0 -C incremental=false -C codegen-units=256" RUST_BACKTRACE: "1" + # avoid rust stack overflows on tpc-ds tests + RUST_MINSTACK: "3000000" test-datafusion-pyarrow: name: cargo test pyarrow (amd64) From 1ddcd05785069d4f10b2b63018c2f990b5b0e9da Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sat, 6 Jan 2024 17:10:26 +1100 Subject: [PATCH 36/39] Commens --- .github/workflows/rust.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 34fba68a7164..bfab1e93a4c3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -103,7 +103,7 @@ jobs: # do not produce debug symbols to keep memory usage down # hardcoding other profile params to avoid profile override values # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings - RUSTFLAGS: "-C debuginfo=line-tables-only -C opt-level=0 -C incremental=false -C codegen-units=256" + RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256" RUST_BACKTRACE: "1" # avoid rust stack overflows on tpc-ds tests RUST_MINSTACK: "3000000" @@ -314,6 +314,8 @@ jobs: # use higher optimization level to overcome Windows rust slowness for tpc-ds # and speed builds: https://github.com/apache/arrow-datafusion/issues/8696 # Cargo profile docs https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings + # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build + # See for more details: https://github.com/rust-lang/rust/issues/119560 RUSTFLAGS: "-C debuginfo=line-tables-only -C opt-level=1 -C target-feature=+crt-static -C incremental=false -C codegen-units=256" RUST_BACKTRACE: "1" # avoid rust stack overflows on tpc-ds tests @@ -354,7 +356,7 @@ jobs: # do not produce debug symbols to keep memory usage down # hardcoding other profile params to avoid profile override values # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings - RUSTFLAGS: "-C debuginfo=line-tables-only -C opt-level=0 -C incremental=false -C codegen-units=256" + RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256" RUST_BACKTRACE: "1" # avoid rust stack overflows on tpc-ds tests RUST_MINSTACK: "3000000" From 37705947d9013c925906580bc1d35dd2df99cd32 Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sat, 6 Jan 2024 18:14:39 +1100 Subject: [PATCH 37/39] win remove other rustflag optimizations --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bfab1e93a4c3..7115171dd81d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -316,7 +316,7 @@ jobs: # Cargo profile docs https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build # See for more details: https://github.com/rust-lang/rust/issues/119560 - RUSTFLAGS: "-C debuginfo=line-tables-only -C opt-level=1 -C target-feature=+crt-static -C incremental=false -C codegen-units=256" + RUSTFLAGS: "-C debuginfo=line-tables-only" RUST_BACKTRACE: "1" # avoid rust stack overflows on tpc-ds tests RUST_MINSTACK: "3000000" From 9afd34982810b77e5a94418c0cfad18f03618183 Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sat, 6 Jan 2024 19:27:04 +1100 Subject: [PATCH 38/39] Fix comments --- .github/workflows/rust.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7115171dd81d..e2faa75e7bc4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -310,10 +310,7 @@ jobs: cd datafusion-cli cargo test --lib --tests --bins --all-features env: - # do not produce debug symbols to keep memory usage down - # use higher optimization level to overcome Windows rust slowness for tpc-ds - # and speed builds: https://github.com/apache/arrow-datafusion/issues/8696 - # Cargo profile docs https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings + # Minimize producing debug symbols to keep memory usage down # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build # See for more details: https://github.com/rust-lang/rust/issues/119560 RUSTFLAGS: "-C debuginfo=line-tables-only" From d24f851822ea445d04a86aa3d235e775ec79d9bd Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sat, 6 Jan 2024 19:48:55 +1100 Subject: [PATCH 39/39] Trigger