Merge branch 'main' into wcy/config_http_for_opendal

.git-blame-ignore-revs

@@ -39,3 +39,6 @@ d70dba827c303373f3220c9733f7c7443e5c2d37
 
 # chore: cargo +nightly fmt (#13162) (format let-chains)
 c583e2c6c054764249acf484438c7bf7197765f4
+
+# chore: replace all ProstXxx with PbXxx (#8621)
+6fd8821f2e053957b183d648bea9c95b6703941f

.github/pr-title-checker-config.json

@@ -4,7 +4,7 @@
     "color": "B60205"
   },
   "CHECKS": {
-    "regexp": "^(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert|deprecate|cherry pick|cherry-pick)(\\(.*\\))?:.*",
+    "regexp": "^(cherry pick|cherry-pick)?(| )+(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert|deprecate)(\\(.*\\))?:.*",
     "ignoreLabels" : ["ignore-title"]
   },
   "MESSAGES": {

Makefile.toml

@@ -86,9 +86,9 @@ if ${is_not_ci}
   no_rust_log = not ${rust_log}
 
   if ${no_rust_log}
-    set_env RUST_LOG "pgwire_query_log=info"
+    set_env RUST_LOG "pgwire_query_log=info,hyper::client::connect::http=info"
   else
-    set_env RUST_LOG "pgwire_query_log=info,${rust_log}"
+    set_env RUST_LOG "pgwire_query_log=info,hyper::client::connect::http=info,${rust_log}"
   end
 end
 

ci/docker-compose.yml

@@ -94,6 +94,7 @@ services:
       - redis-server
       - pulsar-server
       - cassandra-server
+      - doris-server
       - starrocks-fe-server
       - starrocks-be-server
     volumes:
@@ -188,6 +189,18 @@ services:
     environment:
       - CASSANDRA_CLUSTER_NAME=cloudinfra
 
+  doris-server:
+    container_name: doris-server
+    image: apache/doris:doris-all-in-one-2.1.0
+    ports:
+      - 8030:8030
+      - 8040:8040
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9030"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+
   starrocks-fe-server:
     container_name: starrocks-fe-server
     image: starrocks/fe-ubuntu:3.1.7

ci/scripts/e2e-doris-sink-test.sh

@@ -30,8 +30,8 @@ sleep 1
 echo "--- create doris table"
 apt-get update -y && apt-get install -y mysql-client
 sleep 2
-mysql -uroot -P 9030 -h doris-fe-server -e "CREATE database demo;use demo;
-CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 datev2,v8 datetime,v9 boolean) UNIQUE KEY(\`v1\`)
+mysql -uroot -P 9030 -h doris-server -e "CREATE database demo;use demo;
+CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 datev2,v8 datetime,v9 boolean,v10 json) UNIQUE KEY(\`v1\`)
 DISTRIBUTED BY HASH(\`v1\`) BUCKETS 1
 PROPERTIES (
     \"replication_allocation\" = \"tag.location.default: 1\"
@@ -43,11 +43,11 @@ sleep 2
 echo "--- testing sinks"
 sqllogictest -p 4566 -d dev './e2e_test/sink/doris_sink.slt'
 sleep 1
-mysql -uroot -P 9030 -h doris-fe-server -e "select * from demo.demo_bhv_table" > ./query_result.csv
+mysql -uroot -P 9030 -h doris-server -e "select * from demo.demo_bhv_table" > ./query_result.csv
 
 
 if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{
-    exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01" && $9 == 0); }'; then
+    exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01" && $9 == 0 && $10 == "{\"a\":1}"); }'; then
   echo "Doris sink check passed"
 else
   cat ./query_result.csv
@@ -56,4 +56,4 @@ else
 fi
 
 echo "--- Kill cluster"
-cargo make ci-kill
+cargo make ci-kill

ci/scripts/multi-arch-docker.sh

@@ -74,8 +74,8 @@ if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
   pushGchr ${TAG}
 fi
 
-if [ "${BUILDKITE_SOURCE}" == "ui" ] && [[ -n "${IMAGE_TAG+x}" ]]; then
-  # If this is a ui build, tag the image with the $imagetag.
+if [[ -n "${IMAGE_TAG+x}" ]]; then
+  # Tag the image with the $IMAGE_TAG.
   TAG="${IMAGE_TAG}"
   pushGchr ${TAG}
 fi

ci/workflows/main-cron.yml

@@ -896,37 +896,24 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  # Causes ci error, close it first, fix it later
-  # - label: "set vm_max_map_count_2000000"
-  #   key: "set-vm_max_map_count"
-  #   if: |
-  #     !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-  #     || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
-  #     || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
-  #   command: "sudo sysctl -w vm.max_map_count=2000000"
-  #   depends_on:
-  #     - "build"
-  #     - "build-other"
-
-  # - label: "end-to-end doris sink test"
-  #   key: "e2e-doris-sink-tests"
-  #   command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release"
-  #   if: |
-  #     !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-  #     || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
-  #     || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
-  #   depends_on:
-  #     - "build"
-  #     - "build-other"
-  #     - "set-vm_max_map_count"
-  #   plugins:
-  #     - docker-compose#v5.1.0:
-  #         run: sink-doris-env
-  #         config: ci/docker-compose.yml
-  #         mount-buildkite-agent: true
-  #     - ./ci/plugins/upload-failure-logs
-  #   timeout_in_minutes: 10
-  #   retry: *auto-retry
+  - label: "end-to-end doris sink test"
+    key: "e2e-doris-sink-tests"
+    command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
+      || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: sink-test-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 10
+    retry: *auto-retry
 
   - label: "end-to-end starrocks sink test"
     key: "e2e-starrocks-sink-tests"

ci/workflows/pull-request.yml

@@ -308,30 +308,20 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  # Causes ci error, close it first, fix it later
-  # - label: "set vm_max_map_count_2000000"
-  #   key: "set-vm_max_map_count"
-  #   if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
-  #   command: "sudo sysctl -w vm.max_map_count=2000000"
-  #   depends_on:
-  #     - "build"
-  #     - "build-other"
-
-  # - label: "end-to-end doris sink test"
-  #   if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
-  #   command: "ci/scripts/e2e-doris-sink-test.sh -p ci-dev"
-  #   depends_on:
-  #     - "build"
-  #     - "build-other"
-  #     - "set-vm_max_map_count"
-  #   plugins:
-  #     - docker-compose#v5.1.0:
-  #         run: sink-doris-env
-  #         config: ci/docker-compose.yml
-  #         mount-buildkite-agent: true
-  #     - ./ci/plugins/upload-failure-logs
-  #   timeout_in_minutes: 10
-  #   retry: *auto-retry
+  - label: "end-to-end doris sink test"
+    if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
+    command: "ci/scripts/e2e-doris-sink-test.sh -p ci-dev"
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: sink-test-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 10
+    retry: *auto-retry
 
   - label: "end-to-end starrocks sink test"
     if: build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-starrocks-sink-tests?(,|$$)/

e2e_test/batch/catalog/pg_settings.slt.part

@@ -35,6 +35,7 @@ user rw_batch_enable_lookup_join
 user rw_batch_enable_sort_agg
 user rw_enable_join_ordering
 user rw_enable_share_plan
+user rw_enable_shared_source
 user rw_enable_two_phase_agg
 user rw_force_split_distinct_agg
 user rw_force_two_phase_agg

e2e_test/batch/transaction/cursor.slt

@@ -0,0 +1,95 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+create table a(aid int, c1 int);
+
+statement ok
+insert into a values(1, 11), (2, 12), (3, 13);
+
+statement ok
+create table test(a int,b varchar);
+
+statement ok
+insert into test values(1, 'hello'), (2, 'world'), (3, 'risingwave'), (4, 'labs');
+
+# Currently, we allow declaring cursors out of TRANSACTION, because we don't have RW txn support. In PG, it's not allowed
+statement ok
+DECLARE
+    test_cursor CURSOR FOR
+        SELECT * FROM test where a > 2 ORDER BY a;
+
+statement ok
+CLOSE test_cursor;
+
+statement ok
+START TRANSACTION ISOLATION LEVEL REPEATABLE READ;
+
+statement ok
+DECLARE
+    test_cursor CURSOR FOR
+        SELECT * FROM test where a > 2 ORDER BY a;
+
+statement error cursor "test_cursor" already exists
+DECLARE
+    test_cursor CURSOR FOR
+        SELECT * FROM test where a > 2;
+
+statement error table or source not found: test
+DECLARE
+    test_cursor CURSOR FOR
+        SELECT * FROM test_non_existent where a > 2;
+
+statement error cursor "test_cursor_non_existent" does not exist
+FETCH NEXT from test_cursor_non_existent;
+
+query II
+FETCH NEXT from test_cursor;
+----
+3 risingwave
+
+query II
+FETCH NEXT from test_cursor;
+----
+4 labs
+
+query II
+FETCH NEXT from test_cursor;
+----
+
+statement error cursor "test_cursor_non_existent" does not exist
+CLOSE test_cursor_non_existent;
+
+statement ok
+CLOSE test_cursor;
+
+statement error cursor "test_cursor" does not exist
+FETCH NEXT from test_cursor;
+
+statement ok
+DECLARE
+    test_cursor CURSOR FOR
+        SELECT * FROM test JOIN a ON test.a > 1 and a.aid = test.a ORDER BY test.a;
+
+query IIII
+FETCH NEXT from test_cursor;
+----
+2 world   2 12
+
+query IIII
+FETCH NEXT from test_cursor;
+----
+3 risingwave   3 13
+
+query IIII
+FETCH NEXT from test_cursor;
+----
+
+statement ok
+COMMIT;
+
+statement ok
+drop table test;
+
+statement ok
+drop table a;