MS SQL Server: troubles with timestamp pushdown

ydb-platform · Jan 29, 2025 · 98f6764 · 98f6764
1 parent 622b81d
commit 98f6764
Show file tree

Hide file tree

Showing 9 changed files with 110 additions and 16 deletions.
diff --git a/app/server/config/config.debug.yaml b/app/server/config/config.debug.yaml
@@ -38,7 +38,7 @@ datasources:
   clickhouse:
     <<: *data_source_default_var
     pushdown:
-      enable_timestamp_pushdown: false # https://github.com/ClickHouse/clickhouse-go/issues/1483
+      enable_timestamp_pushdown: false # YQ-4063
 
   greenplum:
     <<: *data_source_default_var
@@ -47,6 +47,8 @@ datasources:
 
   ms_sql_server:
     <<: *data_source_default_var
+    pushdown:
+      enable_timestamp_pushdown: false # YQ-4062
 
   mysql:
     <<: *data_source_default_var
@@ -61,6 +63,8 @@ datasources:
 
   oracle:
     <<: *data_source_default_var
+    pushdown:
+      enable_timestamp_pushdown: false # YQ-3527
 
   ydb:
     <<: *data_source_default_var

diff --git a/app/server/datasource/rdbms/mysql/connection.go b/app/server/datasource/rdbms/mysql/connection.go
@@ -27,8 +27,9 @@ func (c *connection) Close() error {
 	return c.conn.Close()
 }
 
-func transformArgs(src *rdbms_utils.QueryArgs) []interface{} {
-	dst := make([]interface{}, len(src.Values()))
+func transformArgs(src *rdbms_utils.QueryArgs) []any {
+	dst := make([]any, len(src.Values()))
+
 	for i, v := range src.Values() {
 		switch t := v.(type) {
 		// MySQL driver does not accept time.Time objects
@@ -39,6 +40,7 @@ func transformArgs(src *rdbms_utils.QueryArgs) []interface{} {
 			dst[i] = v
 		}
 	}
+
 	return dst
 }
 

diff --git a/tests/infra/datasource/clickhouse/suite.go b/tests/infra/datasource/clickhouse/suite.go
@@ -33,8 +33,7 @@ func (s *Suite) TestDatetimeFormatYQL() {
 	)
 }
 
-// FIXME: `DateTime64` pushdown doesn't work correctly
-// https://github.com/ClickHouse/clickhouse-go/issues/1483
+// FIXME: YQ-4063
 /*
 func (s *Suite) TestPushdownTimestampEQ() {
 	t := time.Date(1988, 11, 20, 12, 55, 28, 123456000, time.UTC)

diff --git a/tests/infra/datasource/ms_sql_server/init/setup.sql b/tests/infra/datasource/ms_sql_server/init/setup.sql
@@ -39,7 +39,7 @@ CREATE TABLE primitives (
 
 INSERT INTO primitives VALUES
     (0, 1, 2, 3, 4, 5, 6.6, 7.7, 'az', 'az', 'az', 'az', 'az', 'az', 0x1234567890ABCDEF, 0x1234567890ABCDEF, 0x1234567890ABCDEF,
-    '1988-11-20', '1988-11-20 12:55:00', '1988-11-20 12:55:28.123', '1988-11-20 12:55:28.1231231'),
+    '1988-11-20', '1988-11-20 12:55:00', '1988-11-20 12:55:28.123', '1988-11-20 12:55:28.1231230'),
     (1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 
     NULL, NULL, NULL, NULL),
     (2, 0, 2, -3, -4, -5, -6.6, -7.7, N'буки', N'буки', N'буки', N'буки', N'буки', N'буки', 0x1234567890ABCDEF, 0x1234567890ABCDEF, 0x1234567890ABCDEF,
@@ -58,7 +58,7 @@ CREATE TABLE datetimes (
 
 INSERT INTO datetimes VALUES 
     (1, '1950-05-27', '1950-05-27 01:02:00', '1950-05-27 01:02:03.110', '1950-05-27 01:02:03.1111111'),
-    (2, '1988-11-20', '1988-11-20 12:55:00', '1988-11-20 12:55:28.123', '1988-11-20 12:55:28.1231231'),
+    (2, '1988-11-20', '1988-11-20 12:55:00', '1988-11-20 12:55:28.123', '1988-11-20 12:55:28.1231230'),
     (3, '2023-03-21', '2023-03-21 11:21:00', '2023-03-21 11:21:31', '2023-03-21 11:21:31');
 
 SELECT * FROM datetimes;

diff --git a/tests/infra/datasource/ms_sql_server/suite.go b/tests/infra/datasource/ms_sql_server/suite.go
@@ -1,6 +1,8 @@
 package ms_sql_server
 
 import (
+	"time"
+
 	"github.com/apache/arrow/go/v13/arrow/array"
 	"github.com/ydb-platform/ydb-go-genproto/protos/Ydb"
 
@@ -33,6 +35,43 @@ func (s *Suite) TestDatetimeFormatYQL() {
 	)
 }
 
+// FIXME: YQ-4062
+/*
+func (s *Suite) TestPushdownTimestampToDatetimeEQ() {
+	t := time.Date(1988, 11, 20, 12, 55, 28, 123000000, time.UTC)
+
+	s.ValidateTable(
+		s.dataSource,
+		tables["datetime_format_yql_pushdown_timestamp_EQ"],
+		suite.WithPredicate(&api_service_protos.TPredicate{
+			Payload: tests_utils.MakePredicateComparisonColumn(
+				"col_03_datetime",
+				api_service_protos.TPredicate_TComparison_EQ,
+				common.MakeTypedValue(common.MakePrimitiveType(Ydb.Type_TIMESTAMP), t.UnixMicro()),
+			),
+		}),
+		suite.WithDateTimeFormat(api_service_protos.EDateTimeFormat_YQL_FORMAT),
+	)
+}
+*/
+
+func (s *Suite) TestPushdownTimestampToDatetime2EQ() {
+	t := time.Date(1988, 11, 20, 12, 55, 28, 123123000, time.UTC)
+
+	s.ValidateTable(
+		s.dataSource,
+		tables["datetime_format_yql_pushdown_timestamp_EQ"],
+		suite.WithPredicate(&api_service_protos.TPredicate{
+			Payload: tests_utils.MakePredicateComparisonColumn(
+				"col_04_datetime2",
+				api_service_protos.TPredicate_TComparison_EQ,
+				common.MakeTypedValue(common.MakePrimitiveType(Ydb.Type_TIMESTAMP), t.UnixMicro()),
+			),
+		}),
+		suite.WithDateTimeFormat(api_service_protos.EDateTimeFormat_YQL_FORMAT),
+	)
+}
+
 func (s *Suite) TestDatetimeFormatString() {
 	s.ValidateTable(
 		s.dataSource,

diff --git a/tests/infra/datasource/ms_sql_server/tables.go b/tests/infra/datasource/ms_sql_server/tables.go
@@ -128,7 +128,7 @@ var tables = map[string]*test_utils.Table[int32, *array.Int32Builder]{
 					},
 					"col_20_datetime2": []*uint64{
 						ptr.Uint64(common.MustTimeToYDBType(common.TimeToYDBTimestamp,
-							time.Date(1988, 11, 20, 12, 55, 28, 123123100, time.UTC))),
+							time.Date(1988, 11, 20, 12, 55, 28, 123123000, time.UTC))),
 						nil,
 						ptr.Uint64(common.MustTimeToYDBType(common.TimeToYDBTimestamp,
 							time.Date(2023, 03, 21, 11, 21, 31, 0, time.UTC))),
@@ -183,14 +183,54 @@ var tables = map[string]*test_utils.Table[int32, *array.Int32Builder]{
 					"col_04_datetime2": []*uint64{
 						nil,
 						ptr.Uint64(common.MustTimeToYDBType[uint64](
-							common.TimeToYDBTimestamp, time.Date(1988, 11, 20, 12, 55, 28, 123123100, time.UTC))),
+							common.TimeToYDBTimestamp, time.Date(1988, 11, 20, 12, 55, 28, 123123000, time.UTC))),
 						ptr.Uint64(common.MustTimeToYDBType[uint64](
 							common.TimeToYDBTimestamp, time.Date(2023, 03, 21, 11, 21, 31, 0, time.UTC))),
 					},
 				},
 			},
 		},
 	},
+	"datetime_format_yql_pushdown_timestamp_EQ": {
+		Name:                  "datetimes",
+		IDArrayBuilderFactory: newInt32IDArrayBuilder(memPool),
+		Schema: &test_utils.TableSchema{
+			Columns: map[string]*Ydb.Type{
+				"id":                   common.MakeOptionalType(common.MakePrimitiveType(Ydb.Type_INT32)),
+				"col_01_date":          common.MakeOptionalType(common.MakePrimitiveType(Ydb.Type_DATE)),
+				"col_02_smalldatetime": common.MakeOptionalType(common.MakePrimitiveType(Ydb.Type_DATETIME)),
+				"col_03_datetime":      common.MakeOptionalType(common.MakePrimitiveType(Ydb.Type_TIMESTAMP)),
+				"col_04_datetime2":     common.MakeOptionalType(common.MakePrimitiveType(Ydb.Type_TIMESTAMP)),
+			},
+		},
+		Records: []*test_utils.Record[int32, *array.Int32Builder]{
+			{
+				// In YQL mode, PG datetime values exceeding YQL date/datetime/timestamp type bounds
+				// are returned as NULL
+				Columns: map[string]any{
+					"id": []*int32{
+						ptr.Int32(2),
+					},
+					"col_01_date": []*uint16{
+						ptr.Uint16(common.MustTimeToYDBType[uint16](
+							common.TimeToYDBDate, time.Date(1988, 11, 20, 0, 0, 0, 0, time.UTC))),
+					},
+					"col_02_smalldatetime": []*uint32{
+						ptr.Uint32(common.MustTimeToYDBType[uint32](
+							common.TimeToYDBDatetime, time.Date(1988, 11, 20, 12, 55, 0, 0, time.UTC))),
+					},
+					"col_03_datetime": []*uint64{
+						ptr.Uint64(common.MustTimeToYDBType[uint64](
+							common.TimeToYDBTimestamp, time.Date(1988, 11, 20, 12, 55, 28, 123000000, time.UTC))),
+					},
+					"col_04_datetime2": []*uint64{
+						ptr.Uint64(common.MustTimeToYDBType[uint64](
+							common.TimeToYDBTimestamp, time.Date(1988, 11, 20, 12, 55, 28, 123123000, time.UTC))),
+					},
+				},
+			},
+		},
+	},
 	"datetime_format_string": {
 		Name:                  "datetimes",
 		IDArrayBuilderFactory: newInt32IDArrayBuilder(memPool),
@@ -230,7 +270,7 @@ var tables = map[string]*test_utils.Table[int32, *array.Int32Builder]{
 					},
 					"col_04_datetime2": []*string{
 						ptr.String("1950-05-27T01:02:03.1111111Z"),
-						ptr.String("1988-11-20T12:55:28.1231231Z"),
+						ptr.String("1988-11-20T12:55:28.123123Z"),
 						ptr.String("2023-03-21T11:21:31Z"),
 					},
 				},

diff --git a/tests/infra/datasource/mysql/suite.go b/tests/infra/datasource/mysql/suite.go
@@ -42,7 +42,7 @@ func (s *Suite) TestPushdownTimestampToDatetimeEQ() {
 
 	s.ValidateTable(
 		s.dataSource,
-		tables["datetime_format_yql_pushdown_time_EQ"],
+		tables["datetime_format_yql_pushdown_timestamp_EQ"],
 		suite.WithPredicate(&api_service_protos.TPredicate{
 			Payload: tests_utils.MakePredicateComparisonColumn(
 				"col_02_datetime",
@@ -59,7 +59,7 @@ func (s *Suite) TestPushdownTimestampToTimestampEQ() {
 
 	s.ValidateTable(
 		s.dataSource,
-		tables["datetime_format_yql_pushdown_time_EQ"],
+		tables["datetime_format_yql_pushdown_timestamp_EQ"],
 		suite.WithPredicate(&api_service_protos.TPredicate{
 			Payload: tests_utils.MakePredicateComparisonColumn(
 				"col_03_timestamp",

diff --git a/tests/infra/datasource/mysql/tables.go b/tests/infra/datasource/mysql/tables.go
@@ -187,7 +187,7 @@ var tables = map[string]*test_utils.Table[int32, *array.Int32Builder]{
 			},
 		},
 	},
-	"datetime_format_yql_pushdown_time_EQ": { // used in a couple of tests, so table name is generalized
+	"datetime_format_yql_pushdown_timestamp_EQ": {
 		Name:                  "datetimes",
 		IDArrayBuilderFactory: newInt32IDArrayBuilder(memPool),
 		Schema: &test_utils.TableSchema{

diff --git a/tools/clickhouse/timestamp_pushdown/main.go b/tools/clickhouse/timestamp_pushdown/main.go
@@ -13,14 +13,15 @@ import (
 func main() {
 	// Define ClickHouse connection info
 	connStr := "clickhouse://admin:password@localhost:9000"
+
 	db, err := sql.Open("clickhouse", connStr)
 	if err != nil {
 		log.Fatalf("failed to open connection: %v", err)
 	}
 	defer db.Close()
 
 	// Check the connection
-	if err := db.Ping(); err != nil {
+	if err = db.Ping(); err != nil {
 		log.Fatalf("failed to ping database: %v", err)
 	}
 
@@ -34,6 +35,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("failed to drop table: %v", err)
 	}
+
 	fmt.Println("Table dropped successfully.")
 
 	// 2. Create a table
@@ -61,23 +63,31 @@ func main() {
 	if err != nil {
 		log.Fatalf("failed to insert data: %v", err)
 	}
+
 	fmt.Println("Data inserted successfully.")
 
 	// 4. Query the table with a filtering expression
 	timeValue := time.Date(1988, 11, 20, 12, 55, 28, 123456000, time.UTC)
+
 	rows, err := db.QueryContext(ctx, "SELECT id, datetimeValue FROM example_table WHERE datetimeValue = ?", timeValue)
 	if err != nil {
 		log.Fatalf("failed to execute query: %v", err)
 	}
+
 	defer rows.Close()
 
 	fmt.Println("Rows filtered and fetched:")
+
 	for rows.Next() {
-		var id uint32
-		var datetimeValue time.Time
+		var (
+			id            uint32
+			datetimeValue time.Time
+		)
+
 		if err := rows.Scan(&id, &datetimeValue); err != nil {
 			log.Fatalf("failed to scan row: %v", err)
 		}
+
 		fmt.Printf("ID: %d, DateTime: %s\n", id, datetimeValue.Format(time.RFC3339Nano))
 	}