Skip to content

Commit

Permalink
Address comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhuqi-lucas
zhuqi-lucas committed Feb 11, 2025
1 parent 313b348 commit 0e7dac6
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 198 deletions.
5 changes: 3 additions & 2 deletions datafusion/common/src/dfschema.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1046,7 +1046,8 @@ impl SchemaExt for Schema {
.iter()
.zip(other.fields().iter())
.try_for_each(|(f1, f2)| {
if f1.is_nullable() != f2.is_nullable() {
// we only need to refuse the case where the table field is not nullable but the query field is nullable
if !f1.is_nullable() && f2.is_nullable() {
return Err(format!(
"Inserting query must have the same schema nullability as the table. \
Expected table field '{}' nullability: {}, got field: '{}', nullability: {}",
Expand Down Expand Up @@ -1097,7 +1098,7 @@ mod tests {
Column names are case sensitive. \
You can use double quotes to refer to the \"\"t1.c0\"\" column \
or set the datafusion.sql_parser.enable_ident_normalization configuration. \
Did you mean 't1.c0'?.";
Valid fields are t1.c0, t1.c1.";
assert_eq!(err.strip_backtrace(), expected);
Ok(())
}
Expand Down
87 changes: 2 additions & 85 deletions datafusion/sqllogictest/test_files/aggregate_skip_partial.slt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ CREATE EXTERNAL TABLE aggregate_test_100 (
c2 TINYINT NOT NULL,
c3 SMALLINT NOT NULL,
c4 SMALLINT,
c5 INT NOT NULL,
c5 INT,
c6 BIGINT NOT NULL,
c7 SMALLINT NOT NULL,
c8 INT NOT NULL,
Expand Down Expand Up @@ -228,7 +228,7 @@ CREATE TABLE aggregate_test_100_null (
c11 FLOAT
);

statement ok
statement error DataFusion error: Error during planning: Inserting query must have the same schema nullability as the table\. Expected table field 'c5' nullability: false, got field: 'c5', nullability: true
INSERT INTO aggregate_test_100_null
SELECT
c2,
Expand Down Expand Up @@ -301,21 +301,11 @@ SELECT c2, approx_distinct(c1), approx_distinct(c5) FROM aggregate_test_100 GROU
query III
SELECT c2, count(c3), count(c11) FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 19 17
2 17 19
3 15 13
4 16 19
5 12 11

# Test min / max with nullable fields
query IIIRR
SELECT c2, min(c3), max(c3), min(c11), max(c11) FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 -99 125 0.064453244 0.89651865
2 -117 122 0.09683716 0.8315913
3 -101 123 0.034291923 0.94669616
4 -117 123 0.028003037 0.7085086
5 -101 118 0.12559289 0.87989986

# Test sum with nullable fields
query IIR
Expand All @@ -331,31 +321,16 @@ SELECT c2, sum(c3), sum(c11) FROM aggregate_test_100 GROUP BY c2 ORDER BY c2;
query IIR
SELECT c2, median(c3), median(c11) FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 12 0.6067944
2 1 0.46076488
3 14 0.40154034
4 -17 0.48515016
5 -35 0.5536642

# Test approx_median with nullable fields
query IIR
SELECT c2, approx_median(c3), approx_median(c11) FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 12 0.6067944
2 1 0.46076488
3 14 0.40154034
4 -7 0.48515016
5 -39 0.5536642

# Test approx_distinct with nullable fields
query II
SELECT c2, approx_distinct(c3) FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 19
2 16
3 13
4 16
5 12

# Test avg for tinyint / float
query TRR
Expand Down Expand Up @@ -492,11 +467,6 @@ SELECT c2,
COUNT(c11) FILTER(WHERE c5 > 0)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 11 6
2 6 6
3 8 6
4 11 14
5 8 7

# Test avg for tinyint / float
query TRR
Expand All @@ -519,11 +489,6 @@ SELECT c2,
COUNT(c11) FILTER(WHERE c3 > 0)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 10 9
2 7 8
3 3 6
4 3 7
5 6 3

# Test min / max with nullable fields and filter
query IIIRR
Expand All @@ -534,11 +499,6 @@ SELECT c2,
MAX(c11) FILTER (WHERE c5 < 0)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 -99 103 0.2578469 0.89651865
2 -48 93 0.09683716 0.8315913
3 -76 123 0.034291923 0.94669616
4 -117 123 0.06563997 0.57360977
5 -94 68 0.12559289 0.75173044

# Test min / max with nullable fields and nullable filter
query III
Expand All @@ -547,11 +507,6 @@ SELECT c2,
MAX(c3) FILTER (WHERE c11 > 0.5)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 -99 125
2 -106 122
3 -76 73
4 -117 47
5 -82 118

# Test sum with nullable field and nullable / non-nullable filters
query IIIRR
Expand All @@ -562,35 +517,20 @@ SELECT c2,
SUM(c11) FILTER (WHERE c3 > 0)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 -3 77 7.214695632458 5.085060358047
2 100 77 6.197732746601 3.150197088718
3 109 211 2.80575042963 2.80632930994
4 -171 56 2.10740506649 1.939846396446
5 -86 -76 1.8741710186 1.600569307804

# Test approx_distinct with nullable fields and filter
query II
SELECT c2,
approx_distinct(c3) FILTER (WHERE c5 > 0)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 11
2 6
3 6
4 11
5 8

# Test approx_distinct with nullable fields and nullable filter
query II
SELECT c2,
approx_distinct(c3) FILTER (WHERE c11 > 0.5)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 10
2 6
3 3
4 3
5 6

# Test median with nullable fields and filter
query IIR
Expand All @@ -599,23 +539,13 @@ SELECT c2,
median(c11) FILTER (WHERE c5 < 0)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 -5 0.6623719
2 15 0.52930677
3 13 0.32792538
4 -38 0.49774808
5 -18 0.49842384

# Test min / max with nullable fields and nullable filter
query II
SELECT c2,
median(c3) FILTER (WHERE c11 > 0.5)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 33
2 -29
3 22
4 -90
5 -22

# Test approx_median with nullable fields and filter
query IIR
Expand All @@ -624,23 +554,13 @@ SELECT c2,
approx_median(c11) FILTER (WHERE c5 < 0)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 -5 0.6623719
2 12 0.52930677
3 13 0.32792538
4 -38 0.49774808
5 -21 0.47652745

# Test approx_median with nullable fields and nullable filter
query II
SELECT c2,
approx_median(c3) FILTER (WHERE c11 > 0.5)
FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;
----
1 35
2 -29
3 22
4 -90
5 -32

statement ok
DROP TABLE aggregate_test_100_null;
Expand Down Expand Up @@ -711,6 +631,3 @@ true false false false false true false NULL

statement ok
DROP TABLE aggregate_test_100_bool

statement ok
DROP TABLE aggregate_test_100
Loading

0 comments on commit 0e7dac6

Please sign in to comment.