diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 6c9465bbc84c4..e631be322150d 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -1233,6 +1233,7 @@ class ApiGateway { ...query, measures: (query.measures || []).map(m => (typeof m === 'string' ? this.parseMemberExpression(m) : m)), dimensions: (query.dimensions || []).map(m => (typeof m === 'string' ? this.parseMemberExpression(m) : m)), + segments: (query.segments || []).map(m => (typeof m === 'string' ? this.parseMemberExpression(m) : m)), }; } diff --git a/packages/cubejs-api-gateway/src/query.js b/packages/cubejs-api-gateway/src/query.js index fc7e2e994730f..56f40e6b58dd8 100644 --- a/packages/cubejs-api-gateway/src/query.js +++ b/packages/cubejs-api-gateway/src/query.js @@ -105,7 +105,7 @@ const querySchema = Joi.object().keys({ Joi.object().pattern(id, Joi.valid('asc', 'desc')), Joi.array().items(Joi.array().min(2).ordered(id, Joi.valid('asc', 'desc'))) ), - segments: Joi.array().items(id), + segments: Joi.array().items(Joi.alternatives(id, memberExpression)), timezone: Joi.string(), limit: Joi.number().integer().min(1), offset: Joi.number().integer().min(0), diff --git a/packages/cubejs-api-gateway/src/types/query.ts b/packages/cubejs-api-gateway/src/types/query.ts index 9aa5cf1cc6248..e1083e827088f 100644 --- a/packages/cubejs-api-gateway/src/types/query.ts +++ b/packages/cubejs-api-gateway/src/types/query.ts @@ -64,7 +64,7 @@ interface Query { dimensions?: (Member | TimeMember | MemberExpression)[]; filters?: (QueryFilter | LogicalAndFilter | LogicalOrFilter)[]; timeDimensions?: QueryTimeDimension[]; - segments?: Member[]; + segments?: (Member | MemberExpression)[]; limit?: null | number; offset?: number; total?: boolean; diff --git a/packages/cubejs-backend-native/src/transport.rs b/packages/cubejs-backend-native/src/transport.rs index 9fb601caecf20..fc9d8d613989d 100644 --- a/packages/cubejs-backend-native/src/transport.rs +++ b/packages/cubejs-backend-native/src/transport.rs @@ -297,8 +297,8 @@ impl TransportService for NodeBridgeTransport { ) .await?; - if let Some(error) = response.get("error") { - if let Some(stack) = response.get("stack") { + if let Some(error) = response.get("error").and_then(|e| e.as_str()) { + if let Some(stack) = response.get("stack").and_then(|e| e.as_str()) { return Err(CubeError::user(format!( "Error during SQL generation: {}\n{}", error, stack diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts index 0a38daac050c5..7a53e77d58685 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts @@ -104,6 +104,7 @@ export class DatabricksQuery extends BaseQuery { templates.functions.LTRIM = 'LTRIM({{ args|reverse|join(", ") }})'; templates.functions.RTRIM = 'RTRIM({{ args|reverse|join(", ") }})'; templates.functions.DATEDIFF = 'DATEDIFF({{ date_part }}, DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }}), DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}))'; + templates.expressions.timestamp_literal = 'from_utc_timestamp(\'{{ value }}\', \'UTC\')'; return templates; } } diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index c2b1fedc19529..fb43334145ee3 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -2500,6 +2500,7 @@ export class BaseQuery { statements: { select: 'SELECT {{ select_concat | map(attribute=\'aliased\') | join(\', \') }} \n' + 'FROM (\n {{ from }}\n) AS {{ from_alias }} \n' + + '{% if filter %} WHERE {{ filter }}{% endif %}' + '{% if group_by %} GROUP BY {{ group_by | map(attribute=\'index\') | join(\', \') }}{% endif %}' + '{% if order_by %} ORDER BY {{ order_by | map(attribute=\'expr\') | join(\', \') }}{% endif %}' + '{% if limit %}\nLIMIT {{ limit }}{% endif %}' + diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts index c8251498c5c10..711c8a88d8f68 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts @@ -1,12 +1,34 @@ import type { BaseQuery } from './BaseQuery'; export class BaseSegment { + public readonly expression: any; + + public readonly expressionCubeName: any; + + public readonly expressionName: any; + + public readonly isMemberExpression: boolean = false; + public constructor( protected readonly query: BaseQuery, - public readonly segment: string - ) {} + public readonly segment: string | any + ) { + if (segment.expression) { + this.expression = segment.expression; + this.expressionCubeName = segment.cubeName; + this.expressionName = segment.expressionName || `${segment.cubeName}.${segment.name}`; + this.isMemberExpression = !!segment.definition; + } + } public filterToWhere() { + return this.segmentSql(); + } + + public segmentSql() { + if (this.expression) { + return this.query.evaluateSymbolSql(this.expressionCubeName, this.expressionName, this.definition(), 'segment'); + } return this.query.segmentSql(this); } @@ -19,6 +41,11 @@ export class BaseSegment { } public definition() { + if (this.expression) { + return { + sql: this.expression + }; + } return this.segmentDefinition(); } @@ -27,6 +54,9 @@ export class BaseSegment { } public cube() { + if (this.expression) { + return this.query.cubeEvaluator.cubeFromPath(this.expressionCubeName); + } return this.query.cubeEvaluator.cubeFromPath(this.segment); } @@ -35,14 +65,16 @@ export class BaseSegment { } public path() { + if (this.expression) { + return null; + } return this.query.cubeEvaluator.parsePath('segments', this.segment); } public expressionPath() { - // TODO expression support - // if (this.expression) { - // return `expr:${this.expression.expressionName}`; - // } + if (this.expression) { + return `expr:${this.expression.expressionName}`; + } return this.query.cubeEvaluator.pathFromArray(this.path()); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts b/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts index 5e082a9bff2f6..987345dd0a93e 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts @@ -160,6 +160,7 @@ export class BigqueryQuery extends BaseQuery { templates.expressions.binary = '{% if op == \'%\' %}MOD({{ left }}, {{ right }}){% else %}({{ left }} {{ op }} {{ right }}){% endif %}'; templates.expressions.interval = 'INTERVAL {{ interval }}'; templates.expressions.extract = 'EXTRACT({% if date_part == \'DOW\' %}DAYOFWEEK{% elif date_part == \'DOY\' %}DAYOFYEAR{% else %}{{ date_part }}{% endif %} FROM {{ expr }})'; + templates.expressions.timestamp_literal = 'TIMESTAMP(\'{{ value }}\')'; return templates; } } diff --git a/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts b/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts index e3d8f069bda86..e05441b6a4ba2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts @@ -58,6 +58,7 @@ export class PostgresQuery extends BaseQuery { templates.functions.DATEDIFF = 'CASE WHEN LOWER(\'{{ date_part }}\') IN (\'year\', \'quarter\', \'month\') THEN (EXTRACT(YEAR FROM AGE(DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}), DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }}))) * 12 + EXTRACT(MONTH FROM AGE(DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}), DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }})))) / CASE LOWER(\'{{ date_part }}\') WHEN \'year\' THEN 12 WHEN \'quarter\' THEN 3 WHEN \'month\' THEN 1 END ELSE EXTRACT(EPOCH FROM DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}) - DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }})) / EXTRACT(EPOCH FROM \'1 {{ date_part }}\'::interval) END::bigint'; templates.expressions.interval = 'INTERVAL \'{{ interval }}\''; templates.expressions.extract = 'EXTRACT({{ date_part }} FROM {{ expr }})'; + templates.expressions.timestamp_literal = 'timestamptz \'{{ value }}\''; return templates; } diff --git a/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts b/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts index 6c15bb8ab6cb4..8741a66083114 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts @@ -121,6 +121,7 @@ export class PrestodbQuery extends BaseQuery { '{% if limit %}\nLIMIT {{ limit }}{% endif %}'; templates.expressions.extract = 'EXTRACT({{ date_part }} FROM {{ expr }})'; templates.expressions.interval = 'INTERVAL \'{{ num }}\' {{ date_part }}'; + templates.expressions.timestamp_literal = 'from_iso8601_timestamp(\'{{ value }}\')'; return templates; } } diff --git a/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts b/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts index 8170d9b52ef99..4f0eb80b6e8e3 100644 --- a/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts @@ -62,6 +62,7 @@ export class SnowflakeQuery extends BaseQuery { templates.functions.BTRIM = 'TRIM({{ args_concat }})'; templates.expressions.extract = 'EXTRACT({{ date_part }} FROM {{ expr }})'; templates.expressions.interval = 'INTERVAL \'{{ interval }}\''; + templates.expressions.timestamp_literal = '\'{{ value }}\'::timestamp_tz'; return templates; } } diff --git a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs index ed09a7fcc63fa..fa1487da9ed1c 100644 --- a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs +++ b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs @@ -10,7 +10,7 @@ use crate::{ }, CubeError, }; -use chrono::{Days, NaiveDate}; +use chrono::{Days, NaiveDate, SecondsFormat, TimeZone, Utc}; use cubeclient::models::V1LoadRequestQuery; use datafusion::{ error::{DataFusionError, Result}, @@ -165,6 +165,48 @@ lazy_static! { static ref DATE_PART_REGEX: Regex = Regex::new("^[A-Za-z_ ]+$").unwrap(); } +macro_rules! generate_sql_for_timestamp { + (@generic $value:ident, $value_block:expr, $sql_generator:expr, $sql_query:expr) => { + if let Some($value) = $value { + let value = $value_block.to_rfc3339_opts(SecondsFormat::Millis, true); + ( + $sql_generator + .get_sql_templates() + .timestamp_literal_expr(value) + .map_err(|e| { + DataFusionError::Internal(format!( + "Can't generate SQL for timestamp: {}", + e + )) + })?, + $sql_query, + ) + } else { + ("NULL".to_string(), $sql_query) + } + }; + ($value:ident, timestamp, $sql_generator:expr, $sql_query:expr) => { + generate_sql_for_timestamp!( + @generic $value, { Utc.timestamp_opt($value as i64, 0).unwrap() }, $sql_generator, $sql_query + ) + }; + ($value:ident, timestamp_millis_opt, $sql_generator:expr, $sql_query:expr) => { + generate_sql_for_timestamp!( + @generic $value, { Utc.timestamp_millis_opt($value as i64).unwrap() }, $sql_generator, $sql_query + ) + }; + ($value:ident, timestamp_micros, $sql_generator:expr, $sql_query:expr) => { + generate_sql_for_timestamp!( + @generic $value, { Utc.timestamp_micros($value as i64).unwrap() }, $sql_generator, $sql_query + ) + }; + ($value:ident, $method:ident, $sql_generator:expr, $sql_query:expr) => { + generate_sql_for_timestamp!( + @generic $value, { Utc.$method($value as i64) }, $sql_generator, $sql_query + ) + }; +} + impl CubeScanWrapperNode { pub async fn generate_sql( &self, @@ -310,7 +352,7 @@ impl CubeScanWrapperNode { window_expr, from, joins: _joins, - filter_expr: _filter_expr, + filter_expr, having_expr: _having_expr, limit, offset, @@ -443,6 +485,20 @@ impl CubeScanWrapperNode { ) .await?; + let (filter, sql) = Self::generate_column_expr( + plan.clone(), + schema.clone(), + filter_expr.clone(), + sql, + generator.clone(), + &column_remapping, + &mut next_remapping, + alias.clone(), + can_rename_columns, + ungrouped_scan_node.clone(), + ) + .await?; + let (window, sql) = Self::generate_column_expr( plan.clone(), schema.clone(), @@ -548,6 +604,17 @@ impl CubeScanWrapperNode { }) .collect::>()?, ); + load_request.segments = Some( + filter + .iter() + .map(|m| { + Self::ungrouped_member_def( + m, + &ungrouped_scan_node.used_cubes, + ) + }) + .collect::>()?, + ); if !order_expr.is_empty() { load_request.order = Some( order_expr @@ -651,7 +718,16 @@ impl CubeScanWrapperNode { aggregate, // TODO from_alias.unwrap_or("".to_string()), - None, + if !filter.is_empty() { + Some( + filter + .iter() + .map(|f| f.expr.to_string()) + .join(" AND "), + ) + } else { + None + }, None, order, limit, @@ -1284,10 +1360,33 @@ impl CubeScanWrapperNode { } } // ScalarValue::Date64(_) => {} - // ScalarValue::TimestampSecond(_, _) => {} - // ScalarValue::TimestampMillisecond(_, _) => {} - // ScalarValue::TimestampMicrosecond(_, _) => {} - // ScalarValue::TimestampNanosecond(_, _) => {} + ScalarValue::TimestampSecond(s, _) => { + generate_sql_for_timestamp!(s, timestamp, sql_generator, sql_query) + } + ScalarValue::TimestampMillisecond(ms, None) => { + generate_sql_for_timestamp!( + ms, + timestamp_millis_opt, + sql_generator, + sql_query + ) + } + ScalarValue::TimestampMicrosecond(ms, None) => { + generate_sql_for_timestamp!( + ms, + timestamp_micros, + sql_generator, + sql_query + ) + } + ScalarValue::TimestampNanosecond(nanoseconds, None) => { + generate_sql_for_timestamp!( + nanoseconds, + timestamp_nanos, + sql_generator, + sql_query + ) + } ScalarValue::IntervalYearMonth(x) => { if let Some(x) = x { let (num, date_part) = (x, "MONTH"); diff --git a/rust/cubesql/cubesql/src/compile/mod.rs b/rust/cubesql/cubesql/src/compile/mod.rs index 7f76f099c7e42..ec73c1f521591 100644 --- a/rust/cubesql/cubesql/src/compile/mod.rs +++ b/rust/cubesql/cubesql/src/compile/mod.rs @@ -5074,6 +5074,8 @@ from #[tokio::test] async fn test_where_filter_simple() { + init_logger(); + let to_check = vec![ // Binary expression with Measures ( @@ -18317,7 +18319,7 @@ from } init_logger(); - let logical_plan = convert_select_to_query_plan( + let query_plan = convert_select_to_query_plan( r#" WITH "qt_0" AS ( SELECT @@ -18339,11 +18341,16 @@ from .to_string(), DatabaseProtocol::PostgreSQL, ) - .await - .as_logical_plan(); + .await; + + let physical_plan = query_plan.as_physical_plan().await.unwrap(); + println!( + "Physical plan: {}", + displayable(physical_plan.as_ref()).indent() + ); assert_eq!( - logical_plan.find_cube_scan().request, + query_plan.as_logical_plan().find_cube_scan().request, V1LoadRequestQuery { measures: Some(vec![]), dimensions: Some(vec![]), @@ -18352,13 +18359,7 @@ from order: None, limit: None, offset: None, - filters: Some(vec![V1LoadRequestQueryFilterItem { - member: Some("KibanaSampleDataEcommerce.order_date".to_string()), - operator: Some("beforeDate".to_string()), - values: Some(vec!["2014-05-31T00:00:00.000Z".to_string()]), - or: None, - and: None - }]), + filters: None, ungrouped: Some(true), } ) @@ -18882,7 +18883,7 @@ from } init_logger(); - let logical_plan = convert_select_to_query_plan( + let query_plan = convert_select_to_query_plan( r#" WITH "qt_0" AS ( SELECT @@ -18897,11 +18898,16 @@ from .to_string(), DatabaseProtocol::PostgreSQL, ) - .await - .as_logical_plan(); + .await; + + let physical_plan = query_plan.as_physical_plan().await.unwrap(); + println!( + "Physical plan: {}", + displayable(physical_plan.as_ref()).indent() + ); assert_eq!( - logical_plan.find_cube_scan().request, + query_plan.as_logical_plan().find_cube_scan().request, V1LoadRequestQuery { measures: Some(vec![]), dimensions: Some(vec![]), @@ -18913,7 +18919,7 @@ from filters: None, ungrouped: Some(true), } - ) + ); } #[tokio::test] @@ -19374,6 +19380,39 @@ from Ok(()) } + #[tokio::test] + async fn test_where_push_down() { + init_logger(); + + let query_plan = convert_select_to_query_plan( + r#" + SELECT + "customer_gender" AS "uuid.customer_gender", + COUNT(*) AS "count" + FROM "public"."KibanaSampleDataEcommerce" + WHERE CAST(LEFT(RIGHT("customer_gender", 2), 1) AS TEXT) = 'le' + GROUP BY "customer_gender"; + "# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await; + + let physical_plan = query_plan.as_physical_plan().await.unwrap(); + println!( + "Physical plan: {}", + displayable(physical_plan.as_ref()).indent() + ); + + let logical_plan = query_plan.as_logical_plan(); + assert!(logical_plan + .find_cube_scan_wrapper() + .wrapped_sql + .unwrap() + .sql + .contains("LEFT")); + } + #[tokio::test] async fn test_simple_wrapper() { if !Rewriter::sql_push_down_enabled() { diff --git a/rust/cubesql/cubesql/src/compile/rewrite/converter.rs b/rust/cubesql/cubesql/src/compile/rewrite/converter.rs index aae71b2f3d543..a1690418baa08 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/converter.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/converter.rs @@ -1846,15 +1846,36 @@ impl LanguageToLogicalPlanConverter { let alias = match_data_node!(node_by_id, params[12], WrappedSelectAlias); let ungrouped = match_data_node!(node_by_id, params[13], WrappedSelectUngrouped); + let filter_expr = normalize_cols(filter_expr, &from)?; let group_expr = normalize_cols(group_expr, &from)?; let aggr_expr = normalize_cols(aggr_expr, &from)?; - let projection_expr = normalize_cols(projection_expr, &from)?; + let projection_expr = if projection_expr.is_empty() + && matches!(select_type, WrappedSelectType::Projection) + { + from.schema() + .fields() + .iter() + .map(|f| Expr::Column(f.qualified_column())) + .collect::>() + } else { + normalize_cols(projection_expr, &from)? + }; let all_expr_without_window = match select_type { WrappedSelectType::Projection => projection_expr.clone(), WrappedSelectType::Aggregate => { group_expr.iter().chain(aggr_expr.iter()).cloned().collect() } }; + // TODO support asterisk query? + let all_expr_without_window = if all_expr_without_window.is_empty() { + from.schema() + .fields() + .iter() + .map(|f| Expr::Column(f.qualified_column())) + .collect::>() + } else { + all_expr_without_window + }; let without_window_fields = exprlist_to_fields(all_expr_without_window.iter(), from.schema())?; let replace_map = all_expr_without_window diff --git a/rust/cubesql/cubesql/src/compile/rewrite/cost.rs b/rust/cubesql/cubesql/src/compile/rewrite/cost.rs index 7c93ba15bd268..7c7b734c4a5b5 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/cost.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/cost.rs @@ -39,15 +39,15 @@ pub struct CubePlanCost { table_scans: i64, empty_wrappers: i64, non_detected_cube_scans: i64, - filters: i64, - structure_points: i64, - filter_members: i64, member_errors: i64, // TODO if pre-aggregation can be used for window functions, then it'd be suboptimal non_pushed_down_window: i64, ungrouped_aggregates: usize, wrapper_nodes: i64, ast_size_outside_wrapper: usize, + filters: i64, + structure_points: i64, + filter_members: i64, cube_members: i64, errors: i64, time_dimensions_used_as_dimensions: i64, diff --git a/rust/cubesql/cubesql/src/compile/rewrite/mod.rs b/rust/cubesql/cubesql/src/compile/rewrite/mod.rs index dce357115a519..d577f9d3c7642 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/mod.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/mod.rs @@ -845,7 +845,6 @@ fn wrapped_select_joins_empty_tail() -> String { "WrappedSelectJoins".to_string() } -#[allow(dead_code)] fn wrapped_select_filter_expr(left: impl Display, right: impl Display) -> String { format!("(WrappedSelectFilterExpr {} {})", left, right) } diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/aggregate.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/aggregate.rs index 461d5949f706b..ed100f1ee2520 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/aggregate.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/aggregate.rs @@ -73,7 +73,12 @@ impl WrapperRules { "?cube_members", ), wrapped_select_joins_empty_tail(), - wrapped_select_filter_expr_empty_tail(), + wrapper_pullup_replacer( + wrapped_select_filter_expr_empty_tail(), + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), wrapped_select_having_expr_empty_tail(), "WrappedSelectLimit:None", "WrappedSelectOffset:None", diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/filter.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/filter.rs new file mode 100644 index 0000000000000..d38971df7fab4 --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/filter.rs @@ -0,0 +1,221 @@ +use crate::{ + compile::rewrite::{ + analysis::LogicalPlanAnalysis, cube_scan_wrapper, filter, rules::wrapper::WrapperRules, + transforming_rewrite, wrapped_select, wrapped_select_aggr_expr_empty_tail, + wrapped_select_filter_expr, wrapped_select_filter_expr_empty_tail, + wrapped_select_group_expr_empty_tail, wrapped_select_having_expr_empty_tail, + wrapped_select_joins_empty_tail, wrapped_select_order_expr_empty_tail, + wrapped_select_projection_expr_empty_tail, wrapped_select_window_expr_empty_tail, + wrapper_pullup_replacer, wrapper_pushdown_replacer, LogicalPlanLanguage, + WrappedSelectUngrouped, WrapperPullupReplacerUngrouped, + }, + var, var_iter, +}; +use egg::{EGraph, Rewrite, Subst}; + +impl WrapperRules { + pub fn filter_rules(&self, rules: &mut Vec>) { + // TODO respect having filter for push down to wrapped select + // rules.extend(vec![rewrite( + // "wrapper-push-down-filter-to-wrapped-select", + // filter( + // "?filter_expr", + // cube_scan_wrapper( + // wrapper_pullup_replacer( + // wrapped_select( + // "?select_type", + // "?projection_expr", + // "?group_expr", + // "?aggr_expr", + // "?window_expr", + // "?cube_scan_input", + // "?joins", + // "?old_filter_expr", + // "?having_expr", + // "?wrapped_select_limit", + // "?wrapped_select_offset", + // "?order_expr", + // "?select_alias", + // "?select_ungrouped", + // ), + // "?alias_to_cube", + // "?ungrouped", + // "?cube_members", + // ), + // "CubeScanWrapperFinalized:false".to_string(), + // ), + // ), + // cube_scan_wrapper( + // wrapped_select( + // "?select_type", + // wrapper_pullup_replacer( + // "?projection_expr", + // "?alias_to_cube", + // "?ungrouped", + // "?cube_members", + // ), + // wrapper_pullup_replacer( + // "?group_expr", + // "?alias_to_cube", + // "?ungrouped", + // "?cube_members", + // ), + // wrapper_pullup_replacer( + // "?aggr_expr", + // "?alias_to_cube", + // "?ungrouped", + // "?cube_members", + // ), + // wrapper_pullup_replacer( + // "?window_expr", + // "?alias_to_cube", + // "?ungrouped", + // "?cube_members", + // ), + // wrapper_pullup_replacer( + // "?cube_scan_input", + // "?alias_to_cube", + // "?ungrouped", + // "?cube_members", + // ), + // "?joins", + // wrapped_select_filter_expr( + // wrapper_pullup_replacer( + // "?old_filter_expr", + // "?alias_to_cube", + // "?ungrouped", + // "?cube_members", + // ), + // wrapper_pushdown_replacer( + // "?filter_expr", + // "?alias_to_cube", + // "?ungrouped", + // "?cube_members", + // ), + // ), + // "?having_expr", + // "?wrapped_select_limit", + // "?wrapped_select_offset", + // wrapper_pullup_replacer( + // "?order_expr", + // "?alias_to_cube", + // "?ungrouped", + // "?cube_members", + // ), + // "?select_alias", + // "?select_ungrouped", + // ), + // "CubeScanWrapperFinalized:false", + // ), + // )]); + + rules.extend(vec![transforming_rewrite( + "wrapper-push-down-filter-to-cube-scan", + filter( + "?filter_expr", + cube_scan_wrapper( + wrapper_pullup_replacer( + "?cube_scan_input", + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), + "CubeScanWrapperFinalized:false", + ), + ), + cube_scan_wrapper( + wrapped_select( + "WrappedSelectSelectType:Projection", + wrapper_pullup_replacer( + wrapped_select_projection_expr_empty_tail(), + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), + wrapper_pullup_replacer( + wrapped_select_group_expr_empty_tail(), + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), + wrapper_pullup_replacer( + wrapped_select_aggr_expr_empty_tail(), + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), + wrapper_pullup_replacer( + wrapped_select_window_expr_empty_tail(), + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), + wrapper_pullup_replacer( + "?cube_scan_input", + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), + wrapped_select_joins_empty_tail(), + wrapped_select_filter_expr( + wrapper_pushdown_replacer( + "?filter_expr", + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), + wrapper_pullup_replacer( + wrapped_select_filter_expr_empty_tail(), + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), + ), + wrapped_select_having_expr_empty_tail(), + "WrappedSelectLimit:None", + "WrappedSelectOffset:None", + wrapper_pullup_replacer( + wrapped_select_order_expr_empty_tail(), + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), + "WrappedSelectAlias:None", + "?select_ungrouped", + ), + "CubeScanWrapperFinalized:false", + ), + self.transform_filter("?ungrouped", "?select_ungrouped"), + )]); + + Self::list_pushdown_pullup_rules( + rules, + "wrapper-filter-expr", + "WrappedSelectFilterExpr", + "WrappedSelectFilterExpr", + ); + } + + fn transform_filter( + &self, + ungrouped_var: &'static str, + select_ungrouped_var: &'static str, + ) -> impl Fn(&mut EGraph, &mut Subst) -> bool { + let ungrouped_var = var!(ungrouped_var); + let select_ungrouped_var = var!(select_ungrouped_var); + move |egraph, subst| { + for ungrouped in + var_iter!(egraph[subst[ungrouped_var]], WrapperPullupReplacerUngrouped).cloned() + { + subst.insert( + select_ungrouped_var, + egraph.add(LogicalPlanLanguage::WrappedSelectUngrouped( + WrappedSelectUngrouped(ungrouped), + )), + ); + return true; + } + false + } + } +} diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/literal.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/literal.rs index 8d40bdb5c898c..134454f4bf97c 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/literal.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/literal.rs @@ -1,15 +1,21 @@ -use crate::compile::rewrite::{ - analysis::LogicalPlanAnalysis, literal_expr, rewrite, rules::wrapper::WrapperRules, - wrapper_pullup_replacer, wrapper_pushdown_replacer, LogicalPlanLanguage, +use crate::{ + compile::rewrite::{ + analysis::LogicalPlanAnalysis, literal_expr, rules::wrapper::WrapperRules, + transforming_rewrite, wrapper_pullup_replacer, wrapper_pushdown_replacer, LiteralExprValue, + LogicalPlanLanguage, WrapperPullupReplacerAliasToCube, + }, + var, var_iter, }; -use egg::Rewrite; + +use datafusion::scalar::ScalarValue; +use egg::{EGraph, Rewrite, Subst}; impl WrapperRules { pub fn literal_rules( &self, rules: &mut Vec>, ) { - rules.extend(vec![rewrite( + rules.extend(vec![transforming_rewrite( "wrapper-push-down-literal", wrapper_pushdown_replacer( literal_expr("?value"), @@ -23,6 +29,43 @@ impl WrapperRules { "?ungrouped", "?cube_members", ), + self.transform_literal("?alias_to_cube", "?value"), )]); } + + fn transform_literal( + &self, + alias_to_cube_var: &str, + value_var: &str, + ) -> impl Fn(&mut EGraph, &mut Subst) -> bool { + let alias_to_cube_var = var!(alias_to_cube_var); + let value_var = var!(value_var); + let meta = self.meta_context.clone(); + move |egraph, subst| { + for alias_to_cube in var_iter!( + egraph[subst[alias_to_cube_var]], + WrapperPullupReplacerAliasToCube + ) + .cloned() + { + if let Some(sql_generator) = meta.sql_generator_by_alias_to_cube(&alias_to_cube) { + for literal in var_iter!(egraph[subst[value_var]], LiteralExprValue) { + match literal { + ScalarValue::TimestampNanosecond(_, _) + | ScalarValue::TimestampMillisecond(_, _) + | ScalarValue::TimestampMicrosecond(_, _) + | ScalarValue::TimestampSecond(_, _) => { + return sql_generator + .get_sql_templates() + .templates + .contains_key("expressions/timestamp_literal"); + } + _ => return true, + } + } + } + } + false + } + } } diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/mod.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/mod.rs index 57ee14be6d096..cb3927872c25e 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/mod.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/mod.rs @@ -7,6 +7,7 @@ mod cast; mod column; mod cube_scan_wrapper; mod extract; +mod filter; mod in_list_expr; mod is_null_expr; mod limit; @@ -50,6 +51,7 @@ impl RewriteRules for WrapperRules { self.aggregate_rules(&mut rules); self.projection_rules(&mut rules); self.limit_rules(&mut rules); + self.filter_rules(&mut rules); self.order_rules(&mut rules); self.window_rules(&mut rules); self.aggregate_function_rules(&mut rules); diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/order.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/order.rs index 30449176b1827..1ec8980aa95b1 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/order.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/order.rs @@ -70,7 +70,12 @@ impl WrapperRules { "?cube_members", ), "?joins", - "?filter_expr", + wrapper_pullup_replacer( + "?filter_expr", + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), "?having_expr", "?limit", "?offset", diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/projection.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/projection.rs index aba2c26458dbc..2b2d640de9930 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/projection.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/projection.rs @@ -67,7 +67,12 @@ impl WrapperRules { "?cube_members", ), wrapped_select_joins_empty_tail(), - wrapped_select_filter_expr_empty_tail(), + wrapper_pullup_replacer( + wrapped_select_filter_expr_empty_tail(), + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), wrapped_select_having_expr_empty_tail(), "WrappedSelectLimit:None", "WrappedSelectOffset:None", diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/window.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/window.rs index 81c366cb102e8..d39706163c697 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/window.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/window.rs @@ -70,7 +70,12 @@ impl WrapperRules { "?cube_members", ), "?joins", - "?filter_expr", + wrapper_pullup_replacer( + "?filter_expr", + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), "?having_expr", "?limit", "?offset", diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/wrapper_pull_up.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/wrapper_pull_up.rs index fa701c9f2948b..c312601e254f1 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/wrapper_pull_up.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/wrapper_pull_up.rs @@ -1,9 +1,9 @@ use crate::{ compile::rewrite::{ analysis::LogicalPlanAnalysis, cube_scan_wrapper, rules::wrapper::WrapperRules, - transforming_rewrite, wrapped_select, wrapped_select_filter_expr_empty_tail, - wrapped_select_having_expr_empty_tail, wrapped_select_joins_empty_tail, - wrapper_pullup_replacer, LogicalPlanLanguage, WrappedSelectSelectType, WrappedSelectType, + transforming_rewrite, wrapped_select, wrapped_select_having_expr_empty_tail, + wrapped_select_joins_empty_tail, wrapper_pullup_replacer, LogicalPlanLanguage, + WrappedSelectSelectType, WrappedSelectType, }, var, var_iter, var_list_iter, }; @@ -51,7 +51,12 @@ impl WrapperRules { "?cube_members", ), wrapped_select_joins_empty_tail(), - wrapped_select_filter_expr_empty_tail(), + wrapper_pullup_replacer( + "?filter_expr", + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), wrapped_select_having_expr_empty_tail(), "WrappedSelectLimit:None", "WrappedSelectOffset:None", @@ -76,7 +81,7 @@ impl WrapperRules { "?window_expr", "?cube_scan_input", wrapped_select_joins_empty_tail(), - wrapped_select_filter_expr_empty_tail(), + "?filter_expr", wrapped_select_having_expr_empty_tail(), "WrappedSelectLimit:None", "WrappedSelectOffset:None", @@ -144,7 +149,12 @@ impl WrapperRules { "?cube_members", ), wrapped_select_joins_empty_tail(), - wrapped_select_filter_expr_empty_tail(), + wrapper_pullup_replacer( + "?filter_expr", + "?alias_to_cube", + "?ungrouped", + "?cube_members", + ), wrapped_select_having_expr_empty_tail(), "WrappedSelectLimit:None", "WrappedSelectOffset:None", @@ -184,7 +194,7 @@ impl WrapperRules { "?inner_ungrouped", ), wrapped_select_joins_empty_tail(), - wrapped_select_filter_expr_empty_tail(), + "?filter_expr", wrapped_select_having_expr_empty_tail(), "WrappedSelectLimit:None", "WrappedSelectOffset:None", diff --git a/rust/cubesql/cubesql/src/compile/test/mod.rs b/rust/cubesql/cubesql/src/compile/test/mod.rs index 8105402e89888..8380d1996a4e8 100644 --- a/rust/cubesql/cubesql/src/compile/test/mod.rs +++ b/rust/cubesql/cubesql/src/compile/test/mod.rs @@ -287,11 +287,14 @@ pub fn get_test_tenant_ctx_customized(custom_templates: Vec<(String, String)>) - ("functions/DATE_ADD".to_string(), "DATE_ADD({{ args_concat }})".to_string()), ("functions/CONCAT".to_string(), "CONCAT({{ args_concat }})".to_string()), ("functions/DATE".to_string(), "DATE({{ args_concat }})".to_string()), + ("functions/LEFT".to_string(), "LEFT({{ args_concat }})".to_string()), + ("functions/RIGHT".to_string(), "RIGHT({{ args_concat }})".to_string()), ("expressions/extract".to_string(), "EXTRACT({{ date_part }} FROM {{ expr }})".to_string()), ( "statements/select".to_string(), r#"SELECT {{ select_concat | map(attribute='aliased') | join(', ') }} - FROM ({{ from }}) AS {{ from_alias }} + FROM ({{ from }}) AS {{ from_alias }} + {% if filter %} WHERE {{ filter }}{% endif %} {% if group_by %} GROUP BY {{ group_by | map(attribute='index') | join(', ') }}{% endif %} {% if order_by %} ORDER BY {{ order_by | map(attribute='expr') | join(', ') }}{% endif %}{% if limit %} LIMIT {{ limit }}{% endif %}{% if offset %} @@ -313,6 +316,7 @@ pub fn get_test_tenant_ctx_customized(custom_templates: Vec<(String, String)>) - ("expressions/not".to_string(), "NOT ({{ expr }})".to_string()), ("expressions/true".to_string(), "TRUE".to_string()), ("expressions/false".to_string(), "FALSE".to_string()), + ("expressions/timestamp_literal".to_string(), "timestamptz '{{ value }}'".to_string()), ("quotes/identifiers".to_string(), "\"".to_string()), ("quotes/escape".to_string(), "\"\"".to_string()), ("params/param".to_string(), "${{ param_index + 1 }}".to_string()) diff --git a/rust/cubesql/cubesql/src/transport/service.rs b/rust/cubesql/cubesql/src/transport/service.rs index bafd50cf0c899..233bbc2968f93 100644 --- a/rust/cubesql/cubesql/src/transport/service.rs +++ b/rust/cubesql/cubesql/src/transport/service.rs @@ -383,7 +383,7 @@ impl SqlTemplates { group_by: Vec, aggregate: Vec, alias: String, - _filter: Option, + filter: Option, _having: Option, order_by: Vec, limit: Option, @@ -409,6 +409,7 @@ impl SqlTemplates { aggregate => aggregate, projection => projection, order_by => order_by, + filter => filter, from_alias => quoted_from_alias, limit => limit, offset => offset, @@ -654,6 +655,10 @@ impl SqlTemplates { } } + pub fn timestamp_literal_expr(&self, value: String) -> Result { + self.render_template("expressions/timestamp_literal", context! { value => value }) + } + pub fn param(&self, param_index: usize) -> Result { self.render_template("params/param", context! { param_index => param_index }) }