diff --git a/packages/cubejs-backend-native/Cargo.lock b/packages/cubejs-backend-native/Cargo.lock index 65113ac0e7061..53e5c3c02446a 100644 --- a/packages/cubejs-backend-native/Cargo.lock +++ b/packages/cubejs-backend-native/Cargo.lock @@ -615,6 +615,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -738,7 +747,7 @@ dependencies = [ "async-trait", "axum", "bytes", - "convert_case", + "convert_case 0.6.0", "cubenativeutils", "cubeorchestrator", "cubesql", @@ -769,7 +778,7 @@ version = "0.1.0" dependencies = [ "async-channel", "async-trait", - "convert_case", + "convert_case 0.6.0", "cubesql", "lazy_static", "log", @@ -861,7 +870,7 @@ dependencies = [ "async-trait", "chrono", "chrono-tz 0.8.6", - "convert_case", + "convert_case 0.7.1", "cubeclient", "cubenativeutils", "datafusion", diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index d337cfce3fa17..687297c0b865e 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -278,11 +278,9 @@ export class BaseQuery { return dimension; }).filter(R.identity).map(this.newTimeDimension.bind(this)); this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters); + this.useNativeSqlPlanner = this.options.useNativeSqlPlanner ?? getEnv('nativeSqlPlanner'); + this.prebuildJoin(); - if (!getEnv('nativeSqlPlanner')) { - // Tesseract doesn't require join to be prebuilt and there's a case where single join can't be built for multi-fact query - this.join = this.joinGraph.buildJoin(this.allJoinHints); - } this.cubeAliasPrefix = this.options.cubeAliasPrefix; this.preAggregationsSchemaOption = this.options.preAggregationsSchema ?? DEFAULT_PREAGGREGATIONS_SCHEMA; this.externalQueryClass = this.options.externalQueryClass; @@ -299,6 +297,15 @@ export class BaseQuery { this.initUngrouped(); } + prebuildJoin() { + /* if (!this.useNativeSqlPlanner) { We still need this join for the follback to preaggregation to work properly. This condition should be returned after the tesseract starts working with pre-aggregations + // Tesseract doesn't require join to be prebuilt and there's a case where single join can't be built for multi-fact query + this.join = this.joinGraph.buildJoin(this.allJoinHints); + } */ + + this.join = this.joinGraph.buildJoin(this.allJoinHints); + } + cacheValue(key, fn, { contextPropNames, inputProps, cache } = {}) { const currentContext = this.safeEvaluateSymbolContext(); if (contextPropNames) { @@ -375,8 +382,7 @@ export class BaseQuery { initUngrouped() { this.ungrouped = this.options.ungrouped; if (this.ungrouped) { - // this.join is not defined for Tesseract - if (!this.options.allowUngroupedWithoutPrimaryKey && !getEnv('nativeSqlPlanner')) { + if (!this.options.allowUngroupedWithoutPrimaryKey) { const cubes = R.uniq([this.join.root].concat(this.join.joins.map(j => j.originalTo))); const primaryKeyNames = cubes.flatMap(c => this.primaryKeyNames(c)); const missingPrimaryKeys = primaryKeyNames.filter(key => !this.dimensions.find(d => d.dimension === key)); @@ -603,33 +609,61 @@ export class BaseQuery { return false; } + newQueryNotUseNative() { + const QueryClass = this.constructor; + return new QueryClass(this.compilers, { ...this.options, useNativeSqlPlanner: false }); + } + /** * Returns a pair of SQL query string and parameter values for the query. * @param {boolean} [exportAnnotatedSql] - returns annotated sql with not rendered params if true * @returns {[string, Array]} */ buildSqlAndParams(exportAnnotatedSql) { - if (getEnv('nativeSqlPlanner')) { - return this.buildSqlAndParamsRust(exportAnnotatedSql); - } else { - if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) { - if (this.externalPreAggregationQuery()) { // TODO performance - return this.externalQuery().buildSqlAndParams(exportAnnotatedSql); + if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) { + if (this.externalPreAggregationQuery()) { // TODO performance + return this.externalQuery().buildSqlAndParams(exportAnnotatedSql); + } + } + + if (this.useNativeSqlPlanner) { + let isRelatedToPreAggregation = false; + if (this.options.preAggregationQuery) { + isRelatedToPreAggregation = true; + } else if (!this.options.disableExternalPreAggregations && this.externalQueryClass) { + if (this.externalPreAggregationQuery()) { + isRelatedToPreAggregation = true; + } + } else { + let preAggForQuery = + this.preAggregations.findPreAggregationForQuery(); + if (this.options.disableExternalPreAggregations && preAggForQuery && preAggForQuery.preAggregation.external) { + preAggForQuery = undefined; + } + if (preAggForQuery) { + isRelatedToPreAggregation = true; } } - return this.compilers.compiler.withQuery( - this, - () => this.cacheValue( - ['buildSqlAndParams', exportAnnotatedSql], - () => this.paramAllocator.buildSqlAndParams( - this.buildParamAnnotatedSql(), - exportAnnotatedSql, - this.shouldReuseParams - ), - { cache: this.queryCache } - ) - ); + + if (isRelatedToPreAggregation) { + return this.newQueryNotUseNative().buildSqlAndParams(exportAnnotatedSql); + } + + return this.buildSqlAndParamsRust(exportAnnotatedSql); } + + return this.compilers.compiler.withQuery( + this, + () => this.cacheValue( + ['buildSqlAndParams', exportAnnotatedSql], + () => this.paramAllocator.buildSqlAndParams( + this.buildParamAnnotatedSql(), + exportAnnotatedSql, + this.shouldReuseParams + ), + { cache: this.queryCache } + ) + ); } buildSqlAndParamsRust(exportAnnotatedSql) { @@ -2960,6 +2994,7 @@ export class BaseQuery { } newSubQueryForCube(cube, options) { + options = { ...options, useNativeSqlPlanner: false }; // We not use tesseract for pre-aggregations generation yet if (this.options.queryFactory) { // When dealing with rollup joins, it's crucial to use the correct parameter allocator for the specific cube in use. // By default, we'll use BaseQuery, but it's important to note that different databases (Oracle, PostgreSQL, MySQL, Druid, etc.) @@ -2983,6 +3018,7 @@ export class BaseQuery { historyQueries: this.options.historyQueries, externalQueryClass: this.options.externalQueryClass, queryFactory: this.options.queryFactory, + useNativeSqlPlanner: this.options.useNativeSqlPlanner, ...options, }; } diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index efc642af9d727..0c0a06031ee99 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -57,10 +57,6 @@ export class PreAggregations { } preAggregationCubes() { - if (getEnv('nativeSqlPlanner')) { - // No join defined in Tesseract - return []; - } const { join } = this.query; return join.joins.map(j => j.originalTo).concat([join.root]); } diff --git a/rust/cubenativeutils/src/wrappers/context.rs b/rust/cubenativeutils/src/wrappers/context.rs index 805e7923ffa54..964ec4925f46c 100644 --- a/rust/cubenativeutils/src/wrappers/context.rs +++ b/rust/cubenativeutils/src/wrappers/context.rs @@ -9,7 +9,6 @@ pub trait NativeContext: Clone { fn null(&self) -> Result, CubeError>; fn empty_array(&self) -> Result; fn empty_struct(&self) -> Result; - //fn boxed(&self, value: T) -> impl NativeBox; fn to_string_fn(&self, result: String) -> Result; } diff --git a/rust/cubesqlplanner/Cargo.lock b/rust/cubesqlplanner/Cargo.lock index c2732a97cef95..6cd85623c9b19 100644 --- a/rust/cubesqlplanner/Cargo.lock +++ b/rust/cubesqlplanner/Cargo.lock @@ -556,6 +556,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -692,7 +701,7 @@ version = "0.1.0" dependencies = [ "async-channel", "async-trait", - "convert_case", + "convert_case 0.6.0", "cubesql", "lazy_static", "log", @@ -764,7 +773,7 @@ dependencies = [ "async-trait", "chrono", "chrono-tz 0.8.6", - "convert_case", + "convert_case 0.7.1", "cubeclient", "cubenativeutils", "datafusion", diff --git a/rust/cubesqlplanner/cubesqlplanner/Cargo.toml b/rust/cubesqlplanner/cubesqlplanner/Cargo.toml index cbf1392dac058..43ed6ccc5db5a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/Cargo.toml +++ b/rust/cubesqlplanner/cubesqlplanner/Cargo.toml @@ -16,7 +16,7 @@ serde = "1.0.115" serde_json = "1.0.56" cubenativeutils = { path = "../../cubenativeutils/" } minijinja = { version = "1", features = ["json", "loader"] } -convert_case = "0.6" +convert_case = "0.7.1" chrono = "0.4.15" chrono-tz = "0.8.2" lazy_static = "1.4.0" diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index 4b77c9eef4e96..5eb3e24c57285 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -9,6 +9,10 @@ use lazy_static::lazy_static; use regex::Regex; use std::rc::Rc; +const FROM_PARTITION_RANGE: &'static str = "__FROM_PARTITION_RANGE"; + +const TO_PARTITION_RANGE: &'static str = "__TO_PARTITION_RANGE"; + #[derive(Debug, Clone, PartialEq, Eq)] pub enum FilterType { Dimension, @@ -393,7 +397,7 @@ impl BaseFilter { let from = if let Some(from_str) = &self.values[0] { let from = self.format_from_date(&from_str)?; - if use_db_time_zone { + if use_db_time_zone && &from != FROM_PARTITION_RANGE { self.query_tools.base_tools().in_db_time_zone(from)? } else { from @@ -407,7 +411,7 @@ impl BaseFilter { let to = if let Some(to_str) = &self.values[1] { let to = self.format_to_date(&to_str)?; - if use_db_time_zone { + if use_db_time_zone && &to != TO_PARTITION_RANGE { self.query_tools.base_tools().in_db_time_zone(to)? } else { to diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs index cf7d79475bf07..9bb7aa3cee1c2 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs @@ -65,6 +65,12 @@ impl SimpleQueryPlanner { } let render_references = dimension_subquery_planner.dimensions_refs().clone(); context_factory.set_render_references(render_references); + context_factory.set_rendered_as_multiplied_measures( + self.query_properties + .full_key_aggregate_measures()? + .rendered_as_multiplied_measures + .clone(), + ); select_builder.set_filter(filter); select_builder.set_group_by(self.query_properties.group_by()); select_builder.set_order_by(self.order_planner.default_order()); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs index e75372e7c4896..da597cbd85bcb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs @@ -11,7 +11,7 @@ use crate::plan::FilterItem; use crate::planner::sql_evaluator::collectors::collect_join_hints; use crate::planner::sql_templates::PlanSqlTemplates; use chrono_tz::Tz; -use convert_case::{Case, Casing}; +use convert_case::{Boundary, Case, Casing}; use cubenativeutils::CubeError; use itertools::Itertools; use lazy_static::lazy_static; @@ -187,7 +187,9 @@ impl QueryTools { } pub fn alias_name(&self, name: &str) -> String { - name.to_case(Case::Snake).replace(".", "__") + name.without_boundaries(&[Boundary::LOWER_DIGIT, Boundary::UPPER_DIGIT]) + .to_case(Case::Snake) + .replace(".", "__") } pub fn escaped_alias_name(&self, name: &str) -> String { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs index e83ee51d5525c..ba4e7c01c1220 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs @@ -1,7 +1,7 @@ use super::{TemplateGroupByColumn, TemplateOrderByColumn, TemplateProjectionColumn}; use crate::cube_bridge::sql_templates_render::SqlTemplatesRender; use crate::plan::join::JoinType; -use convert_case::{Case, Casing}; +use convert_case::{Boundary, Case, Casing}; use cubenativeutils::CubeError; use minijinja::context; use std::rc::Rc; @@ -17,7 +17,12 @@ impl PlanSqlTemplates { } pub fn alias_name(name: &str) -> String { - name.to_case(Case::Snake).replace(".", "__") + let res = name + .from_case(Case::Camel) + .without_boundaries(&[Boundary::LOWER_DIGIT, Boundary::UPPER_DIGIT]) + .to_case(Case::Snake) + .replace(".", "__"); + res } pub fn memeber_alias_name(cube_name: &str, name: &str, suffix: &Option) -> String {