Skip to content

Commit

Permalink
feat(cubesql): Support KPI chart in Thoughtspot
Browse files Browse the repository at this point in the history
  • Loading branch information
@MazterQyou
MazterQyou committed Jan 22, 2024
1 parent 1a66d97 commit 1cd1b22
Show file tree
Hide file tree
Showing 9 changed files with 283 additions and 59 deletions.
12 changes: 6 additions & 6 deletions packages/cubejs-backend-native/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions rust/cubesql/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion rust/cubesql/cubesql/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ homepage = "https://cube.dev"

[dependencies]
arc-swap = "1"
datafusion = { git = 'https://github.com/cube-js/arrow-datafusion.git', rev = "208a6889c6067b760c6cc2775036570c30a086b5", default-features = false, features = ["regex_expressions", "unicode_expressions"] }
datafusion = { git = 'https://github.com/cube-js/arrow-datafusion.git', rev = "493ce35e46884b37d4d2de4e2e72a9854a89d288", default-features = false, features = ["regex_expressions", "unicode_expressions"] }
anyhow = "1.0"
thiserror = "1.0.50"
cubeclient = { path = "../cubeclient" }
Expand Down
68 changes: 61 additions & 7 deletions rust/cubesql/cubesql/src/compile/engine/df/scan.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use cubeclient::models::{V1LoadRequestQuery, V1LoadResult, V1LoadResultAnnotatio
pub use datafusion::{
arrow::{
array::{
ArrayRef, BooleanBuilder, Date32Builder, Float64Builder, Int32Builder, Int64Builder,
StringBuilder,
ArrayRef, BooleanBuilder, Date32Builder, DecimalBuilder, Float64Builder, Int32Builder,
Int64Builder, StringBuilder,
},
datatypes::{DataType, SchemaRef},
error::{ArrowError, Result as ArrowResult},
Expand Down Expand Up @@ -499,11 +499,17 @@ macro_rules! build_column {
let len = $response.len()?;
let mut builder = <$builder_ty>::new(len);

build_column_custom_builder!($data_type, len, builder, $response, $field_name, { $($builder_block)* }, { $($scalar_block)* })
}}
}

macro_rules! build_column_custom_builder {
($data_type:expr, $len:expr, $builder:expr, $response:expr, $field_name: expr, { $($builder_block:tt)* }, { $($scalar_block:tt)* }) => {{
match $field_name {
MemberField::Member(field_name) => {
for i in 0..len {
for i in 0..$len {
let value = $response.get(i, field_name)?;
match (value, &mut builder) {
match (value, &mut $builder) {
(FieldValue::Null, builder) => builder.append_null()?,
$($builder_block)*
#[allow(unreachable_patterns)]
Expand All @@ -518,8 +524,8 @@ macro_rules! build_column {
}
}
MemberField::Literal(value) => {
for _ in 0..len {
match (value, &mut builder) {
for _ in 0..$len {
match (value, &mut $builder) {
$($scalar_block)*
(v, _) => {
return Err(CubeError::user(format!(
Expand All @@ -533,7 +539,7 @@ macro_rules! build_column {
}
};

Arc::new(builder.finish()) as ArrayRef
Arc::new($builder.finish()) as ArrayRef
}}
}

Expand Down Expand Up @@ -1134,6 +1140,54 @@ pub fn transform_response<V: ValueObject>(
}
)
}
DataType::Decimal(precision, scale) => {
let len = response.len()?;
let mut builder = DecimalBuilder::new(len, *precision, *scale);

Check warning on line 1145 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/scan.rs#L1143-L1145 

Added lines #L1143 - L1145 were not covered by tests

build_column_custom_builder!(
DataType::Decimal(*precision, *scale),
len,
builder,
response,
field_name,

Check warning on line 1152 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/scan.rs#L1147-L1152 

Added lines #L1147 - L1152 were not covered by tests
{
(FieldValue::String(s), builder) => {
let mut parts = s.split(".");
match parts.next() {
None => builder.append_null()?,
Some(int_part) => {
let frac_part = format!("{:0<width$}", parts.next().unwrap_or(""), width=scale);
if frac_part.len() > *scale {
Err(DataFusionError::Execution(format!("Decimal scale is higher than requested: expected {}, got {}", scale, frac_part.len())))?;

Check warning on line 1161 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/scan.rs#L1154-L1161 

Added lines #L1154 - L1161 were not covered by tests
}
if let Some(_) = parts.next() {
Err(DataFusionError::Execution(format!("Unable to parse decimal, value contains two dots: {}", s)))?;

Check warning on line 1164 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/scan.rs#L1163-L1164 

Added lines #L1163 - L1164 were not covered by tests
}
let decimal_str = format!("{}{}", int_part, frac_part);
if decimal_str.len() > *precision {
Err(DataFusionError::Execution(format!("Decimal precision is higher than requested: expected {}, got {}", precision, decimal_str.len())))?;

Check warning on line 1168 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/scan.rs#L1166-L1168 

Added lines #L1166 - L1168 were not covered by tests
}
if let Ok(value) = decimal_str.parse::<i128>() {
builder.append_value(value)?;

Check warning on line 1171 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/scan.rs#L1170-L1171 

Added lines #L1170 - L1171 were not covered by tests
} else {
Err(DataFusionError::Execution(format!("Unable to parse decimal as an i128: {}", decimal_str)))?;

Check warning on line 1173 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/scan.rs#L1173 

Added line #L1173 was not covered by tests
}
}
};
},
},
{
(ScalarValue::Decimal128(v, _, _), builder) => {

Check warning on line 1180 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/scan.rs#L1180 

Added line #L1180 was not covered by tests
// TODO: check precision and scale, adjust accordingly
if let Some(v) = v {
builder.append_value(*v)?;

Check warning on line 1183 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/scan.rs#L1182-L1183 

Added lines #L1182 - L1183 were not covered by tests
} else {
builder.append_null()?;
}
},
}
)
}
t => {
return Err(CubeError::user(format!(
"Type {} is not supported in response transformation from Cube",
Expand Down
118 changes: 90 additions & 28 deletions rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use crate::{
},
CubeError,
};
use chrono::{Days, NaiveDate};
use cubeclient::models::V1LoadRequestQuery;
use datafusion::{
error::{DataFusionError, Result},
Expand All @@ -23,7 +24,10 @@ use datafusion::{
use itertools::Itertools;
use regex::{Captures, Regex};
use serde_derive::*;
use std::{any::Any, collections::HashMap, fmt, future::Future, pin::Pin, result, sync::Arc};
use std::{
any::Any, collections::HashMap, convert::TryInto, fmt, future::Future, pin::Pin, result,
sync::Arc,
};

#[derive(Debug, Clone, Deserialize)]
pub struct SqlQuery {
Expand Down Expand Up @@ -1099,30 +1103,33 @@ impl CubeScanWrapperNode {
)
.await?;
let data_type = match data_type {
DataType::Null => "NULL",
DataType::Boolean => "BOOLEAN",
DataType::Int8 => "INTEGER",
DataType::Int16 => "INTEGER",
DataType::Int32 => "INTEGER",
DataType::Int64 => "INTEGER",
DataType::UInt8 => "INTEGER",
DataType::UInt16 => "INTEGER",
DataType::UInt32 => "INTEGER",
DataType::UInt64 => "INTEGER",
DataType::Float16 => "FLOAT",
DataType::Float32 => "FLOAT",
DataType::Float64 => "DOUBLE PRECISION",
DataType::Timestamp(_, _) => "TIMESTAMP",
DataType::Date32 => "DATE",
DataType::Date64 => "DATE",
DataType::Time32(_) => "TIME",
DataType::Time64(_) => "TIME",
DataType::Duration(_) => "INTERVAL",
DataType::Interval(_) => "INTERVAL",
DataType::Binary => "BYTEA",
DataType::FixedSizeBinary(_) => "BYTEA",
DataType::Utf8 => "TEXT",
DataType::LargeUtf8 => "TEXT",
DataType::Null => "NULL".to_string(),
DataType::Boolean => "BOOLEAN".to_string(),
DataType::Int8 => "INTEGER".to_string(),
DataType::Int16 => "INTEGER".to_string(),

Check warning on line 1109 in rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs#L1106-L1109 

Added lines #L1106 - L1109 were not covered by tests
DataType::Int32 => "INTEGER".to_string(),
DataType::Int64 => "INTEGER".to_string(),
DataType::UInt8 => "INTEGER".to_string(),
DataType::UInt16 => "INTEGER".to_string(),
DataType::UInt32 => "INTEGER".to_string(),
DataType::UInt64 => "INTEGER".to_string(),
DataType::Float16 => "FLOAT".to_string(),
DataType::Float32 => "FLOAT".to_string(),
DataType::Float64 => "DOUBLE PRECISION".to_string(),

Check warning on line 1118 in rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs#L1112-L1118 

Added lines #L1112 - L1118 were not covered by tests
DataType::Timestamp(_, _) => "TIMESTAMP".to_string(),
DataType::Date32 => "DATE".to_string(),
DataType::Date64 => "DATE".to_string(),
DataType::Time32(_) => "TIME".to_string(),
DataType::Time64(_) => "TIME".to_string(),
DataType::Duration(_) => "INTERVAL".to_string(),
DataType::Interval(_) => "INTERVAL".to_string(),
DataType::Binary => "BYTEA".to_string(),
DataType::FixedSizeBinary(_) => "BYTEA".to_string(),
DataType::Utf8 => "TEXT".to_string(),
DataType::LargeUtf8 => "TEXT".to_string(),

Check warning on line 1129 in rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs#L1121-L1129 

Added lines #L1121 - L1129 were not covered by tests
DataType::Decimal(precision, scale) => {
format!("NUMERIC({}, {})", precision, scale)
}
x => {
return Err(DataFusionError::Execution(format!(
"Can't generate SQL for cast: type isn't supported: {:?}",
Expand All @@ -1132,7 +1139,7 @@ impl CubeScanWrapperNode {
};
let resulting_sql = sql_generator
.get_sql_templates()
.cast_expr(expr, data_type.to_string())
.cast_expr(expr, data_type)
.map_err(|e| {
DataFusionError::Internal(format!("Can't generate SQL for cast: {}", e))
})?;
Expand Down Expand Up @@ -1217,13 +1224,68 @@ impl CubeScanWrapperNode {
// ScalarValue::Binary(_) => {}
// ScalarValue::LargeBinary(_) => {}
// ScalarValue::List(_, _) => {}
// ScalarValue::Date32(_) => {}
ScalarValue::Date32(x) => {
if let Some(x) = x {
let days = Days::new(x.abs().try_into().unwrap());
let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap();
let new_date = if x < 0 {
epoch.checked_sub_days(days)

Check warning on line 1232 in rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs#L1232 

Added line #L1232 was not covered by tests
} else {
epoch.checked_add_days(days)
};
let Some(new_date) = new_date else {
return Err(DataFusionError::Internal(format!(

Check warning on line 1237 in rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs#L1237 

Added line #L1237 was not covered by tests
"Can't generate SQL for date: day out of bounds ({})",
x
)));
};
let formatted_date = new_date.format("%Y-%m-%d").to_string();
(
sql_generator
.get_sql_templates()
.scalar_function(
"DATE".to_string(),
vec![format!("'{}'", formatted_date)],
None,
None,
)
.map_err(|e| {
DataFusionError::Internal(format!(

Check warning on line 1253 in rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs#L1252-L1253 

Added lines #L1252 - L1253 were not covered by tests
"Can't generate SQL for date: {}",
e
))
})?,
sql_query,
)
} else {
("NULL".to_string(), sql_query)

Check warning on line 1261 in rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs#L1261 

Added line #L1261 was not covered by tests
}
}
// ScalarValue::Date64(_) => {}
// ScalarValue::TimestampSecond(_, _) => {}
// ScalarValue::TimestampMillisecond(_, _) => {}
// ScalarValue::TimestampMicrosecond(_, _) => {}
// ScalarValue::TimestampNanosecond(_, _) => {}
// ScalarValue::IntervalYearMonth(_) => {}
ScalarValue::IntervalYearMonth(x) => {
if let Some(x) = x {
let (num, date_part) = (x, "MONTH");
let interval = format!("{} {}", num, date_part);
(
sql_generator
.get_sql_templates()
.interval_expr(interval, num.into(), date_part.to_string())
.map_err(|e| {
DataFusionError::Internal(format!(

Check warning on line 1278 in rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs#L1277-L1278 

Added lines #L1277 - L1278 were not covered by tests
"Can't generate SQL for interval: {}",
e
))
})?,
sql_query,
)
} else {
("NULL".to_string(), sql_query)

Check warning on line 1286 in rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs

View check run for this annotation

Codecov / codecov/patch

rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs#L1286 

Added line #L1286 was not covered by tests
}
}
ScalarValue::IntervalDayTime(x) => {
if let Some(x) = x {
let days = x >> 32;
Expand Down
Loading

0 comments on commit 1cd1b22

Please sign in to comment.