Skip to content

Commit

Permalink
feat: add sql support (#12)
Browse files Browse the repository at this point in the history
* feat: sql

* refactor: use mixin

* ts

* fix ts

* fix query

* fix sql type

* support ts range query for sql

* null sql

* use table name in sql

* fix no meta info

* feat: default query type

* fix: sqltype
  • Loading branch information
sunchanglong authored Dec 17, 2024
1 parent ad9deb4 commit abed377
Show file tree
Hide file tree
Showing 65 changed files with 5,185 additions and 42 deletions.
6 changes: 6 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,20 @@
"@lezer/highlight": "^1.2.0",
"@lezer/lr": "^1.4.0",
"@prometheus-io/lezer-promql": "^0.37.0-rc.1",
"@react-awesome-query-builder/core": "^6.6.3",
"@react-awesome-query-builder/ui": "^6.6.3",
"@reduxjs/toolkit": "^2.2.3",
"@testing-library/user-event": "^14.5.2",
"debounce-promise": "^3.1.2",
"monaco-promql": "^1.7.4",
"pluralize": "^8.0.0",
"react": "18.2.0",
"react-awesome-query-builder": "^5.4.2",
"react-dom": "18.2.0",
"react-use": "^17.5.1",
"react-virtualized-auto-sizer": "^1.0.24",
"semver": "^7.6.0",
"sql-formatter-plus": "^1.3.6",
"tslib": "2.5.3"
},
"overrides": {
Expand Down
7 changes: 5 additions & 2 deletions src/components/PromQueryEditorByApp.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ import { CoreApp } from '@grafana/data';
import { PromQueryEditorSelector } from '../querybuilder/components/PromQueryEditorSelector';

import { PromQueryEditorForAlerting } from './PromQueryEditorForAlerting';
import { PromQueryEditorProps } from './types';
import { PromQueryEditorProps, SqlQueryEditorProps } from './types';
import QueryWrapper from '../querybuilder/QueryWrapper';
import { QueryEditorProperty } from 'querybuilder/mysql/sql/expressions';


export function PromQueryEditorByApp(props: PromQueryEditorProps) {
const { app } = props;
Expand All @@ -14,7 +17,7 @@ export function PromQueryEditorByApp(props: PromQueryEditorProps) {
case CoreApp.CloudAlerting:
return <PromQueryEditorForAlerting {...props} />;
default:
return <PromQueryEditorSelector {...props} />;
return <QueryWrapper {...(props as PromQueryEditorProps & SqlQueryEditorProps)} />;
}
}

Expand Down
4 changes: 4 additions & 0 deletions src/components/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,9 @@ import { QueryEditorProps } from '@grafana/data';

import { PrometheusDatasource } from '../datasource';
import { PromOptions, PromQuery } from '../types';
import { SqlDatasource } from 'querybuilder/mysql/sql/datasource/SqlDatasource';
import { SQLOptions, SQLQuery } from 'querybuilder/mysql/sql';

export type PromQueryEditorProps = QueryEditorProps<PrometheusDatasource, PromQuery, PromOptions>;

export type SqlQueryEditorProps = QueryEditorProps<SqlDatasource, SQLQuery, SQLOptions>
85 changes: 81 additions & 4 deletions src/datasource.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { cloneDeep, defaults } from 'lodash';
import { lastValueFrom, Observable, throwError } from 'rxjs';
import { map, tap } from 'rxjs/operators';
import { cloneDeep, defaults, extend } from 'lodash';
import { lastValueFrom, Observable, throwError, of } from 'rxjs';
import { map, tap, switchMap } from 'rxjs/operators';
import semver from 'semver';
import store from 'app/core/store';

import {
AbstractQuery,
Expand Down Expand Up @@ -65,11 +66,16 @@ import {
} from './types';
import { PrometheusVariableSupport } from './variables';

import { transformSqlResponse, addTsCondition } from './greptimedb'
import {MySqlDatasource} from './querybuilder/mysql/MySqlDatasource'
import { SQLExpression, SQLOptions, SQLQuery } from 'querybuilder/mysql/sql/types';

const ANNOTATION_QUERY_STEP_DEFAULT = '60s';
const GET_AND_POST_METADATA_ENDPOINTS = ['v1/prometheus/api/v1/query', 'v1/prometheus/api/v1/query_range', 'v1/prometheus/api/v1/series', 'v1/prometheus/api/v1/labels'];

export const InstantQueryRefIdIndex = '-Instant';


export class PrometheusDatasource
extends DataSourceWithBackend<PromQuery, PromOptions>
implements DataSourceWithQueryImportSupport<PromQuery>, DataSourceWithQueryExportSupport<PromQuery>
Expand Down Expand Up @@ -385,7 +391,7 @@ export class PrometheusDatasource
return processedTargets;
}

query(request: DataQueryRequest<PromQuery>): Observable<DataQueryResponse> {
executePromQuery(request: DataQueryRequest<PromQuery>): Observable<DataQueryResponse> {
if (this.access === 'direct') {
return this.directAccessError();
}
Expand Down Expand Up @@ -968,6 +974,77 @@ export class PrometheusDatasource
}
}

function isPromQuery(query: PromQuery | SQLQuery): query is PromQuery {
if (query.sqltype === 'sql') {
return false
}
return true
// return store.get('sqltype') === 'promql' || query.sqltype === 'promql' || !query.sqltype
}

function mixin (thisObj, instance) {
Object.assign(thisObj, instance)
const proto = Object.getPrototypeOf(instance);
const parentProto = Object.getPrototypeOf(proto)

// Get all property names (including methods) of the prototype
const methods = [...Object.getOwnPropertyNames(parentProto), ...Object.getOwnPropertyNames(proto)].filter(prop => typeof proto[prop] === 'function' && prop !== 'query' && prop !== 'contructor')
for (const method of methods) {
thisObj[method] = instance[method]
}
}

const TableNameReg = /(?<=from|FROM)\s+([^\s;]+)/
export class GreptimeDBDatasource extends DataSourceWithBackend {
constructor(
instanceSettings: DataSourceInstanceSettings<PromOptions>,
private readonly templateSrv: TemplateSrv = getTemplateSrv(),
languageProvider?: PrometheusLanguageProvider
) {
super(instanceSettings)
const promInstance = new PrometheusDatasource(instanceSettings, templateSrv, languageProvider)
const mysqlInstance = new MySqlDatasource(instanceSettings as DataSourceInstanceSettings<SQLOptions>)
mixin(this, mysqlInstance)
mixin(this, promInstance)

}
query(request: DataQueryRequest<PromQuery | SQLQuery>): Observable<DataQueryResponse> {
if (!isPromQuery(request.targets[0])) {
if (!request.targets[0].rawSql) {
return Promise.resolve({data: []})
}
const promises = (request.targets as SQLQuery[]).map(async (target) => {
// console.log(target)
let table = target.table
let dataset = target.dataset
if (!table) {
const result = target.rawSql?.match(TableNameReg)
if (result && result.length) {
table = result[1].trim()
dataset = undefined
}
}
return this.fetchFields({dataset: dataset, table: table}).then(columns => {
return columns.filter(column => column.type.indexOf('timestamp') > -1)[0]

}).then((tsColumn) => {
let sql = target.rawSql
if (tsColumn) {
sql = addTsCondition(target.rawSql, tsColumn.name, request.range.from.toISOString(), request.range.to.toISOString())
}
return lastValueFrom(transformSqlResponse((this as any)._request('/v1/sql', {sql: sql})))
})
// const sql = addTsCondition(target.rawSql, request.range.from.toISOString(), request.range.to.toISOString())

})
// TODO fix ts
return Promise.all(promises).then((data) => ({ data })) as unknown as Observable<DataQueryResponse>
} else {
return (this as any).executePromQuery(request)
}
}
}

/**
* Align query range to step.
* Rounds start and end down to a multiple of step.
Expand Down
152 changes: 152 additions & 0 deletions src/greptimedb/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
import { FieldType, MutableDataFrame } from '@grafana/data';
import { lastValueFrom, Observable, throwError, of } from 'rxjs';
import { map, tap, switchMap } from 'rxjs/operators';
import {
FetchResponse,
} from '@grafana/runtime';
import { GreptimeDataTypes } from './types';

export const greptimeTypeToGrafana: Record<GreptimeDataTypes, FieldType> = {
[GreptimeDataTypes.Null]: FieldType.other,

// Numeric types:
[GreptimeDataTypes.Boolean]: FieldType.boolean,
[GreptimeDataTypes.UInt8]: FieldType.number,
[GreptimeDataTypes.UInt16]: FieldType.number,
[GreptimeDataTypes.UInt32]: FieldType.number,
[GreptimeDataTypes.UInt64]: FieldType.number,
[GreptimeDataTypes.Int8]: FieldType.number,
[GreptimeDataTypes.Int16]: FieldType.number,
[GreptimeDataTypes.Int32]: FieldType.number,
[GreptimeDataTypes.Int64]: FieldType.number,
[GreptimeDataTypes.Float32]: FieldType.number,
[GreptimeDataTypes.Float64]: FieldType.number,

// String types:
[GreptimeDataTypes.String]: FieldType.string,
[GreptimeDataTypes.Binary]: FieldType.string,

// Date & Time types:
[GreptimeDataTypes.Date]: FieldType.time,
[GreptimeDataTypes.DateTime]: FieldType.time,

[GreptimeDataTypes.TimestampSecond]: FieldType.time,
[GreptimeDataTypes.TimestampMillisecond]: FieldType.time,
[GreptimeDataTypes.TimestampMicrosecond]: FieldType.time,
[GreptimeDataTypes.TimestampNanosecond]: FieldType.time,

[GreptimeDataTypes.List]: FieldType.other,
};

function buildDataFrame(columns, rows) {
const frame = new MutableDataFrame();

// Example: Assuming `sqlResult` is an array of rows
if (rows.length > 0) {
// Get column names from the first row

// Create fields (columns) for the data frame
columns.forEach((col, index) => {
frame.addField({
name: col.name,
values: rows.map(row => row[index]),
type: greptimeTypeToGrafana[col.data_type],
});
});
}

return frame;
}

// Utility function to determine field type (number, string, time, etc.)
function getFieldType(values: any[]) {
if (typeof values[0] === 'number') {
return 'number';
} else if (values[0] instanceof Date) {
return 'time';
} else {
return 'string';
}
}

export function transformSqlResponse(response: Observable<FetchResponse>) {
return response.pipe(switchMap((raw) => {
// console.log(raw)

// const rsp = toDataQueryResponse(raw, queries as DataQuery[]);
// // Check if any response should subscribe to a live stream
// if (rsp.data?.length && rsp.data.find((f: DataFrame) => f.meta?.channel)) {
// return toStreamingDataResponse(rsp, request, this.streamOptionsProvider);
// }
return of(raw);
})).pipe(map(data => {
// console.log(data)
return data.data
})).pipe(map(
response => {
// console.log(response)
const columnSchemas = response.output[0].records.schema.column_schemas;
const dataRows = response.output[0].records.rows;

const frame = new MutableDataFrame({
refId: 'A',
fields: columnSchemas.map((columnSchema, idx) => {
return {
name: columnSchema.name,
type: greptimeTypeToGrafana[columnSchema.data_type],
values: dataRows.map((row) => row[idx]),
};
}),
});
// const frame = buildDataFrame(columnSchemas, dataRows)
// const result = {
// data: {
// ...frame.toJSON(),
// refId: 'A'
// },
// state: 'Done'
// }
// console.log(result, 'result')
return frame
}
))
}

export function addTsCondition (sql, column, start, end) {
const upperSql = sql.toUpperCase();
const whereIndex = upperSql.indexOf('WHERE')
if (whereIndex > -1) {
return sql.slice(0, whereIndex + 5) + ` ${column} >= '${start}' and ${column} < '${end}' and ` + sql.slice(whereIndex + 5)
} else {
const whereIndex = findWhereClausePosition(sql);
return sql.slice(0, whereIndex) + ` where ${column} >= '${start}' and ${column} < '${end}' ` + sql.slice(whereIndex)
}
}

function findWhereClausePosition(sql) {
// Normalize case for easier comparison
const upperSql = sql.toUpperCase();

// Find the first keyword after FROM where WHERE should go before
const groupByIndex = upperSql.indexOf('GROUP BY');
const orderByIndex = upperSql.indexOf('ORDER BY');
const limitIndex = upperSql.indexOf('LIMIT');

// Find the position to insert WHERE clause:
// Insert before GROUP BY, ORDER BY, or LIMIT, whichever comes first
let insertPosition = upperSql.length; // Default to end of the query if no keywords

if (groupByIndex !== -1 && groupByIndex < insertPosition) {
insertPosition = groupByIndex;
}

if (orderByIndex !== -1 && orderByIndex < insertPosition) {
insertPosition = orderByIndex;
}

if (limitIndex !== -1 && limitIndex < insertPosition) {
insertPosition = limitIndex;
}

return insertPosition;
}
Loading

0 comments on commit abed377

Please sign in to comment.