Skip to content

Commit

Permalink
feat: support json type (#51)
Browse files Browse the repository at this point in the history
* feat: support json type

* feat: add readme

* feat: add json type test

* fix: ci failed

* chore: revert package import
  • Loading branch information
daviderli614 authored Nov 12, 2024
1 parent 5a19d5e commit acfc40f
Show file tree
Hide file tree
Showing 11 changed files with 222 additions and 9 deletions.
2 changes: 1 addition & 1 deletion client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ func newMysql() *Mysql {

func init() {
repo := "greptime/greptimedb"
tag := "v0.7.0"
tag := "v0.9.5"

pool, err := dockertest.NewPool("")
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ docker run --rm -p 4000-4003:4000-4003 \
- [healthcheck](healthcheck/README.md)
- [opentelemetry](opentelemetry/README.md)
- [hints](hints/README.md)
- [jsondata](jsondata/README.md)

## Query

Expand Down
48 changes: 48 additions & 0 deletions examples/jsondata/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# Insert data into GreptimeDB

## Insert

```go
go run main.go
```

Output:

```log
2024/11/11 14:59:56 affected rows: 1
```

## Query

Your can using [MySQL Client](https://docs.greptime.com/user-guide/protocols/mysql) to query the data from GreptimeDB.

```shell
$ mysql -h 127.0.0.1 -P 4002

mysql> select * from json_data;
+-----------------------------------------------------------------------------------------+----------------------------+
| my_json | timestamp |
+-----------------------------------------------------------------------------------------+----------------------------+
| {"Age":25,"Courses":["math","history","chemistry"],"IsStudent":false,"Name":"Jain Doe"} | 2024-11-11 06:59:56.340132 |
+-----------------------------------------------------------------------------------------+----------------------------+
1 row in set (0.04 sec)
```
You can view table fields using `show create table` command:
```mysql
mysql> show create table json_data;
+-----------+-------------------------------------------------------------------------------------------------------------------------------------------------+
| Table | Create Table |
+-----------+-------------------------------------------------------------------------------------------------------------------------------------------------+
| json_data | CREATE TABLE IF NOT EXISTS `json_data` (
`my_json` JSON NULL,
`timestamp` TIMESTAMP(6) NOT NULL,
TIME INDEX (`timestamp`)
)

ENGINE=mito
|
+-----------+-------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)
```
119 changes: 119 additions & 0 deletions examples/jsondata/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
// Copyright 2024 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"context"
"encoding/json"
"log"
"time"

greptime "github.com/GreptimeTeam/greptimedb-ingester-go"
"github.com/GreptimeTeam/greptimedb-ingester-go/table"
"github.com/GreptimeTeam/greptimedb-ingester-go/table/types"
)

const (
// The GreptimeDB address.
host = "127.0.0.1"

// The database name.
database = "public"
)

type Person struct {
Name string
Age int
IsStudent bool
Courses []string
}

type client struct {
client *greptime.Client
}

func newClient() (*client, error) {
cfg := greptime.NewConfig(host).WithDatabase(database)

gtClient, err := greptime.NewClient(cfg)
if err != nil {
return nil, err
}

c := &client{
client: gtClient,
}

return c, nil
}

func main() {
c, err := newClient()
if err != nil {
log.Fatalf("failed to new client: %v:", err)
}

data, err := initData()
if err != nil {
log.Fatalf("failed to init data: %v:", err)
}
if err = c.write(data[0]); err != nil {
log.Fatalf("failed to write data: %v:", err)
}
}

func initData() ([]*table.Table, error) {
time1 := time.Now()

itbl, err := table.New("json_data")
if err != nil {
return nil, err
}

p := Person{
Name: "Jain Doe",
Age: 25,
IsStudent: false,
Courses: []string{"math", "history", "chemistry"},
}
jsonData, err := json.Marshal(p)
if err != nil {
return nil, err
}

// add column at first. This is to define the schema of the table.
if err := itbl.AddFieldColumn("my_json", types.JSON); err != nil {
return nil, err
}
if err := itbl.AddTimestampColumn("timestamp", types.TIMESTAMP_MICROSECOND); err != nil {
return nil, err
}
if err := itbl.AddRow(string(jsonData), time1); err != nil {
return nil, err
}

return []*table.Table{itbl}, nil
}

func (c client) write(data *table.Table) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
defer cancel()
resp, err := c.client.Write(ctx, data)
if err != nil {
return err
}
log.Printf("affected rows: %d\n", resp.GetAffectedRows().GetValue())
return nil
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/GreptimeTeam/greptimedb-ingester-go
go 1.20

require (
github.com/GreptimeTeam/greptime-proto v0.7.0
github.com/GreptimeTeam/greptime-proto v0.9.0
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/stoewer/go-strcase v1.3.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/GreptimeTeam/greptime-proto v0.7.0 h1:WHBjAu+NWDFcbZgW9kPtksxEKEAeqYemP1HY63QuO48=
github.com/GreptimeTeam/greptime-proto v0.7.0/go.mod h1:jk5XBR9qIbSBiDF2Gix1KALyIMCVktcpx91AayOWxmE=
github.com/GreptimeTeam/greptime-proto v0.9.0 h1:UC2vhGEQun75aejAyr6SdHTMjHBM5dlSMDr72ue0U8Y=
github.com/GreptimeTeam/greptime-proto v0.9.0/go.mod h1:jk5XBR9qIbSBiDF2Gix1KALyIMCVktcpx91AayOWxmE=
github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg=
github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
Expand Down
8 changes: 7 additions & 1 deletion schema/field.go
Original file line number Diff line number Diff line change
Expand Up @@ -253,9 +253,15 @@ func parseValue(typ gpb.ColumnDataType, val reflect.Value) (*gpb.Value, error) {
// TODO(yuanbohan): support decimal 128
case gpb.ColumnDataType_DECIMAL128:
return nil, fmt.Errorf("DECIMAL 128 not supported for %#v", val)

case gpb.ColumnDataType_JSON:
if val.Kind() != reflect.String {
return nil, fmt.Errorf("%#v is not compatible with String", val)
}
return cell.New(val.String(), typ).Build()

default:
return nil, fmt.Errorf("unknown column data type: %v", typ)

}
}

Expand Down
24 changes: 18 additions & 6 deletions schema/schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,7 @@ func TestParseWithValues(t *testing.T) {
BOOLEAN := true
BINARY := []byte{1, 2, 3}
STRING := "string"
JSON := `{"key1":"value1","key2":10}`

TIMESTAMP := time.Now()
DATE_INT := TIMESTAMP.Unix() / int64(cell.ONE_DAY_IN_SECONDS)
Expand All @@ -342,7 +343,7 @@ func TestParseWithValues(t *testing.T) {
TIMESTAMP_NANOSECOND_INT := TIMESTAMP.UnixNano()

assertSchema := func(cols []*gpb.ColumnSchema) {
assert.Len(t, cols, 54)
assert.Len(t, cols, 56)

assert.EqualValues(t, newColumnSchema("int_column", gpb.SemanticType_TAG, gpb.ColumnDataType_INT64), cols[0])
assert.EqualValues(t, newColumnSchema("int8_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_INT8), cols[1])
Expand Down Expand Up @@ -371,8 +372,9 @@ func TestParseWithValues(t *testing.T) {
assert.EqualValues(t, newColumnSchema("timestamp_millisecond_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_TIMESTAMP_MILLISECOND), cols[24])
assert.EqualValues(t, newColumnSchema("timestamp_microsecond_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_TIMESTAMP_MICROSECOND), cols[25])
assert.EqualValues(t, newColumnSchema("timestamp_nanosecond_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_TIMESTAMP_NANOSECOND), cols[26])
assert.EqualValues(t, newColumnSchema("json_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_JSON), cols[27])

offset := 27
offset := 28
assert.EqualValues(t, newColumnSchema("ptr_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_INT64), cols[0+offset])
assert.EqualValues(t, newColumnSchema("ptr_int8_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_INT8), cols[1+offset])
assert.EqualValues(t, newColumnSchema("ptr_int16_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_INT16), cols[2+offset])
Expand Down Expand Up @@ -400,11 +402,12 @@ func TestParseWithValues(t *testing.T) {
assert.EqualValues(t, newColumnSchema("ptr_timestamp_millisecond_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_TIMESTAMP_MILLISECOND), cols[24+offset])
assert.EqualValues(t, newColumnSchema("ptr_timestamp_microsecond_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_TIMESTAMP_MICROSECOND), cols[25+offset])
assert.EqualValues(t, newColumnSchema("ptr_timestamp_nanosecond_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_TIMESTAMP_NANOSECOND), cols[26+offset])
assert.EqualValues(t, newColumnSchema("ptr_json_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_JSON), cols[27+offset])
}

assertValue := func(row *gpb.Row) {
vals := row.Values
assert.Len(t, vals, 54)
assert.Len(t, vals, 56)

assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_I64Value{I64Value: int64(INT)}}, vals[0])
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_I8Value{I8Value: int32(INT8)}}, vals[1])
Expand Down Expand Up @@ -433,8 +436,9 @@ func TestParseWithValues(t *testing.T) {
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_TimestampMillisecondValue{TimestampMillisecondValue: TIMESTAMP.UnixMilli()}}, vals[24])
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_TimestampMicrosecondValue{TimestampMicrosecondValue: TIMESTAMP.UnixMicro()}}, vals[25])
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_TimestampNanosecondValue{TimestampNanosecondValue: TIMESTAMP.UnixNano()}}, vals[26])
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: JSON}}, vals[27])

offset := 27
offset := 28

assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_I64Value{I64Value: int64(INT)}}, vals[0+offset])
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_I8Value{I8Value: int32(INT8)}}, vals[1+offset])
Expand Down Expand Up @@ -463,6 +467,7 @@ func TestParseWithValues(t *testing.T) {
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_TimestampMillisecondValue{TimestampMillisecondValue: TIMESTAMP.UnixMilli()}}, vals[24+offset])
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_TimestampMicrosecondValue{TimestampMicrosecondValue: TIMESTAMP.UnixMicro()}}, vals[25+offset])
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_TimestampNanosecondValue{TimestampNanosecondValue: TIMESTAMP.UnixNano()}}, vals[26+offset])
assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: JSON}}, vals[27+offset])
}

type Monitor struct {
Expand Down Expand Up @@ -493,6 +498,7 @@ func TestParseWithValues(t *testing.T) {
TIMESTAMP_MILLISECOND_INT int64 `greptime:"field;column:timestamp_millisecond_int_column;type:timestamp;precision:millisecond"`
TIMESTAMP_MICROSECOND_INT int64 `greptime:"field;column:timestamp_microsecond_int_column;type:timestamp;precision:microsecond"`
TIMESTAMP_NANOSECOND_INT int64 `greptime:"field;column:timestamp_nanosecond_int_column;type:timestamp;precision:nanosecond"`
JSON string `greptime:"field;column:json_column;type:json"`

PtrINT *int `greptime:"field;column:ptr_int_column;type:int"`
PtrINT8 *int8 `greptime:"field;column:ptr_int8_column;type:int8"`
Expand Down Expand Up @@ -521,6 +527,7 @@ func TestParseWithValues(t *testing.T) {
PtrTIMESTAMP_MILLISECOND_INT *int64 `greptime:"field;column:ptr_timestamp_millisecond_int_column;type:timestamp;precision:millisecond"`
PtrTIMESTAMP_MICROSECOND_INT *int64 `greptime:"field;column:ptr_timestamp_microsecond_int_column;type:timestamp;precision:microsecond"`
PtrTIMESTAMP_NANOSECOND_INT *int64 `greptime:"field;column:ptr_timestamp_nanosecond_int_column;type:timestamp;precision:nanosecond"`
PtrJSON *string `greptime:"field;column:ptr_json_column;type:json"`

privateField string // will be ignored
}
Expand Down Expand Up @@ -553,6 +560,7 @@ func TestParseWithValues(t *testing.T) {
TIMESTAMP_MILLISECOND_INT: TIMESTAMP_MILLISECOND_INT,
TIMESTAMP_MICROSECOND_INT: TIMESTAMP_MICROSECOND_INT,
TIMESTAMP_NANOSECOND_INT: TIMESTAMP_NANOSECOND_INT,
JSON: JSON,

PtrINT: &INT,
PtrINT8: &INT8,
Expand Down Expand Up @@ -581,6 +589,7 @@ func TestParseWithValues(t *testing.T) {
PtrTIMESTAMP_MILLISECOND_INT: &TIMESTAMP_MILLISECOND_INT,
PtrTIMESTAMP_MICROSECOND_INT: &TIMESTAMP_MICROSECOND_INT,
PtrTIMESTAMP_NANOSECOND_INT: &TIMESTAMP_NANOSECOND_INT,
PtrJSON: &JSON,

privateField: "private",
}
Expand Down Expand Up @@ -715,6 +724,7 @@ func TestParseWithNilValues(t *testing.T) {
PtrTIMESTAMP_MILLISECOND_INT *int64 `greptime:"field;column:ptr_timestamp_millisecond_int_column;type:timestamp;precision:millisecond"`
PtrTIMESTAMP_MICROSECOND_INT *int64 `greptime:"field;column:ptr_timestamp_microsecond_int_column;type:timestamp;precision:microsecond"`
PtrTIMESTAMP_NANOSECOND_INT *int64 `greptime:"field;column:ptr_timestamp_nanosecond_int_column;type:timestamp;precision:nanosecond"`
PtrJSON *string `greptime:"field;column:ptr_json_column;type:json"`
}

monitor := Monitor{
Expand Down Expand Up @@ -745,6 +755,7 @@ func TestParseWithNilValues(t *testing.T) {
PtrTIMESTAMP_MILLISECOND_INT: nil,
PtrTIMESTAMP_MICROSECOND_INT: nil,
PtrTIMESTAMP_NANOSECOND_INT: nil,
PtrJSON: nil,
}

{
Expand All @@ -756,7 +767,7 @@ func TestParseWithNilValues(t *testing.T) {
assert.NotNil(t, rows)

cols := rows.Schema
assert.Len(t, cols, 27)
assert.Len(t, cols, 28)

assert.EqualValues(t, newColumnSchema("ptr_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_INT64), cols[0])
assert.EqualValues(t, newColumnSchema("ptr_int8_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_INT8), cols[1])
Expand Down Expand Up @@ -785,10 +796,11 @@ func TestParseWithNilValues(t *testing.T) {
assert.EqualValues(t, newColumnSchema("ptr_timestamp_millisecond_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_TIMESTAMP_MILLISECOND), cols[24])
assert.EqualValues(t, newColumnSchema("ptr_timestamp_microsecond_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_TIMESTAMP_MICROSECOND), cols[25])
assert.EqualValues(t, newColumnSchema("ptr_timestamp_nanosecond_int_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_TIMESTAMP_NANOSECOND), cols[26])
assert.EqualValues(t, newColumnSchema("ptr_json_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_JSON), cols[27])

assert.Len(t, rows.Rows, 1)
vals := rows.Rows[0].Values
assert.Len(t, vals, 27)
assert.Len(t, vals, 28)
for _, val := range vals {
assert.Nil(t, val)
}
Expand Down
14 changes: 14 additions & 0 deletions table/cell/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -490,3 +490,17 @@ func BuildTimeNanosecond(v any) (*gpb.Value, error) {

return &gpb.Value{ValueData: &gpb.Value_TimeNanosecondValue{TimeNanosecondValue: val}}, nil
}

func BuildJSON(v any) (*gpb.Value, error) {
var val string
switch t := v.(type) {
case string:
val = t
case *string:
val = *t
default:
return nil, fmt.Errorf(formatter+"string", t, v)
}

return &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: val}}, nil
}
3 changes: 3 additions & 0 deletions table/cell/cell.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ func (c Cell) Build() (*gpb.Value, error) {
// TODO(yuanbohan): support decimal 128
case gpb.ColumnDataType_DECIMAL128:
return nil, fmt.Errorf("DECIMAL 128 not supported for %#v", c.Val)

case gpb.ColumnDataType_JSON:
return BuildJSON(c.Val)
default:
return nil, fmt.Errorf("unknown column data type: %v", c.DataType)
}
Expand Down
Loading

0 comments on commit acfc40f

Please sign in to comment.