feat: implement use the '-' mark to ignore writing values for field (#56

)

* feat: implement use the '-' mark to ignore writing values for field

* refactor: ignore fields

* chore: change the name from 'ignore' to 'skip'

* refactor: Remove the ignorefields example and use simple unit test

README.md

@@ -130,7 +130,9 @@ If you prefer ORM style, and define column-field relationship via struct field t
 - `type` is to define the data type. if type is timestamp, `precision` is supported
 - the metadata separator is `;` and the key value separator is `:`
 
-type supported is the same as described [Datatypes supported](#datatypes-supported), and case insensitive
+type supported is the same as described [Datatypes supported](#datatypes-supported), and case insensitive.
+
+When fields marked with `greptime:"-"`, writing field will be ignored.
 
 ##### define struct with tags
 

schema/field.go

@@ -52,6 +52,10 @@ func newColumnSchema(columnName string, semanticType gpb.SemanticType, datatype
 func parseField(structField reflect.StructField) (*Field, error) {
 	tags := parseTag(structField)
 
+	if _, ok := tags[IgnoreFiledTag]; ok && len(tags) == 1 {
+		return nil, nil
+	}
+
 	columnName, err := util.SanitateName(structField.Name)
 	if err != nil {
 		return nil, err
@@ -85,7 +89,7 @@ func parseField(structField reflect.StructField) (*Field, error) {
 func parseTag(structField reflect.StructField) map[string]string {
 	tags := map[string]string{}
 
-	str, ok := structField.Tag.Lookup("greptime")
+	str, ok := structField.Tag.Lookup(GreptimeFieldTagKey)
 	if !ok {
 		return tags
 	}

schema/schema.go

@@ -25,6 +25,11 @@ import (
 	"github.com/GreptimeTeam/greptimedb-ingester-go/util"
 )
 
+const (
+	IgnoreFiledTag      = "-"
+	GreptimeFieldTagKey = "greptime"
+)
+
 type Schema struct {
 	tableName string
 
@@ -52,7 +57,7 @@ func getTableName(typ reflect.Type) (string, error) {
 
 func Parse(input any) (*table.Table, error) {
 	if input == nil {
-		return nil, fmt.Errorf("unsupported empty data. %#v", input)
+		return nil, fmt.Errorf("unsupported empty data: %#v", input)
 	}
 
 	schema_, err := parseSchema(input)
@@ -114,7 +119,9 @@ func parseSchema(input any) (*Schema, error) {
 		if err != nil {
 			return nil, err
 		}
-		fields = append(fields, field.ToColumnSchema())
+		if field != nil {
+			fields = append(fields, field.ToColumnSchema())
+		}
 	}
 
 	return &Schema{tableName: tableName, fields: fields}, nil
@@ -142,15 +149,19 @@ func (s *Schema) parseValues(input any) error {
 		return fmt.Errorf("unsupported type %T of %+v", input, input)
 	}
 
-	size := len(reflect.VisibleFields(typ))
-	values := make([]*gpb.Value, 0, size)
-	for i, structField := range reflect.VisibleFields(typ) {
-		if !structField.IsExported() {
+	visibleFields := reflect.VisibleFields(typ)
+	processingFields := make([]reflect.StructField, 0, len(visibleFields))
+	values := make([]*gpb.Value, 0, len(processingFields))
+
+	for _, structField := range visibleFields {
+		if !structField.IsExported() || structField.Tag.Get(GreptimeFieldTagKey) == IgnoreFiledTag {
 			continue
 		}
+		processingFields = append(processingFields, structField)
+	}
 
+	for i, structField := range processingFields {
 		field := s.fields[i]
-
 		value, err := parseValue(field.Datatype, val.FieldByName(structField.Name))
 		if err != nil {
 			return err

schema/schema_test.go

@@ -975,3 +975,96 @@ func TestParseWithUnmatchedDatatype(t *testing.T) {
 		assert.Nil(t, tbl)
 	}
 }
+
+func TestParseSchemaWithIgnoreFields(t *testing.T) {
+	INT16 := int16(1)
+	UINT64 := uint64(2)
+	FLOAT32 := float32(3)
+	STRING := "string"
+	JSON := `{"key1":"value1","key2":10}`
+
+	TIMESTAMP := time.Now()
+	DATETIME_INT := TIMESTAMP.UnixMilli()
+
+	length := 6
+	assertSchema := func(cols []*gpb.ColumnSchema) {
+		assert.Len(t, cols, length)
+
+		assert.EqualValues(t, newColumnSchema("uint64_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_UINT64), cols[0])
+		assert.EqualValues(t, newColumnSchema("string_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_STRING), cols[1])
+		assert.EqualValues(t, newColumnSchema("json_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_JSON), cols[2])
+
+		offset := length / 2
+		assert.EqualValues(t, newColumnSchema("ptr_uint64_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_UINT64), cols[0+offset])
+		assert.EqualValues(t, newColumnSchema("ptr_string_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_STRING), cols[1+offset])
+		assert.EqualValues(t, newColumnSchema("ptr_json_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_JSON), cols[2+offset])
+	}
+
+	assertValue := func(row *gpb.Row) {
+		vals := row.Values
+		assert.Len(t, vals, length)
+
+		assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_U64Value{U64Value: UINT64}}, vals[0])
+		assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: STRING}}, vals[1])
+		assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: JSON}}, vals[2])
+
+		offset := length / 2
+
+		assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_U64Value{U64Value: UINT64}}, vals[0+offset])
+		assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: STRING}}, vals[1+offset])
+		assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: JSON}}, vals[2+offset])
+	}
+
+	type Monitor struct {
+		INT16        int16   `greptime:"-"`
+		UINT64       uint64  `greptime:"field;column:uint64_column;type:uint64"`
+		FLOAT32      float32 `greptime:"-"`
+		STRING       string  `greptime:"field;column:string_column;type:string"`
+		DATETIME_INT int64   `greptime:"-"`
+		JSON         string  `greptime:"field;column:json_column;type:json"`
+
+		PtrINT16        *int16   `greptime:"-"`
+		PtrUINT64       *uint64  `greptime:"field;column:ptr_uint64_column;type:uint64"`
+		PtrFLOAT32      *float32 `greptime:"-"`
+		PtrSTRING       *string  `greptime:"field;column:ptr_string_column;type:string"`
+		PtrDATETIME_INT *int64   `greptime:"-"`
+		PtrJSON         *string  `greptime:"field;column:ptr_json_column;type:json"`
+
+		privateField string // will be ignored
+	}
+
+	monitor := Monitor{
+		INT16:        INT16,
+		UINT64:       UINT64,
+		FLOAT32:      FLOAT32,
+		STRING:       STRING,
+		DATETIME_INT: DATETIME_INT,
+		JSON:         JSON,
+
+		PtrINT16:        &INT16,
+		PtrUINT64:       &UINT64,
+		PtrFLOAT32:      &FLOAT32,
+		PtrSTRING:       &STRING,
+		PtrDATETIME_INT: &DATETIME_INT,
+		PtrJSON:         &JSON,
+
+		privateField: "private",
+	}
+
+	{
+		tbl, err := Parse(monitor)
+		assert.Nil(t, err)
+		assert.NotNil(t, tbl)
+
+		rows := tbl.GetRows()
+		assert.NotNil(t, rows)
+
+		assertSchema(rows.Schema)
+
+		assert.Len(t, rows.Rows, 1)
+		assert.Len(t, rows.Rows[0].Values, length)
+		for _, row := range rows.Rows {
+			assertValue(row)
+		}
+	}
+}