diff --git a/README.md b/README.md index 02697d2..a2825b1 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,6 @@ qq large-file.json -o yaml > /dev/null 2>&1 2.72s user 0.16s system 190% cpu 1. ``` ## Supported Formats -Note: these unsupported formats are on a roadmap for inclusion. | Format | Input | Output | |-------------|----------------|----------------| | JSON | ✅ Supported | ✅ Supported | @@ -106,19 +105,18 @@ Note: these unsupported formats are on a roadmap for inclusion. | TF | ✅ Supported | ✅ Supported | | GRON | ✅ Supported | ✅ Supported | | CSV | ✅ Supported | ✅ Supported | -| Protobuf | ❌ Not Supported | ❌ Not Supported | +| Proto (.proto) | ✅ Supported | ❌ Not Supported | | HTML | ✅ Supported | ✅ Supported | | TXT (newline)| ✅ Supported | ❌ Not Supported | ## Caveats -1. `qq` is not a full `jq`/`*q` replacement and comes with idiosyncrasies from the underlying `gojq` library. -2. the encoders and decoders are not perfect and may not be able to handle all edge cases. -3. `qq` is under active development and more codecs are intended to be supported along with improvements to `interactive mode`. +1. `qq` is not a full `jq` replacement, some flags may or may not be supported. +3. `qq` is under active development, more codecs in the future may be supported along with improvements to `interactive mode`. ## Contributions -All contributions are welcome to `qq`, especially for upkeep/optimization/addition of new encodings. For ideas on contributions [please refer to the todo docs](https://github.com/JFryy/qq/blob/main/docs/TODO.md) or make an issue/PR for a suggestion if there's something that's wanted or fixes. +All contributions are welcome to `qq`, especially for upkeep/optimization/addition of new encodings. ## Thanks and Acknowledgements / Related Projects This tool would not be possible without the following projects, this project is arguably more of a composition of these projects than a truly original work, with glue code, some dedicated encoders/decoders, and the interactive mode being original work. diff --git a/codec/codec.go b/codec/codec.go index f93ac30..d9e1577 100644 --- a/codec/codec.go +++ b/codec/codec.go @@ -21,6 +21,7 @@ import ( "github.com/JFryy/qq/codec/ini" qqjson "github.com/JFryy/qq/codec/json" "github.com/JFryy/qq/codec/line" + proto "github.com/JFryy/qq/codec/proto" "github.com/JFryy/qq/codec/xml" ) @@ -41,11 +42,11 @@ const ( HTML LINE TXT - MD + PROTO ) func (e EncodingType) String() string { - return [...]string{"json", "yaml", "yml", "toml", "hcl", "tf", "csv", "xml", "ini", "gron", "html", "line", "txt", "md"}[e] + return [...]string{"json", "yaml", "yml", "toml", "hcl", "tf", "csv", "xml", "ini", "gron", "html", "line", "txt", "proto"}[e] } type Encoding struct { @@ -73,6 +74,7 @@ var ( inii = ini.Codec{} lines = line.Codec{} sv = csv.Codec{} + pb = proto.Codec{} ) var SupportedFileTypes = []Encoding{ {JSON, json.Unmarshal, jsn.Marshal}, @@ -88,6 +90,7 @@ var SupportedFileTypes = []Encoding{ {HTML, htm.Unmarshal, xmll.Marshal}, {LINE, lines.Unmarshal, jsn.Marshal}, {TXT, lines.Unmarshal, jsn.Marshal}, + {PROTO, pb.Unmarshal, jsn.Marshal}, } func Unmarshal(input []byte, inputFileType EncodingType, data interface{}) error { diff --git a/codec/proto/proto.go b/codec/proto/proto.go new file mode 100644 index 0000000..fb57a14 --- /dev/null +++ b/codec/proto/proto.go @@ -0,0 +1,157 @@ +package codec + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/goccy/go-json" +) + +type ProtoFile struct { + PackageName string + Messages map[string]Message + Enums map[string]Enum +} + +type Message struct { + Name string + Fields map[string]Field +} + +type Field struct { + Name string + Type string + Number int +} + +type Enum struct { + Name string + Values map[string]int +} + +type Codec struct{} + +func (c *Codec) Unmarshal(input []byte, v interface{}) error { + protoContent := string(input) + + protoContent = removeComments(protoContent) + + protoFile := &ProtoFile{Messages: make(map[string]Message), Enums: make(map[string]Enum)} + + messagePattern := `message\s+([A-Za-z0-9_]+)\s*\{([^}]*)\}` + fieldPattern := `([A-Za-z0-9_]+)\s+([A-Za-z0-9_]+)\s*=\s*(\d+);` + enumPattern := `enum\s+([A-Za-z0-9_]+)\s*\{([^}]*)\}` + enumValuePattern := `([A-Za-z0-9_]+)\s*=\s*(-?\d+);` + + re := regexp.MustCompile(messagePattern) + fieldRe := regexp.MustCompile(fieldPattern) + enumRe := regexp.MustCompile(enumPattern) + enumValueRe := regexp.MustCompile(enumValuePattern) + + packagePattern := `package\s+([A-Za-z0-9_]+);` + packageRe := regexp.MustCompile(packagePattern) + packageMatch := packageRe.FindStringSubmatch(protoContent) + if len(packageMatch) > 0 { + protoFile.PackageName = packageMatch[1] + } + + matches := re.FindAllStringSubmatch(protoContent, -1) + for _, match := range matches { + messageName := match[1] + messageContent := match[2] + + fields := make(map[string]Field) + fieldMatches := fieldRe.FindAllStringSubmatch(messageContent, -1) + for _, fieldMatch := range fieldMatches { + fieldType := fieldMatch[1] + fieldName := fieldMatch[2] + fieldNumber, err := strconv.Atoi(fieldMatch[3]) + if err != nil { + return err + } + fields[fieldName] = Field{ + Name: fieldName, + Type: fieldType, + Number: fieldNumber, + } + } + + protoFile.Messages[messageName] = Message{ + Name: messageName, + Fields: fields, + } + } + + enumMatches := enumRe.FindAllStringSubmatch(protoContent, -1) + for _, match := range enumMatches { + enumName := match[1] + enumContent := match[2] + + enumValues := make(map[string]int) + enumValueMatches := enumValueRe.FindAllStringSubmatch(enumContent, -1) + for _, enumValueMatch := range enumValueMatches { + enumValueName := enumValueMatch[1] + enumValueNumber := enumValueMatch[2] + number, err := strconv.Atoi(enumValueNumber) + if err != nil { + return nil + } + enumValues[enumValueName] = number + } + + protoFile.Enums[enumName] = Enum{ + Name: enumName, + Values: enumValues, + } + } + jsonMap, err := ConvertProtoToJSON(protoFile) + if err != nil { + return fmt.Errorf("error converting to JSON: %v", err) + } + jsonData, err := json.Marshal(jsonMap) + if err != nil { + return fmt.Errorf("error marshaling JSON: %v", err) + } + return json.Unmarshal(jsonData, v) +} + +func removeComments(input string) string { + reSingleLine := regexp.MustCompile(`//.*`) + input = reSingleLine.ReplaceAllString(input, "") + reMultiLine := regexp.MustCompile(`/\*.*?\*/`) + input = reMultiLine.ReplaceAllString(input, "") + return strings.TrimSpace(input) +} + +func ConvertProtoToJSON(protoFile *ProtoFile) (map[string]interface{}, error) { + jsonMap := make(map[string]interface{}) + packageMap := make(map[string]interface{}) + packageMap["message"] = make(map[string]interface{}) + packageMap["enums"] = make(map[string]interface{}) + + for messageName, message := range protoFile.Messages { + fieldsList := []interface{}{} + for name, field := range message.Fields { + values := make(map[string]interface{}) + values["name"] = name + values["type"] = field.Type + values["number"] = field.Number + fieldsList = append(fieldsList, values) + } + packageMap["message"].(map[string]interface{})[messageName] = fieldsList + } + + for enumName, enum := range protoFile.Enums { + valuesMap := make(map[string]interface{}) + for enumValueName, enumValueNumber := range enum.Values { + valuesMap[enumValueName] = enumValueNumber + } + packageMap["enums"].(map[string]interface{})[enumName] = valuesMap + } + + jsonMap[protoFile.PackageName] = packageMap + + return jsonMap, nil +} diff --git a/go.mod b/go.mod index 30d71fa..db88471 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/tmccombs/hcl2json v0.6.4 github.com/zclconf/go-cty v1.15.0 golang.org/x/net v0.30.0 + google.golang.org/protobuf v1.35.1 gopkg.in/ini.v1 v1.67.0 ) @@ -45,6 +46,7 @@ require ( github.com/muesli/termenv v0.15.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.26.0 // indirect diff --git a/go.sum b/go.sum index 410ccdb..507f0fe 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,9 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tmccombs/hcl2json v0.6.4 h1:/FWnzS9JCuyZ4MNwrG4vMrFrzRgsWEOVi+1AyYUVLGw= github.com/tmccombs/hcl2json v0.6.4/go.mod h1:+ppKlIW3H5nsAsZddXPy2iMyvld3SHxyjswOZhavRDk= github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ= @@ -116,6 +117,8 @@ golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/tests/example.proto b/tests/example.proto new file mode 100644 index 0000000..375d06e --- /dev/null +++ b/tests/example.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; +package company; + +enum Status { + ACTIVE = 0; + INACTIVE = 1; + RETIRED = 2; +} + +message Address { + string street = 1; + string city = 2; +} + +message Employee { + string first_name = 1; + string last_name = 2; + int32 employee_id = 3; + Status status = 4; + string email = 5; + optional string phone_number = 6; + reserved 7, 8; + string department_name = 9; + bool is_manager = 10; +} + +message Department { + string name = 1; + repeated Employee employees = 2; +} + +message Project { + string name = 1; + string description = 2; + repeated Employee team_members = 3; +} + +message Company { + string name = 1; + repeated Department departments = 2; + reserved 3 to 5; +} +