Skip to content

Commit

Permalink
add codec for protobuf .proto (#25)
Browse files Browse the repository at this point in the history
This change just adds proto definition parsing for conversions to format or querying with gojq to a format of the following:

$package_name:
   enum: 
     example:
       ACTIVE: 1
       INACTIVE: 0
   message:
     message_example:
        - name: field1
        - type: "int"
        - number: 1
  • Loading branch information
JFryy authored Nov 10, 2024
1 parent c3bc2dd commit 9849050
Show file tree
Hide file tree
Showing 6 changed files with 212 additions and 9 deletions.
10 changes: 4 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ qq large-file.json -o yaml > /dev/null 2>&1 2.72s user 0.16s system 190% cpu 1.
```

## Supported Formats
Note: these unsupported formats are on a roadmap for inclusion.
| Format | Input | Output |
|-------------|----------------|----------------|
| JSON | ✅ Supported | ✅ Supported |
Expand All @@ -106,19 +105,18 @@ Note: these unsupported formats are on a roadmap for inclusion.
| TF | ✅ Supported | ✅ Supported |
| GRON | ✅ Supported | ✅ Supported |
| CSV | ✅ Supported | ✅ Supported |
| Protobuf | ❌ Not Supported | ❌ Not Supported |
| Proto (.proto) | Supported | ❌ Not Supported |
| HTML | ✅ Supported | ✅ Supported |
| TXT (newline)| ✅ Supported | ❌ Not Supported |


## Caveats
1. `qq` is not a full `jq`/`*q` replacement and comes with idiosyncrasies from the underlying `gojq` library.
2. the encoders and decoders are not perfect and may not be able to handle all edge cases.
3. `qq` is under active development and more codecs are intended to be supported along with improvements to `interactive mode`.
1. `qq` is not a full `jq` replacement, some flags may or may not be supported.
3. `qq` is under active development, more codecs in the future may be supported along with improvements to `interactive mode`.


## Contributions
All contributions are welcome to `qq`, especially for upkeep/optimization/addition of new encodings. For ideas on contributions [please refer to the todo docs](https://github.com/JFryy/qq/blob/main/docs/TODO.md) or make an issue/PR for a suggestion if there's something that's wanted or fixes.
All contributions are welcome to `qq`, especially for upkeep/optimization/addition of new encodings.

## Thanks and Acknowledgements / Related Projects
This tool would not be possible without the following projects, this project is arguably more of a composition of these projects than a truly original work, with glue code, some dedicated encoders/decoders, and the interactive mode being original work.
Expand Down
7 changes: 5 additions & 2 deletions codec/codec.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/JFryy/qq/codec/ini"
qqjson "github.com/JFryy/qq/codec/json"
"github.com/JFryy/qq/codec/line"
proto "github.com/JFryy/qq/codec/proto"
"github.com/JFryy/qq/codec/xml"
)

Expand All @@ -41,11 +42,11 @@ const (
HTML
LINE
TXT
MD
PROTO
)

func (e EncodingType) String() string {
return [...]string{"json", "yaml", "yml", "toml", "hcl", "tf", "csv", "xml", "ini", "gron", "html", "line", "txt", "md"}[e]
return [...]string{"json", "yaml", "yml", "toml", "hcl", "tf", "csv", "xml", "ini", "gron", "html", "line", "txt", "proto"}[e]
}

type Encoding struct {
Expand Down Expand Up @@ -73,6 +74,7 @@ var (
inii = ini.Codec{}
lines = line.Codec{}
sv = csv.Codec{}
pb = proto.Codec{}
)
var SupportedFileTypes = []Encoding{
{JSON, json.Unmarshal, jsn.Marshal},
Expand All @@ -88,6 +90,7 @@ var SupportedFileTypes = []Encoding{
{HTML, htm.Unmarshal, xmll.Marshal},
{LINE, lines.Unmarshal, jsn.Marshal},
{TXT, lines.Unmarshal, jsn.Marshal},
{PROTO, pb.Unmarshal, jsn.Marshal},
}

func Unmarshal(input []byte, inputFileType EncodingType, data interface{}) error {
Expand Down
157 changes: 157 additions & 0 deletions codec/proto/proto.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
package codec

import (
"fmt"
"regexp"
"strconv"
"strings"

"github.com/goccy/go-json"
)

type ProtoFile struct {
PackageName string
Messages map[string]Message
Enums map[string]Enum
}

type Message struct {
Name string
Fields map[string]Field
}

type Field struct {
Name string
Type string
Number int
}

type Enum struct {
Name string
Values map[string]int
}

type Codec struct{}

func (c *Codec) Unmarshal(input []byte, v interface{}) error {
protoContent := string(input)

protoContent = removeComments(protoContent)

protoFile := &ProtoFile{Messages: make(map[string]Message), Enums: make(map[string]Enum)}

messagePattern := `message\s+([A-Za-z0-9_]+)\s*\{([^}]*)\}`
fieldPattern := `([A-Za-z0-9_]+)\s+([A-Za-z0-9_]+)\s*=\s*(\d+);`
enumPattern := `enum\s+([A-Za-z0-9_]+)\s*\{([^}]*)\}`
enumValuePattern := `([A-Za-z0-9_]+)\s*=\s*(-?\d+);`

re := regexp.MustCompile(messagePattern)
fieldRe := regexp.MustCompile(fieldPattern)
enumRe := regexp.MustCompile(enumPattern)
enumValueRe := regexp.MustCompile(enumValuePattern)

packagePattern := `package\s+([A-Za-z0-9_]+);`
packageRe := regexp.MustCompile(packagePattern)
packageMatch := packageRe.FindStringSubmatch(protoContent)
if len(packageMatch) > 0 {
protoFile.PackageName = packageMatch[1]
}

matches := re.FindAllStringSubmatch(protoContent, -1)
for _, match := range matches {
messageName := match[1]
messageContent := match[2]

fields := make(map[string]Field)
fieldMatches := fieldRe.FindAllStringSubmatch(messageContent, -1)
for _, fieldMatch := range fieldMatches {
fieldType := fieldMatch[1]
fieldName := fieldMatch[2]
fieldNumber, err := strconv.Atoi(fieldMatch[3])
if err != nil {
return err
}
fields[fieldName] = Field{
Name: fieldName,
Type: fieldType,
Number: fieldNumber,
}
}

protoFile.Messages[messageName] = Message{
Name: messageName,
Fields: fields,
}
}

enumMatches := enumRe.FindAllStringSubmatch(protoContent, -1)
for _, match := range enumMatches {
enumName := match[1]
enumContent := match[2]

enumValues := make(map[string]int)
enumValueMatches := enumValueRe.FindAllStringSubmatch(enumContent, -1)
for _, enumValueMatch := range enumValueMatches {
enumValueName := enumValueMatch[1]
enumValueNumber := enumValueMatch[2]
number, err := strconv.Atoi(enumValueNumber)
if err != nil {
return nil
}
enumValues[enumValueName] = number
}

protoFile.Enums[enumName] = Enum{
Name: enumName,
Values: enumValues,
}
}
jsonMap, err := ConvertProtoToJSON(protoFile)
if err != nil {
return fmt.Errorf("error converting to JSON: %v", err)
}
jsonData, err := json.Marshal(jsonMap)
if err != nil {
return fmt.Errorf("error marshaling JSON: %v", err)
}
return json.Unmarshal(jsonData, v)
}

func removeComments(input string) string {
reSingleLine := regexp.MustCompile(`//.*`)
input = reSingleLine.ReplaceAllString(input, "")
reMultiLine := regexp.MustCompile(`/\*.*?\*/`)
input = reMultiLine.ReplaceAllString(input, "")
return strings.TrimSpace(input)
}

func ConvertProtoToJSON(protoFile *ProtoFile) (map[string]interface{}, error) {
jsonMap := make(map[string]interface{})
packageMap := make(map[string]interface{})
packageMap["message"] = make(map[string]interface{})
packageMap["enum"] = make(map[string]interface{})

for messageName, message := range protoFile.Messages {
fieldsList := []interface{}{}
for name, field := range message.Fields {
values := make(map[string]interface{})
values["name"] = name
values["type"] = field.Type
values["number"] = field.Number
fieldsList = append(fieldsList, values)
}
packageMap["message"].(map[string]interface{})[messageName] = fieldsList
}

for enumName, enum := range protoFile.Enums {
valuesMap := make(map[string]interface{})
for enumValueName, enumValueNumber := range enum.Values {
valuesMap[enumValueName] = enumValueNumber
}
packageMap["enum"].(map[string]interface{})[enumName] = valuesMap
}

jsonMap[protoFile.PackageName] = packageMap

return jsonMap, nil
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ require (
github.com/muesli/termenv v0.15.2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.9.0 // indirect
golang.org/x/mod v0.21.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.26.0 // indirect
Expand Down
3 changes: 2 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,9 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tmccombs/hcl2json v0.6.4 h1:/FWnzS9JCuyZ4MNwrG4vMrFrzRgsWEOVi+1AyYUVLGw=
github.com/tmccombs/hcl2json v0.6.4/go.mod h1:+ppKlIW3H5nsAsZddXPy2iMyvld3SHxyjswOZhavRDk=
github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ=
Expand Down
43 changes: 43 additions & 0 deletions tests/example.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
syntax = "proto3";
package company;

enum Status {
ACTIVE = 0;
INACTIVE = 1;
RETIRED = 2;
}

message Address {
string street = 1;
string city = 2;
}

message Employee {
string first_name = 1;
string last_name = 2;
int32 employee_id = 3;
Status status = 4;
string email = 5;
optional string phone_number = 6;
reserved 7, 8;
string department_name = 9;
bool is_manager = 10;
}

message Department {
string name = 1;
repeated Employee employees = 2;
}

message Project {
string name = 1;
string description = 2;
repeated Employee team_members = 3;
}

message Company {
string name = 1;
repeated Department departments = 2;
reserved 3 to 5;
}

0 comments on commit 9849050

Please sign in to comment.