diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..9aae56c9 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @lingrino diff --git a/Makefile b/Makefile index e434cc49..b3ebe2c2 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ test: docker run -d --name=test-vault -p 8300:8300 \ -e VAULT_DEV_ROOT_TOKEN_ID=hunter2 \ -e VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8300 \ + -e VAULT_LOG=debug \ vault:latest && sleep 8 export VAULT_ADDR=http://localhost:8300 && \ export VAULT_TOKEN=hunter2 && \ - go test ./... + go test -v ./... diff --git a/vault/client.go b/vault/client.go index be851bb2..3ef12f46 100644 --- a/vault/client.go +++ b/vault/client.go @@ -1,9 +1,8 @@ package vault import ( - "fmt" - vapi "github.com/hashicorp/vault/api" + "github.com/pkg/errors" ) // Client is a wrapper around a real Vault API client. @@ -23,49 +22,89 @@ func (c *Client) simpleInit() error { client, err := vapi.NewClient(vapi.DefaultConfig()) if err != nil { - return fmt.Errorf("[FATAL]: simpleInit: Failed to init the vault client: %s", err) + return errors.Wrap(err, "Failed to init the vault client") } c.client = client return err } -// seed uses a client to write dummy data used for teting to vault +// seed uses a client to write dummy data used for testing to vault // strings generated here: https://www.random.org/strings func (c *Client) seed() error { var err error + var mountPath string + + c.client.Sys().EnableAuditWithOptions("audit_stdout", &vapi.EnableAuditOptions{ + Type: "file", + Options: map[string]string{ + "file_path": "stdout", + "log_raw": "true", + }, + }) seeds := map[string]map[string]string{ - "secret/data/test/foo": { + "test/foo": { "value": "bar", }, - "secret/data/test/value": { + "test/value": { "fizz": "buzz", "foo": "bar", }, - "secret/data/test/fizz": { + "test/fizz": { "fizz": "buzz", "foo": "bar", }, - "secret/data/test/HToOeKKD": { + "test/HToOeKKD": { "3zqxVbJY": "TvOjGxvC", }, - "secret/data/test/inner/WKNC3muM": { + "test/inner/WKNC3muM": { "IY1C148K": "JxBfEt91", "iwVzPqbY": "0NH9GlR1", }, - "secret/data/test/inner/A2xlzTfE": { + "test/inner/A2xlzTfE": { "Eg5ljS7t": "BHRMKjj1", "quqr32S5": "pcidzSMW", }, - "secret/data/test/inner/again/inner/UCrt6sZT": { + "test/inner/again/inner/UCrt6sZT": { "Eg5ljS7t": "6F1B5nBg", "quqr32S5": "81iY4HAN", "r6R0JUzX": "rs1mCRB5", }, } + // Seed v1 mount + mountPath = "secretv1/" + err = c.client.Sys().Mount(mountPath, &vapi.MountInput{ + Type: "kv", + Options: map[string]string{ + "version": "1", + }, + }) + for path, secret := range seeds { + writePath := mountPath + path + data := make(map[string]interface{}) + + for k, v := range secret { + data[k] = v + } + + _, err = c.client.Logical().Write(writePath, data) + if err != nil { + return errors.Wrapf(err, "Failed to seed vault at path %s", writePath) + } + } + + // Seed v2 mount + mountPath = "secretv2/" + c.client.Sys().Mount(mountPath, &vapi.MountInput{ + Type: "kv", + Options: map[string]string{ + "version": "2", + }, + }) for path, secret := range seeds { + writePath := mountPath + "data/" + path data := make(map[string]interface{}) for k, v := range secret { @@ -78,9 +117,9 @@ func (c *Client) seed() error { "data": data, } - _, err = c.client.Logical().Write(path, data) + _, err = c.client.Logical().Write(writePath, data) if err != nil { - return fmt.Errorf("[FATAL]: seed: Failed to seed vault at path %s: %s", path, err) + return errors.Wrapf(err, "Failed to seed vault at path %s", writePath) } } return err diff --git a/vault/kv_list.go b/vault/kv_list.go new file mode 100644 index 00000000..ec209e43 --- /dev/null +++ b/vault/kv_list.go @@ -0,0 +1,225 @@ +package vault + +import ( + "fmt" + "sort" + "strings" + "sync" + + "github.com/pkg/errors" +) + +// KVListInput is the required input for KVList +type KVListInput struct { + Path string + Recurse bool + TrimPathPrefix bool + MountPath string + MountVersion string +} + +// NewKVListInput takes a path and returns a kvListInput struct with +// default values to produce similar to what is returned by Vault CLI +func NewKVListInput(p string) *KVListInput { + return &KVListInput{ + Path: p, + Recurse: false, + TrimPathPrefix: true, + MountPath: "", + MountVersion: "", + } +} + +// listWorkerResult holds the key and any errors from a job +type listWorkerResult struct { + key string + err error +} + +// listWorker listens on the inputs channel for new paths to list and then does the +// work of listing those paths and returning the result. If recurse is true any listed keys +// that are folders will be added back to the inputs channel for further processing +func (c *Client) listWorker(inputs chan *KVListInput, results chan<- *listWorkerResult, inputsWG *sync.WaitGroup, resultsWG *sync.WaitGroup) { + // listen on the inputs channel until it is closed + for { + i, more := <-inputs + if more { + // This lets us only get mount info only once, by passing it in future inputs + if i.MountPath == "" { + mountPath, version, err := c.PathMountInfo(i.Path) + if err != nil { + fmt.Println("hi") + resultsWG.Add(1) + results <- &listWorkerResult{ + key: "", + err: errors.Wrapf(err, "Failed to describe mount for path %s", i.Path), + } + inputsWG.Done() + continue + } + i.MountPath = mountPath + i.MountVersion = version + } + + // For v2 mounts lists happen on mount/metadata/path instead of mount/path + var listPath string + if i.MountVersion == "2" { + listPath = c.PathJoin(i.MountPath, "metadata", strings.TrimPrefix(i.Path, i.MountPath)) + } else { + listPath = i.Path + } + + // Do the actual list + secret, err := c.client.Logical().List(listPath) + if err != nil { + resultsWG.Add(1) + results <- &listWorkerResult{ + key: "", + err: errors.Wrapf(err, "Failed to list path at %s", listPath), + } + inputsWG.Done() + continue + } + + // extract list data from the returned secret + if secret == nil || secret.Data == nil { + resultsWG.Add(1) + results <- &listWorkerResult{ + key: "", + err: fmt.Errorf("Secret at %s was nil", listPath), + } + inputsWG.Done() + continue + + } + keys, ok := secret.Data["keys"] + if !ok || keys == nil { + resultsWG.Add(1) + results <- &listWorkerResult{ + key: "", + err: fmt.Errorf("No Data[\"keys\"] in secret at %s", listPath), + } + inputsWG.Done() + continue + } + list, ok := keys.([]interface{}) + if !ok { + resultsWG.Add(1) + results <- &listWorkerResult{ + key: "", + err: fmt.Errorf("Failed to convert keys to interface at %s", listPath), + } + inputsWG.Done() + continue + } + + // For each key, either add it to results or add it back to inputs if recurse and folder + for _, v := range list { + key, ok := v.(string) + if !ok { + resultsWG.Add(1) + results <- &listWorkerResult{ + key: "", + err: fmt.Errorf("Failed to assert %s as a string at %s", key, listPath), + } + inputsWG.Done() + continue + } + // If we're recursing and the key is a folder, add it back as an input to be listed + if c.PathIsFolder(key) && i.Recurse { + inputsWG.Add(1) + inputs <- &KVListInput{ + Path: c.PathJoin(i.Path, key), + Recurse: i.Recurse, + TrimPathPrefix: i.TrimPathPrefix, + MountPath: i.MountPath, + MountVersion: i.MountVersion, + } + } else if c.PathIsFolder(key) { + resultsWG.Add(1) + results <- &listWorkerResult{ + key: c.PathJoin(i.Path, key) + "/", + err: nil, + } + + } else { + resultsWG.Add(1) + results <- &listWorkerResult{ + key: c.PathJoin(i.Path, key), + err: nil, + } + } + } + inputsWG.Done() + } else { + return + } + } +} + +// KVList takes a path and returns a slice of all values at that path +// If Recurse, also list all nested paths/folders +// If TrimPathPrefix, do not prefix keys with leading path +func (c *Client) KVList(i *KVListInput) ([]string, error) { + var err error + var output []string + + if i.Path == "" { + return nil, errors.Wrap(err, "Path is not specified") + } + + inputs := make(chan *KVListInput, 5) + results := make(chan *listWorkerResult, 5) + + var inputsWG sync.WaitGroup + var resultsWG sync.WaitGroup + + // Add our first input + inputsWG.Add(1) + inputs <- i + + // Listen on results channel and add keys to an output list + go func() { + for { + o, more := <-results + if more { + if o.err != nil { + err = errors.Wrapf(err, "Failed to list path %s", i.Path) + } else { + output = append(output, o.key) + } + + resultsWG.Done() + } else { + return + } + } + }() + + // Spawn 5 workers if recursing, otherise just one + // TODO - read worker count from configuration + if i.Recurse { + for w := 1; w <= 5; w++ { + go c.listWorker(inputs, results, &inputsWG, &resultsWG) + } + } else { + go c.listWorker(inputs, results, &inputsWG, &resultsWG) + } + + // Wait until all lists are complete + inputsWG.Wait() + resultsWG.Wait() + close(inputs) + close(results) + + // Remove the prefix if it is not wanted + if i.TrimPathPrefix == true { + for idx, pth := range output { + output[idx] = strings.TrimPrefix(strings.TrimPrefix(pth, i.Path), "/") + } + } + + sort.Strings(output) + + return output, err +} diff --git a/vault/kv_list_test.go b/vault/kv_list_test.go new file mode 100644 index 00000000..d6a21091 --- /dev/null +++ b/vault/kv_list_test.go @@ -0,0 +1,78 @@ +package vault + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// kvListTest holds the test data for KVList +// The 'Path' here should be specified without +// the leading mounts secretv1/2. Same with output +type kvListTest struct { + input *KVListInput + output []string +} + +func TestKVList(t *testing.T) { + c := NewClient() + c.simpleInit() + mounts := []string{"secretv1", "secretv2"} + + nameToTest := map[string]*kvListTest{ + "1": &kvListTest{ + input: &KVListInput{ + Path: "test", + Recurse: false, + TrimPathPrefix: true, + MountPath: "", + MountVersion: "", + }, + output: []string{"HToOeKKD", "fizz", "foo", "inner/", "value"}, + }, + "2": &kvListTest{ + input: &KVListInput{ + Path: "test", + Recurse: true, + TrimPathPrefix: true, + }, + output: []string{"HToOeKKD", "fizz", "foo", "inner/A2xlzTfE", "inner/WKNC3muM", "inner/again/inner/UCrt6sZT", "value"}, + }, + "3": &kvListTest{ + input: &KVListInput{ + Path: "test/fizz", + Recurse: false, + TrimPathPrefix: false, + }, + output: nil, + }, + "4": &kvListTest{ + input: &KVListInput{ + Path: "test/inner", + Recurse: true, + TrimPathPrefix: false, + }, + output: []string{"test/inner/A2xlzTfE", "test/inner/WKNC3muM", "test/inner/again/inner/UCrt6sZT"}, + }, + } + + for _, mount := range mounts { + for _, v := range nameToTest { + var origPath string + + origPath = v.input.Path + v.input.Path = c.PathJoin(mount, v.input.Path) + v.input.MountPath = "" + v.input.MountVersion = "" + + l, _ := c.KVList(v.input) + for i, p := range l { + l[i] = strings.TrimPrefix(p, mount+"/") + } + + assert.Equal(t, v.output, l) + v.input.Path = origPath + } + } +} diff --git a/vault/path.go b/vault/path.go index aa314f39..8b8be8aa 100644 --- a/vault/path.go +++ b/vault/path.go @@ -3,6 +3,8 @@ package vault import ( pth "path" str "strings" + + "github.com/pkg/errors" ) // PathIsFolder returns true if the string ends in a '/' @@ -15,6 +17,35 @@ func (c *Client) PathIsFolder(s string) bool { return false } +// PathMountInfo checks if a path is on a V2 mount +// Returns the "mount" path for the given path +// An empty return string here implies an error +func (c *Client) PathMountInfo(p string) (string, string, error) { + var err error + var mountPath string + var version string + + if p != "" { + mounts, err := c.client.Sys().ListMounts() + if err != nil { + errors.Wrap(err, "Failed to list mounts") + return mountPath, version, err + } + + for mount, data := range mounts { + if str.HasPrefix(p, mount) { + mountPath = mount + version, ok := data.Options["version"] + if !ok { + version = "unknown" + } + return mountPath, version, err + } + } + } + return mountPath, version, err +} + // PathJoin takes n strings and combines them into a clean vault path func (c *Client) PathJoin(paths ...string) string { return str.TrimPrefix(pth.Join(paths...), "/") diff --git a/vault/path_test.go b/vault/path_test.go index 1f8177cb..e25c990e 100644 --- a/vault/path_test.go +++ b/vault/path_test.go @@ -23,6 +23,26 @@ func TestPathIsFolder(t *testing.T) { } } +func TestPathMountInfo(t *testing.T) { + inputToOutput := map[string][]string{ + "": {"", ""}, + "dflskdjf": {"", ""}, + "secretv2": {"", ""}, + "secretv1/": {"secretv1/", "1"}, + "secretv1/test": {"secretv1/", "1"}, + "secretv2/": {"secretv2/", "2"}, + "secretv2/test": {"secretv2/", "2"}, + "secretv2/test/again": {"secretv2/", "2"}, + } + c := NewClient() + c.simpleInit() + for i, o := range inputToOutput { + b, s, _ := c.PathMountInfo(i) + assert.Equal(t, o[0], b) + assert.Equal(t, o[1], s) + } +} + func TestPathJoin(t *testing.T) { outputToInput := map[string][]string{ "": {"/"},