gruntwork-io#1947 Additional formats for sops_decrypt_file (gruntwork…

…-io#1956)

* More sops formatting

* Added test for plain files formatting

* Added test for env, ini

* Documentation update

* Updated test file

* Updated documentation

* Updated function description

config/config_helpers.go

@@ -2,13 +2,14 @@ package config
 
 import (
 	"fmt"
-	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
 	"unicode/utf8"
 
+	"go.mozilla.org/sops/v3/cmd/sops/formats"
+
 	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/hcl/v2"
 	tflang "github.com/hashicorp/terraform/lang"
@@ -615,17 +616,10 @@ func sopsDecryptFile(params []string, trackInclude *TrackInclude, terragruntOpti
 	if numParams != 1 {
 		return "", errors.WithStackTrace(WrongNumberOfParams{Func: "sops_decrypt_file", Expected: "1", Actual: numParams})
 	}
-
-	var format string
-	switch ext := path.Ext(sourceFile); ext {
-	case ".json":
-		format = "json"
-	case ".yaml", ".yml":
-		format = "yaml"
-	default:
-		return "", errors.WithStackTrace(InvalidSopsFormat{SourceFilePath: sourceFile})
+	format, err := getSopsFileFormat(sourceFile)
+	if err != nil {
+		return "", errors.WithStackTrace(err)
 	}
-
 	canonicalSourceFile, err := util.CanonicalPath(sourceFile, terragruntOptions.WorkingDir)
 	if err != nil {
 		return "", errors.WithStackTrace(err)
@@ -649,6 +643,25 @@ func sopsDecryptFile(params []string, trackInclude *TrackInclude, terragruntOpti
 	return "", errors.WithStackTrace(InvalidSopsFormat{SourceFilePath: sourceFile})
 }
 
+// Mapping of SOPS format to string
+var sopsFormatToString = map[formats.Format]string{
+	formats.Binary: "binary",
+	formats.Dotenv: "dotenv",
+	formats.Ini:    "ini",
+	formats.Json:   "json",
+	formats.Yaml:   "yaml",
+}
+
+// getSopsFileFormat - Return file format for SOPS library
+func getSopsFileFormat(sourceFile string) (string, error) {
+	fileFormat := formats.FormatForPath(sourceFile)
+	format, found := sopsFormatToString[fileFormat]
+	if !found {
+		return "", InvalidSopsFormat{SourceFilePath: sourceFile}
+	}
+	return format, nil
+}
+
 // Return the location of the Terraform files provided via --terragrunt-source
 func getTerragruntSourceCliFlag(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) {
 	return terragruntOptions.Source, nil

docs/_docs/04_reference/built-in-functions.md

@@ -700,15 +700,13 @@ inputs = {
 
 ## sops\_decrypt\_file
 
-`sops_decrypt_file(file_path)` decrypts a yaml or json file encrypted with `sops`.
+`sops_decrypt_file(file_path)` decrypts a yaml, json, ini, env or "raw text" file encrypted with `sops`.
 
 [sops](https://github.com/mozilla/sops) is an editor of encrypted files that supports YAML, JSON, ENV, INI and
 BINARY formats and encrypts with AWS KMS, GCP KMS, Azure Key Vault, Hashicorp Vault and PGP.
 
 This allows static secrets to be stored encrypted within your Terragrunt repository.
 
-Only YAML and JSON formats are supported by `sops_decrypt_file`
-
 For example, suppose you have some static secrets required to bootstrap your
 infrastructure in `secrets.yaml`, you can decrypt and merge them into the inputs
 by using `sops_decrypt_file`:

test/fixture-sops/main.tf

@@ -38,6 +38,18 @@ variable "yaml_hello" {
   type = string
 }
 
+variable "text_value" {
+  type = string
+}
+
+variable "env_value" {
+  type = string
+}
+
+variable "ini_value" {
+  type = string
+}
+
 output "json_string_array" {
   value = var.json_string_array
 }
@@ -77,3 +89,15 @@ output "yaml_number" {
 output "yaml_hello" {
   value = var.yaml_hello
 }
+
+output "text_value" {
+  value = var.text_value
+}
+
+output "env_value" {
+  value = var.env_value
+}
+
+output "ini_value" {
+  value = var.ini_value
+}

test/fixture-sops/secrets.env

@@ -0,0 +1,9 @@
+DB_USER=ENC[AES256_GCM,data:4yoBr0w=,iv:ePKzjYwS4yhJKGLKV2+KmtlXiFgtvzuP5+ZfgTvxtEA=,tag:IZyZ7UybZt+nHiFgFYjPNQ==,type:str]
+DB_PASSWORD=ENC[AES256_GCM,data:fQjK+ByY,iv:xhY1TLraqFZfYSGhW8nAO4jnkLnsUNFSJD9Y3+f48NQ=,tag:n39JC2ndmGuvb3F5Ily6aw==,type:str]
+sops_unencrypted_suffix=_unencrypted
+sops_pgp__list_0__map_enc=-----BEGIN PGP MESSAGE-----\n\nhQEMA0sXzMgpEabgAQf/SvsLSgWPoYfaeZTRspheA93oZvA4WWXxklP320JOBLlN\nIC5PH55OyxDde6l+HnpQpgNqp3QlPS15dtPs+U9NoObRfhNl9Bxd2rtdouiHK7LT\nWFFp6FJx+CBuVvMDMt8eEYPT1cNJA5A1gMnjDjt3ByJmV1xDVvHruU/EsL9bT1Br\nizl+4OszzADZ3Ih1vz4FkC7gwT0wmprk3b2IbXqP7wrgpk+BOkCVjzdkwbJIqNAW\nMGA6AHCrL2lSUm1UvhFjgDtlOnWZwtFHiiI8kqM90gtbzQG08nvN81UXTWkzseSJ\nv1AvKLLDpTikD7klugD+4GzSvQEkcZlVy0zxwIppSNJeAXXeFN0cT1dus1s6rSKI\nMBlI1Wc+slUx/zErenHeOxeF4SpPqwCvixoe94kf4kPzluRkS0tuHTpPbNbgpl9o\nQ5Up3V2L+ifq3xiJVGAhxeefJRHADpmvhVnpegUcWg==\n=ilH5\n-----END PGP MESSAGE-----\n
+sops_pgp__list_0__map_fp=3EF98802EEDCAF0C688B81F419546E0C123C664E
+sops_lastmodified=2021-12-17T18:43:48Z
+sops_mac=ENC[AES256_GCM,data:lRGaGq2usY4shlcsX5dR3g06CtNipQvQv8vXUSAV5yzvGSQHl1n/DmKXVwJZgIg6a0xQhr3L9R/1NL106md6kKxUSjeXJPdWjEteHX2qE24NGKBWgeR28JRjSuGytf7O4K10Q8SZm/JYhokAScgjsg3gzRAb8/LGzWzXpfV5ioE=,iv:JFBjodBw2thMVqoh0ItbRvTS5gx4YInt6udczff+CBQ=,tag:5/iJzP+sJPTf4+ZMNV7pgQ==,type:str]
+sops_pgp__list_0__map_created_at=2021-12-17T18:43:48Z
+sops_version=3.7.1

test/fixture-sops/secrets.ini

@@ -0,0 +1,13 @@
+[terragrunt]
+user     = ENC[AES256_GCM,data:jnACR2o=,iv:Qh5xex81KAqgJE+e1R0Xk95fNK6iQpBEejiuG8PcGiA=,tag:pB0/iBG0WxvpZ+1F50cK+A==,type:str]
+password = ENC[AES256_GCM,data:AMuq3/fS,iv:x+y4FAB/dgDwjdxBA6NXtLKbp/qvAlXtHboLTnoirNI=,tag:wsaMJth3eTJb+pNJrNr+hg==,type:str]
+
+[sops]
+pgp__list_0__map_created_at = 2021-12-17T18:43:08Z
+version                     = 3.7.1
+pgp__list_0__map_enc        = -----BEGIN PGP MESSAGE-----\n\nhQEMA0sXzMgpEabgAQf+MWG+tWNvydG/jVWwXQfg/ch2WdarMXKO0b/RZ/NkJT0n\nv4ozaGeATooEDuZXZXm0qJs1NQLYnCBp5PakkVfabHbSR2MByE7AwclgjUV6g4H7\nKHWsw0L6hKfZFeIU9+FTEVIjpNkFIE9EEdkD762ZF4B6n31HxgcK+z7r/sW+2PII\nZf+HKZPuPik0Og0SmmtiCr/nO2p5wAdjEdBBHAaAfD00UCbIR34UG9iAErhJxJ5U\ngSpOHSli8VISaB1LWJldV51F7uT1qhdukcCeBl0W1lWM4wmBdVLxSV6oKuK2gUPF\nuXm9hrlUsDRN6TmViJb9TvgWOe1Quva+pwnwJnJw5dJeAX7tTavnEIxfgJW/cQ8u\n3gP/kRtuGCOhUFukl75ZkmqqbheXGiKmhwRGtzmFMqM3xQBtXw56TxZ9JCu4p09Y\nua8othAS0G2L84/r5pJN5krfdvgsoKAP7XQwrpGYPQ==\n=7lpK\n-----END PGP MESSAGE-----\n
+unencrypted_suffix          = _unencrypted
+lastmodified                = 2021-12-17T18:43:08Z
+mac                         = ENC[AES256_GCM,data:SXl3JEg+gMPfN0c2pzRm/XmBTmI70ZQJ2gr11S8lVZehRpfdAc2UNwz6B5uca0hGRKViO7KhLWPAo4yxNXNi+QznJkeHpxa1fYnT1p+UhYjvOnfg8MQeL4+qEz+Oq5Gx4LJJm62oC8+bAWWZPlFM4L5844R8bSZ6Pd1FUfFF6b4=,iv:x4pQi8UFuSwKLuv4EvXOG4DJx4IxoUEfd/+32s6nsHg=,tag:P6SHIsFol+5n2YylQ5cQYg==,type:str]
+pgp__list_0__map_fp         = 3EF98802EEDCAF0C688B81F419546E0C123C664E
+

test/fixture-sops/secrets.txt

@@ -0,0 +1,21 @@
+{
+	"data": "ENC[AES256_GCM,data:w2jDRJR9BeIMSKE4+qnKWhfM,iv:08ACLYrUGtWriOV/ua4X6NZt57VmiTmAcnxB5V+8AUc=,tag:cVdkIO4EXAmyV3y7n/zbiA==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2021-12-17T18:38:13Z",
+		"mac": "ENC[AES256_GCM,data:8lPZmY8YgA0DqPRxLC9hVoRUXmbzaXgUBv3MHTm4iK44/6URIgJBUnPFPUbwIN7xbIgXd+QPQEMvfsmifqXorynGEwt2WtMKCPANg+2Ctf2KMmj7fGpe3HIlRhQiixip7/xzrIMbSdIRMS098D42JTvOIFNbWVQhByfN64AnDJY=,iv:wtouC/mWjhFwiJKDS6+5LqnQMcAeejElXLaL3H15jbY=,tag:6Bmemr2BMgShaMO3v4uiXw==,type:str]",
+		"pgp": [
+			{
+				"created_at": "2021-12-17T18:38:12Z",
+				"enc": "-----BEGIN PGP MESSAGE-----\n\nhQEMA0sXzMgpEabgAQf+KHsPp4Pp8YNtG7ChRpZO2qB/bFncWtAF9evO+RjAEahb\nM+hzxkB5KDUSMYs0aeWeOrOqYPrjPPJxCspZtQhy8/qrC064kA7gq2PWhYAqGcKP\ntnPI8D0SYDZBgoyHRqFuuD5TZio8swE89SxphftL0W3KkHay7WKQHj/cFqNoISNl\nn0XeCgbacIwo5WxWz1qNFvaeo0rFFFhIhbfaegx/SWwUi1y6WK7sB0QobMRwXHj+\nORiUWVvx/fCIMCaerPN/SjIA/DgzbZ3DWaixYXpW85Ipz7myu/zUQcWnWcGXnMRQ\nERMYc6GyyLHwjZN1XuvXdPXvAt6vvaH4w5U9kW2l19JeAZXkcM14ivDoGwY1oLcX\n4d2/MAS7vM7SgmcPBGmpNsJJgkWTgoc8qeFtu9u3e4e9pR4+dcJCbGQLQ5RiyM2Z\nsyHjL6em/j4JLdtbM16orP6Q3oEPelphG7sxbDXBeA==\n=6u1S\n-----END PGP MESSAGE-----\n",
+				"fp": "3EF98802EEDCAF0C688B81F419546E0C123C664E"
+			}
+		],
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.1"
+	}
+}

test/fixture-sops/terragrunt.hcl

@@ -1,6 +1,9 @@
 locals {
   json = jsondecode(sops_decrypt_file("${get_terragrunt_dir()}/secrets.json"))
   yaml = yamldecode(sops_decrypt_file("${get_terragrunt_dir()}/secrets.yaml"))
+  text = sops_decrypt_file("${get_terragrunt_dir()}/secrets.txt")
+  env  = sops_decrypt_file("${get_terragrunt_dir()}/secrets.env")
+  ini  = sops_decrypt_file("${get_terragrunt_dir()}/secrets.ini")
 }
 
 inputs = {
@@ -14,4 +17,7 @@ inputs = {
   yaml_string       = local.yaml["example_key"]
   yaml_number       = local.yaml["example_number"]
   yaml_hello        = local.yaml["hello"]
+  text_value        = local.text
+  env_value         = local.env
+  ini_value         = local.ini
 }

test/integration_test.go

@@ -3936,6 +3936,9 @@ func TestSopsDecryptedCorrectly(t *testing.T) {
 	assert.Equal(t, outputs["yaml_number"].Value, 1234.5679)
 	assert.Equal(t, outputs["yaml_string"].Value, "example_value")
 	assert.Equal(t, outputs["yaml_hello"].Value, "Welcome to SOPS! Edit this file as you please!")
+	assert.Equal(t, outputs["text_value"].Value, "Raw Secret Example")
+	assert.Contains(t, outputs["env_value"].Value, "DB_PASSWORD=tomato")
+	assert.Contains(t, outputs["ini_value"].Value, "password = potato")
 }
 
 func TestTerragruntRunAllCommandPrompt(t *testing.T) {