feat: 0.18.0 minor release (#579)

* v0.18.0 * feat: extend cors (#578) * fix: cors not applied to all routes (#577) * feat: improve schema read (#576) * feat: collect ip fingerprint and better tstamp (#575) * feat: log schema metadata (#574) * feat: more lambda config options (#573) * fet: more lambda config options * chore: remove unused var * review --------- Co-authored-by: Léonard Henriquez <[email protected]>
silverton-io · Dec 12, 2023 · 2f1908d · 2f1908d
1 parent c251aed
commit 2f1908d
Show file tree

Hide file tree

Showing 16 changed files with 211 additions and 51 deletions.
diff --git a/.VERSION b/.VERSION
@@ -1 +1 @@
-v0.17.0
+v0.18.0
diff --git a/.gitignore b/.gitignore
@@ -19,6 +19,7 @@ examples/quickstart/minio/*
 .terraform.lock.hcl
 .terraform.tfstate.lock*
 terraform.*
+backend.tf
 build/buz
 *.build
 target/*
diff --git a/cmd/buz/app.go b/cmd/buz/app.go
@@ -108,8 +108,6 @@ func (a *App) initializeManifold() {
 func (a *App) initializeRouter() {
 	log.Info().Msg("🟢 initializing router")
 	a.engine = gin.New()
-	a.publicRouterGroup = a.engine.Group("")
-	a.switchableRouterGroup = a.engine.Group("")
 	if err := a.engine.SetTrustedProxies(nil); err != nil {
 		panic(err)
 	}
@@ -147,6 +145,11 @@ func (a *App) initializeMiddleware() {
 	}
 }
 
+func (a *App) initializeRouterGroups() {
+	a.publicRouterGroup = a.engine.Group("")
+	a.switchableRouterGroup = a.engine.Group("")
+}
+
 // 🐝 and healthcheck route are always public
 func (a *App) initializePublicRoutes() {
 	log.Info().Msg("🟢 initializing buz route")
@@ -201,6 +204,7 @@ func (a *App) Initialize() {
 	a.initializeRouter()
 	a.initializeManifold()
 	a.initializeMiddleware()
+	a.initializeRouterGroups()
 	a.initializePublicRoutes()
 	a.initializeOpsRoutes()
 	a.initializeSchemaCacheRoutes()

diff --git a/deploy/terraform/aws/lambda/locals.tf b/deploy/terraform/aws/lambda/locals.tf
@@ -1,19 +1,19 @@
 locals {
-  domain_parts        = split(".", var.buz_domain)
-  cookie_domain       = join(".", slice(local.domain_parts, 1, length(local.domain_parts))) # Assumes Buz is running on a subdomain and the cookie should be on root
-  buz_debug_var       = "DEBUG"
-  buz_config_var      = "BUZ_CONFIG_PATH"
-  buz_config_path     = "/etc/buz/config.yml"
-  system_env_base     = "${var.system}-${var.env}-"
-  artifact_repository = "${local.system_env_base}img"
-  image               = "buz:${var.buz_version}"
-  buz_source_image    = "ghcr.io/silverton-io/${local.image}"
-  service_name        = "${local.system_env_base}collector"
-  config              = "${local.system_env_base}config"
-  schema_bucket       = "${local.system_env_base}${var.schema_bucket_name}"
-  events_bucket       = "${local.system_env_base}${var.events_bucket_name}"
-  default_output        = "buz_events"
-  deadletter_output      = "buz_invalid_events"
+  domain_parts               = split(".", var.buz_domain)
+  cookie_domain              = join(".", slice(local.domain_parts, 1, length(local.domain_parts))) # Assumes Buz is running on a subdomain and the cookie should be on root
+  buz_debug_var              = "DEBUG"
+  buz_config_var             = "BUZ_CONFIG_PATH"
+  buz_config_path            = "/etc/buz/config.yml"
+  system_env_base            = "${var.system}-${var.env}-"
+  artifact_repository        = "${local.system_env_base}img"
+  image                      = "buz:${var.buz_version}"
+  buz_source_image           = "${var.buz_image_repo}/${local.image}"
+  service_name               = "${local.system_env_base}collector"
+  config                     = "${local.system_env_base}config"
+  schema_bucket              = "${local.system_env_base}${var.schema_bucket_name}"
+  events_bucket              = "${local.system_env_base}${var.events_bucket_name}"
+  default_output             = "buz_events"
+  deadletter_output          = "buz_invalid_events"
   metadata_extraction_params = "{isValid:.isValid,vendor:.vendor,namespace:.namespace,version:.version}"
-  s3_dynamic_prefix   = "isValid=!{partitionKeyFromQuery:isValid}/vendor=!{partitionKeyFromQuery:vendor}/namespace=!{partitionKeyFromQuery:namespace}/version=!{partitionKeyFromQuery:version}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
+  s3_dynamic_prefix          = "isValid=!{partitionKeyFromQuery:isValid}/vendor=!{partitionKeyFromQuery:vendor}/namespace=!{partitionKeyFromQuery:namespace}/version=!{partitionKeyFromQuery:version}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
 }
diff --git a/deploy/terraform/aws/lambda/main.tf b/deploy/terraform/aws/lambda/main.tf
@@ -139,7 +139,7 @@ data "aws_ecr_image" "buz_image" {
 
 resource "null_resource" "configure_docker" {
   triggers = {
-    build_number = var.buz_version
+    always_run = timestamp()
   }
   provisioner "local-exec" {
     command = "aws ecr get-login-password --region ${var.aws_region} | docker login --username AWS --password-stdin ${data.aws_caller_identity.current.account_id}.dkr.ecr.${var.aws_region}.amazonaws.com"
@@ -228,6 +228,14 @@ resource "aws_cloudwatch_log_group" "buz" {
   }
 }
 
+data "aws_cloudfront_origin_request_policy" "buz" {
+  name = "Managed-AllViewerExceptHostHeader"
+}
+
+data "aws_cloudfront_cache_policy" "buz" {
+  name = "Managed-CachingDisabled"
+}
+
 resource "aws_cloudfront_distribution" "buz" {
   enabled         = true
   is_ipv6_enabled = true
@@ -248,25 +256,20 @@ resource "aws_cloudfront_distribution" "buz" {
   }
 
   default_cache_behavior {
-    viewer_protocol_policy = "redirect-to-https"
-    min_ttl                = 0
-    default_ttl            = 3600
-    max_ttl                = 86400
-    target_origin_id       = replace(replace(aws_lambda_function_url.buz.function_url, "https://", ""), "/", "")
-    allowed_methods        = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
-    cached_methods         = ["HEAD", "GET"]
-    forwarded_values {
-      query_string = true
-      cookies {
-        forward = "all"
-      }
-    }
+    viewer_protocol_policy   = "redirect-to-https"
+    min_ttl                  = 0
+    default_ttl              = 3600
+    max_ttl                  = 86400
+    target_origin_id         = replace(replace(aws_lambda_function_url.buz.function_url, "https://", ""), "/", "")
+    allowed_methods          = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
+    cached_methods           = ["HEAD", "GET"]
+    origin_request_policy_id = data.aws_cloudfront_origin_request_policy.buz.id
+    cache_policy_id          = data.aws_cloudfront_cache_policy.buz.id
   }
 
   restrictions {
     geo_restriction {
-      restriction_type = "whitelist"
-      locations        = ["US", "CA", "GB", "DE"]
+      restriction_type = "none"
     }
   }
 

diff --git a/deploy/terraform/aws/lambda/provider.tf b/deploy/terraform/aws/lambda/provider.tf
@@ -10,5 +10,6 @@ terraform {
 }
 
 provider "aws" {
-  region = var.aws_region
-}
+  region  = var.aws_region
+  profile = var.aws_profile
+}
diff --git a/deploy/terraform/aws/lambda/variables.tf b/deploy/terraform/aws/lambda/variables.tf
@@ -4,6 +4,12 @@ variable "aws_region" {
   default     = "us-east-1"
 }
 
+variable "aws_profile" {
+  description = "AWS Profile"
+  type        = string
+  default     = "default"
+}
+
 variable "system" {
   description = "The name of the Buz implementation. \n\nExample: buz"
   type        = string
@@ -26,10 +32,16 @@ variable "buz_domain" {
   type        = string
 }
 
+variable "buz_image_repo" {
+  description = "The Buz image repository"
+  type        = string
+  default     = "ghcr.io/silverton-io"
+}
+
 variable "buz_version" {
   description = "The version of Buz to run."
   type        = string
-  default     = "v0.17.0"
+  default     = "v0.18.0"
 }
 
 variable "buz_lambda_memory_limit" {

diff --git a/deploy/terraform/gcp/cloud_run/variables.tf b/deploy/terraform/gcp/cloud_run/variables.tf
@@ -28,7 +28,7 @@ variable "buz_domain" {
 variable "buz_version" {
   description = "The version of Buz to run."
   type        = string
-  default     = "v0.17.0"
+  default     = "v0.18.0"
 }
 
 variable "buz_service_timeout_seconds" {

diff --git a/examples/quickstart/docker-compose.yml b/examples/quickstart/docker-compose.yml
@@ -20,7 +20,7 @@ x-dependency:
 services:
   buz:
     container_name: buz
-    image: ghcr.io/silverton-io/buz:v0.17.0
+    image: ghcr.io/silverton-io/buz:v0.18.0
     volumes:
       - type: bind
         source: ./buz/quickstart.conf.yml

diff --git a/pkg/annotator/annotator.go b/pkg/annotator/annotator.go
@@ -25,6 +25,7 @@ func getSchemaMetadata(schema []byte) schemaMetadata {
 	namespace := schemaContents.Get("self.namespace").String()
 	version := schemaContents.Get("self.version").String()
 	disableValidation := schemaContents.Get("disableValidation").Bool()
+	log.Debug().Msgf("🟡 vendor: %s, namespace: %s, version: %s, disableValidation: %t", vendor, namespace, version, disableValidation)
 	return schemaMetadata{
 		Vendor:            vendor,
 		Namespace:         namespace,

diff --git a/pkg/annotator/annotator_test.go b/pkg/annotator/annotator_test.go
@@ -0,0 +1,33 @@
+package annotator
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetSchemaMetadata(t *testing.T) {
+	testData := []struct {
+		name     string
+		input    []byte
+		expected schemaMetadata
+	}{
+		{
+			name:  "Valid JSON",
+			input: []byte(`{"self":{"vendor":"testVendor","namespace":"testNamespace","version":"testVersion"},"disableValidation":true}`),
+			expected: schemaMetadata{
+				Vendor:            "testVendor",
+				Namespace:         "testNamespace",
+				Version:           "testVersion",
+				DisableValidation: true,
+			},
+		},
+	}
+
+	for _, tc := range testData {
+		t.Run(tc.name, func(t *testing.T) {
+			result := getSchemaMetadata(tc.input)
+			assert.Equal(t, tc.expected, result)
+		})
+	}
+}
diff --git a/pkg/annotator/test_annotator.go b/pkg/annotator/test_annotator.go
diff --git a/pkg/middleware/cors.go b/pkg/middleware/cors.go
@@ -15,8 +15,23 @@ import (
 
 func CORS(conf config.Cors) gin.HandlerFunc {
 	return func(c *gin.Context) {
-		c.Header("Access-Control-Allow-Origin", strings.Join(conf.AllowOrigin, ", "))
-		c.Header("Access-Control-Allow-Credentials", strconv.FormatBool(conf.AllowCredentials))
+		origin := c.Request.Header.Get("Origin")
+
+		for _, domain := range conf.AllowOrigin {
+			if domain == "*" {
+				c.Header("Access-Control-Allow-Origin", origin)
+				break
+			}
+			if strings.HasSuffix(origin, domain) {
+				c.Header("Access-Control-Allow-Origin", origin)
+				break
+			}
+		}
+
+		if strconv.FormatBool(conf.AllowCredentials) == "true" {
+			c.Header("Access-Control-Allow-Credentials", strconv.FormatBool(conf.AllowCredentials))
+		}
+
 		c.Header("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, Set-Cookie, Cookie")
 		c.Header("Access-Control-Allow-Methods", strings.Join(conf.AllowMethods, ", "))
 		c.Header("Access-Control-Max-Age", strconv.Itoa(conf.MaxAge))
@@ -25,6 +40,7 @@ func CORS(conf config.Cors) gin.HandlerFunc {
 			c.AbortWithStatus(http.StatusNoContent)
 			return
 		}
+
 		c.Next()
 	}
 }
diff --git a/pkg/middleware/cors_test.go b/pkg/middleware/cors_test.go
@@ -18,8 +18,8 @@ func TestCors(t *testing.T) {
 	u := "/test"
 	conf := config.Cors{
 		Enabled:          true,
-		AllowOrigin:      []string{"*"},
-		AllowCredentials: false,
+		AllowOrigin:      []string{"http://allowed-origin.com"},
+		AllowCredentials: true,
 		AllowMethods:     []string{"GET", "OPTIONS"},
 		MaxAge:           86400,
 	}
@@ -29,14 +29,28 @@ func TestCors(t *testing.T) {
 	ts := httptest.NewServer(r)
 	defer ts.Close()
 
-	t.Run("preflight", func(t *testing.T) {
+	t.Run("preflight success", func(t *testing.T) {
 		var client = &http.Client{}
 		req, _ := http.NewRequest(http.MethodOptions, ts.URL+u, nil)
+		req.Header.Set("Origin", "http://allowed-origin.com")
 		resp, _ := client.Do(req)
 
-		assert.Equal(t, []string{"false"}, resp.Header["Access-Control-Allow-Credentials"])
+		assert.Equal(t, []string{"true"}, resp.Header["Access-Control-Allow-Credentials"])
 		assert.Equal(t, []string{"GET, OPTIONS"}, resp.Header["Access-Control-Allow-Methods"])
-		assert.Equal(t, []string{"*"}, resp.Header["Access-Control-Allow-Origin"])
+		assert.Equal(t, []string{"http://allowed-origin.com"}, resp.Header["Access-Control-Allow-Origin"])
+		assert.Equal(t, []string{"86400"}, resp.Header["Access-Control-Max-Age"])
+		assert.Equal(t, http.StatusNoContent, resp.StatusCode)
+	})
+
+	t.Run("preflight fail", func(t *testing.T) {
+		var client = &http.Client{}
+		req, _ := http.NewRequest(http.MethodOptions, ts.URL+u, nil)
+		req.Header.Set("Origin", "http://not-allowed-origin.com")
+		resp, _ := client.Do(req)
+
+		assert.Equal(t, []string{"true"}, resp.Header["Access-Control-Allow-Credentials"])
+		assert.Equal(t, []string{"GET, OPTIONS"}, resp.Header["Access-Control-Allow-Methods"])
+		assert.Equal(t, []string([]string(nil)), resp.Header["Access-Control-Allow-Origin"])
 		assert.Equal(t, []string{"86400"}, resp.Header["Access-Control-Max-Age"])
 		assert.Equal(t, http.StatusNoContent, resp.StatusCode)
 	})