From 8bf15a21285d11afad6d2e1fd62daaa3b392e4c4 Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Mon, 13 Jan 2025 18:21:51 +0530 Subject: [PATCH 1/8] feat: isolate server transformer communication --- .../event_sampler/badger_event_sampler.go | 3 +- .../destination_transformer.go | 11 +++++++ .../trackingplan_validation.go | 11 +++++++ .../user_transformer/user_transformer.go | 11 +++++++ processor/processor.go | 26 ++++++++++------- processor/transformer/manager.go | 29 +++++++++++++++++++ 6 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 processor/internal/destination_transformer/destination_transformer.go create mode 100644 processor/internal/trackingplan_validation/trackingplan_validation.go create mode 100644 processor/internal/user_transformer/user_transformer.go create mode 100644 processor/transformer/manager.go diff --git a/enterprise/reporting/event_sampler/badger_event_sampler.go b/enterprise/reporting/event_sampler/badger_event_sampler.go index 33e8c95747..fb8f33cb6f 100644 --- a/enterprise/reporting/event_sampler/badger_event_sampler.go +++ b/enterprise/reporting/event_sampler/badger_event_sampler.go @@ -2,6 +2,7 @@ package event_sampler import ( "context" + "errors" "fmt" "os" "sync" @@ -117,7 +118,7 @@ func (es *BadgerEventSampler) Get(key string) (bool, error) { return nil }) - if err == badger.ErrKeyNotFound { + if errors.Is(err, badger.ErrKeyNotFound) { return false, nil } else if err != nil { return false, err diff --git a/processor/internal/destination_transformer/destination_transformer.go b/processor/internal/destination_transformer/destination_transformer.go new file mode 100644 index 0000000000..6d7305c948 --- /dev/null +++ b/processor/internal/destination_transformer/destination_transformer.go @@ -0,0 +1,11 @@ +package destination_transformer + +import "fmt" + +type DestTransformer struct{} + +func (d *DestTransformer) SendRequest(data interface{}) (interface{}, error) { + fmt.Println("Sending request to Service A") + // Add service-specific logic + return "Response from Service A", nil +} diff --git a/processor/internal/trackingplan_validation/trackingplan_validation.go b/processor/internal/trackingplan_validation/trackingplan_validation.go new file mode 100644 index 0000000000..c6c48a19a1 --- /dev/null +++ b/processor/internal/trackingplan_validation/trackingplan_validation.go @@ -0,0 +1,11 @@ +package trackingplan_validation + +import "fmt" + +type TPValidator struct{} + +func (t *TPValidator) SendRequest(data interface{}) (interface{}, error) { + fmt.Println("Sending request to Service A") + // Add service-specific logic + return "Response from Service A", nil +} diff --git a/processor/internal/user_transformer/user_transformer.go b/processor/internal/user_transformer/user_transformer.go new file mode 100644 index 0000000000..9cd90dd902 --- /dev/null +++ b/processor/internal/user_transformer/user_transformer.go @@ -0,0 +1,11 @@ +package user_transformer + +import "fmt" + +type UserTransformer struct{} + +func (u *UserTransformer) SendRequest(data interface{}) (interface{}, error) { + fmt.Println("Sending request to Service A") + // Add service-specific logic + return "Response from Service A", nil +} diff --git a/processor/processor.go b/processor/processor.go index cbf40de16b..12b1bbe1bb 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -13,16 +13,10 @@ import ( "time" "github.com/google/uuid" - - "github.com/rudderlabs/rudder-server/enterprise/trackedusers" - - "golang.org/x/sync/errgroup" - - "github.com/rudderlabs/rudder-go-kit/stringify" - jsoniter "github.com/json-iterator/go" "github.com/samber/lo" "github.com/tidwall/gjson" + "golang.org/x/sync/errgroup" "github.com/rudderlabs/rudder-go-kit/bytesize" "github.com/rudderlabs/rudder-go-kit/config" @@ -30,14 +24,19 @@ import ( "github.com/rudderlabs/rudder-go-kit/ro" "github.com/rudderlabs/rudder-go-kit/stats" "github.com/rudderlabs/rudder-go-kit/stats/metric" + "github.com/rudderlabs/rudder-go-kit/stringify" kitsync "github.com/rudderlabs/rudder-go-kit/sync" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/enterprise/trackedusers" "github.com/rudderlabs/rudder-server/internal/enricher" "github.com/rudderlabs/rudder-server/jobsdb" "github.com/rudderlabs/rudder-server/processor/delayed" "github.com/rudderlabs/rudder-server/processor/eventfilter" "github.com/rudderlabs/rudder-server/processor/integrations" + "github.com/rudderlabs/rudder-server/processor/internal/destination_transformer" + "github.com/rudderlabs/rudder-server/processor/internal/trackingplan_validation" + "github.com/rudderlabs/rudder-server/processor/internal/user_transformer" "github.com/rudderlabs/rudder-server/processor/isolation" "github.com/rudderlabs/rudder-server/processor/stash" "github.com/rudderlabs/rudder-server/processor/transformer" @@ -86,10 +85,11 @@ type trackedUsersReporter interface { // Handle is a handle to the processor module type Handle struct { - conf *config.Config - tracer stats.Tracer - backendConfig backendconfig.BackendConfig - transformer transformer.Transformer + conf *config.Config + tracer stats.Tracer + backendConfig backendconfig.BackendConfig + transformer transformer.Transformer + transformerManager *transformer.CommunicationManager gatewayDB jobsdb.JobsDB routerDB jobsdb.JobsDB @@ -396,6 +396,10 @@ func (proc *Handle) Setup( proc.setupReloadableVars() proc.logger = logger.NewLogger().Child("processor") proc.backendConfig = backendConfig + proc.transformerManager = transformer.NewCommunicationManager() + proc.transformerManager.RegisterService("user_transformer", &user_transformer.UserTransformer{}) + proc.transformerManager.RegisterService("destination_transformer", &destination_transformer.DestTransformer{}) + proc.transformerManager.RegisterService("trackingplan_validation", &trackingplan_validation.TPValidator{}) proc.gatewayDB = gatewayDB proc.routerDB = routerDB diff --git a/processor/transformer/manager.go b/processor/transformer/manager.go new file mode 100644 index 0000000000..8ef742f42a --- /dev/null +++ b/processor/transformer/manager.go @@ -0,0 +1,29 @@ +package transformer + +import "errors" + +type ServiceClient interface { + SendRequest(data interface{}) (response interface{}, err error) +} + +type CommunicationManager struct { + clients map[string]ServiceClient +} + +func NewCommunicationManager() *CommunicationManager { + return &CommunicationManager{ + clients: make(map[string]ServiceClient), + } +} + +func (m *CommunicationManager) RegisterService(name string, client ServiceClient) { + m.clients[name] = client +} + +func (m *CommunicationManager) GetServiceClient(name string) (ServiceClient, error) { + client, exists := m.clients[name] + if !exists { + return nil, errors.New("service client not registered") + } + return client, nil +} From 7ed43e39d9a085c8fe42f5bcafb4d19edabebd4a Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Tue, 14 Jan 2025 14:43:00 +0530 Subject: [PATCH 2/8] chore: introduce http client --- .../destination_transformer.go | 10 +- processor/internal/http_client/http_client.go | 92 +++---------------- .../trackingplan_validation.go | 11 ++- .../user_transformer/user_transformer.go | 11 ++- 4 files changed, 38 insertions(+), 86 deletions(-) diff --git a/processor/internal/destination_transformer/destination_transformer.go b/processor/internal/destination_transformer/destination_transformer.go index 61ace38be4..7aa276c691 100644 --- a/processor/internal/destination_transformer/destination_transformer.go +++ b/processor/internal/destination_transformer/destination_transformer.go @@ -7,6 +7,8 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" + + "github.com/rudderlabs/rudder-server/processor/internal/http_client" warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) @@ -14,9 +16,10 @@ type DestTransformer struct { config struct { destTransformationURL string } - conf *config.Config - log logger.Logger - stat stats.Stats + conf *config.Config + log logger.Logger + stat stats.Stats + client http_client.HTTPDoer } func (d *DestTransformer) SendRequest(data interface{}) (interface{}, error) { @@ -30,6 +33,7 @@ func NewDestTransformer(conf *config.Config, log logger.Logger, stat stats.Stats handle.conf = conf handle.log = log handle.stat = stat + handle.client = http_client.NewHTTPClient(conf) handle.config.destTransformationURL = handle.conf.GetString("Warehouse.destTransformationURL", "http://localhost:9090") return handle diff --git a/processor/internal/http_client/http_client.go b/processor/internal/http_client/http_client.go index 6ef7fa40df..d925f5d13a 100644 --- a/processor/internal/http_client/http_client.go +++ b/processor/internal/http_client/http_client.go @@ -1,54 +1,21 @@ package http_client import ( - "fmt" + "net" + "net/http" + "time" + "github.com/bufbuild/httplb" "github.com/bufbuild/httplb/resolver" + "github.com/rudderlabs/rudder-go-kit/config" - "github.com/rudderlabs/rudder-go-kit/logger" - "github.com/rudderlabs/rudder-go-kit/stats" "github.com/rudderlabs/rudder-server/utils/sysUtils" - "net" - "net/http" - "time" ) type HTTPDoer interface { Do(req *http.Request) (*http.Response, error) } -type HTTPHandle struct { - sentStat stats.Measurement - receivedStat stats.Measurement - cpDownGauge stats.Measurement - - conf *config.Config - logger logger.Logger - stat stats.Stats - - httpClient HTTPDoer - - guardConcurrency chan struct{} - - config struct { - maxConcurrency int - maxHTTPConnections int - maxHTTPIdleConnections int - maxIdleConnDuration time.Duration - disableKeepAlives bool - - timeoutDuration time.Duration - - maxRetry config.ValueLoader[int] - failOnUserTransformTimeout config.ValueLoader[bool] - failOnError config.ValueLoader[bool] - maxRetryBackoffInterval config.ValueLoader[time.Duration] - - destTransformationURL string - userTransformationURL string - } -} - type HTTPLBTransport struct { *http.Transport } @@ -57,56 +24,29 @@ func (t *HTTPLBTransport) NewRoundTripper(scheme, target string, config httplb.T return httplb.RoundTripperResult{RoundTripper: t.Transport, Close: t.CloseIdleConnections} } -func NewHTTPClient(conf *config.Config, log logger.Logger, stat stats.Stats) (HTTPHandle, error) { - trans := HTTPHandle{} - - trans.conf = conf - trans.logger = log.Child("transformer") - trans.stat = stat - - trans.sentStat = stat.NewStat("processor.transformer_sent", stats.CountType) - trans.receivedStat = stat.NewStat("processor.transformer_received", stats.CountType) - trans.cpDownGauge = stat.NewStat("processor.control_plane_down", stats.GaugeType) - - trans.config.maxConcurrency = conf.GetInt("Processor.maxConcurrency", 200) - trans.config.maxHTTPConnections = conf.GetInt("Transformer.Client.maxHTTPConnections", 100) - trans.config.maxHTTPIdleConnections = conf.GetInt("Transformer.Client.maxHTTPIdleConnections", 10) - trans.config.maxIdleConnDuration = conf.GetDuration("Transformer.Client.maxIdleConnDuration", 30, time.Second) - trans.config.disableKeepAlives = conf.GetBool("Transformer.Client.disableKeepAlives", true) - trans.config.timeoutDuration = conf.GetDuration("HttpClient.procTransformer.timeout", 600, time.Second) - trans.config.destTransformationURL = conf.GetString("DEST_TRANSFORM_URL", "http://localhost:9090") - trans.config.userTransformationURL = conf.GetString("USER_TRANSFORM_URL", trans.config.destTransformationURL) - - trans.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") - trans.config.failOnUserTransformTimeout = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnUserTransformTimeout") - trans.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") - - trans.config.maxRetryBackoffInterval = conf.GetReloadableDurationVar(30, time.Second, "Processor.Transformer.maxRetryBackoffInterval") - - trans.guardConcurrency = make(chan struct{}, trans.config.maxConcurrency) - +func NewHTTPClient(conf *config.Config) HTTPDoer { clientType := conf.GetString("Transformer.Client.type", "stdlib") transport := &http.Transport{ - DisableKeepAlives: trans.config.disableKeepAlives, - MaxConnsPerHost: trans.config.maxHTTPConnections, - MaxIdleConnsPerHost: trans.config.maxHTTPIdleConnections, - IdleConnTimeout: trans.config.maxIdleConnDuration, + DisableKeepAlives: conf.GetBool("Transformer.Client.disableKeepAlives", true), + MaxConnsPerHost: conf.GetInt("Transformer.Client.maxHTTPConnections", 100), + MaxIdleConnsPerHost: conf.GetInt("Transformer.Client.maxHTTPIdleConnections", 10), + IdleConnTimeout: conf.GetDuration("Transformer.Client.maxIdleConnDuration", 30, time.Second), } client := &http.Client{ Transport: transport, - Timeout: trans.config.timeoutDuration, + Timeout: conf.GetDuration("HttpClient.procTransformer.timeout", 600, time.Second), } switch clientType { case "stdlib": - trans.httpClient = client + return client case "recycled": - trans.httpClient = sysUtils.NewRecycledHTTPClient(func() *http.Client { + return sysUtils.NewRecycledHTTPClient(func() *http.Client { return client }, config.GetDuration("Transformer.Client.ttl", 120, time.Second)) case "httplb": - trans.httpClient = httplb.NewClient( + return httplb.NewClient( httplb.WithTransport("http", &HTTPLBTransport{ Transport: transport, }), @@ -119,8 +59,6 @@ func NewHTTPClient(conf *config.Config, log logger.Logger, stat stats.Stats) (HT ), ) default: - return HTTPHandle{}, fmt.Errorf("unknown transformer client type: %s", clientType) + return client } - - return trans, nil } diff --git a/processor/internal/trackingplan_validation/trackingplan_validation.go b/processor/internal/trackingplan_validation/trackingplan_validation.go index bfb9746031..56caba741c 100644 --- a/processor/internal/trackingplan_validation/trackingplan_validation.go +++ b/processor/internal/trackingplan_validation/trackingplan_validation.go @@ -2,18 +2,22 @@ package trackingplan_validation import ( "fmt" + "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" + + "github.com/rudderlabs/rudder-server/processor/internal/http_client" ) type TPValidator struct { config struct { destTransformationURL string } - conf *config.Config - log logger.Logger - stat stats.Stats + conf *config.Config + log logger.Logger + stat stats.Stats + client http_client.HTTPDoer } func (t *TPValidator) SendRequest(data interface{}) (interface{}, error) { @@ -27,6 +31,7 @@ func NewTPValidator(conf *config.Config, log logger.Logger, stat stats.Stats) *T handle.conf = conf handle.log = log handle.stat = stat + handle.client = http_client.NewHTTPClient(conf) handle.config.destTransformationURL = handle.conf.GetString("Warehouse.destTransformationURL", "http://localhost:9090") return handle } diff --git a/processor/internal/user_transformer/user_transformer.go b/processor/internal/user_transformer/user_transformer.go index 692bd1b69c..ef7ac3fadd 100644 --- a/processor/internal/user_transformer/user_transformer.go +++ b/processor/internal/user_transformer/user_transformer.go @@ -2,18 +2,22 @@ package user_transformer import ( "fmt" + "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" + + "github.com/rudderlabs/rudder-server/processor/internal/http_client" ) type UserTransformer struct { config struct { userTransformationURL string } - conf *config.Config - log logger.Logger - stat stats.Stats + conf *config.Config + log logger.Logger + stat stats.Stats + client http_client.HTTPDoer } func (u *UserTransformer) SendRequest(data interface{}) (interface{}, error) { @@ -27,6 +31,7 @@ func NewUserTransformer(conf *config.Config, log logger.Logger, stat stats.Stats handle.conf = conf handle.log = log handle.stat = stat + handle.client = http_client.NewHTTPClient(conf) handle.config.userTransformationURL = handle.conf.GetString("Warehouse.userTransformationURL", "http://localhost:9090") return handle } From abd835882b4a96a3bf63ca81c640ba76598b0cee Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Tue, 14 Jan 2025 17:26:33 +0530 Subject: [PATCH 3/8] chore: refactor types --- .../reporting_dropped_events_test.go | 5 +- .../reporting_error_index_test.go | 5 +- integration_test/tracing/tracing_test.go | 9 +- .../transformer_contract_test.go | 8 +- integration_test/warehouse/warehouse_test.go | 30 +- internal/enricher/enricher.go | 2 +- internal/enricher/geolocation.go | 2 +- internal/enricher/geolocation_test.go | 2 +- .../processor/transformer/mock_transformer.go | 14 +- processor/consent.go | 2 +- processor/consent_test.go | 6 +- processor/delayed/stats.go | 5 +- processor/delayed/stats_test.go | 9 +- processor/eventfilter/eventfilter.go | 19 +- processor/eventfilter/eventfilter_test.go | 20 +- processor/events_response.go | 10 +- processor/integrations/integrations.go | 4 +- .../destination_transformer.go | 21 +- .../trackingplan_validation.go | 20 +- .../user_transformer/user_transformer.go | 23 +- processor/processor.go | 285 ++++++++------- processor/processorBenchmark_test.go | 3 +- processor/processor_test.go | 345 +++++++++--------- processor/trackingplan.go | 33 +- processor/trackingplan_test.go | 50 +-- processor/transformer/manager.go | 9 +- processor/transformer/transformer.go | 126 ++----- processor/transformer/transformer_test.go | 141 ++++--- processor/types/types.go | 122 +++++++ .../transformationStatusUploader.go | 15 +- .../transformationStatusUploader_test.go | 19 +- testhelper/transformertest/builder.go | 7 +- testhelper/transformertest/handler_funcs.go | 36 +- utils/misc/misc.go | 10 - utils/types/types.go | 24 -- warehouse/integrations/testhelper/staging.go | 6 +- 36 files changed, 745 insertions(+), 702 deletions(-) create mode 100644 processor/types/types.go diff --git a/integration_test/reporting_dropped_events/reporting_dropped_events_test.go b/integration_test/reporting_dropped_events/reporting_dropped_events_test.go index 4554c2dfb8..72b23d47ee 100644 --- a/integration_test/reporting_dropped_events/reporting_dropped_events_test.go +++ b/integration_test/reporting_dropped_events/reporting_dropped_events_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -25,7 +27,6 @@ import ( kithelper "github.com/rudderlabs/rudder-go-kit/testhelper" "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/postgres" "github.com/rudderlabs/rudder-go-kit/testhelper/rand" - "github.com/rudderlabs/rudder-server/processor/transformer" "github.com/rudderlabs/rudder-server/runner" "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" "github.com/rudderlabs/rudder-server/testhelper/health" @@ -120,7 +121,7 @@ func TestReportingDroppedEvents(t *testing.T) { transformertest.ViolationErrorTransformerHandler( http.StatusBadRequest, "tracking plan validation failed", - []transformer.ValidationError{{Type: "Datatype-Mismatch", Message: "must be number"}}, + []types.ValidationError{{Type: "Datatype-Mismatch", Message: "must be number"}}, ), ). Build() diff --git a/integration_test/reporting_error_index/reporting_error_index_test.go b/integration_test/reporting_error_index/reporting_error_index_test.go index 46249289d6..6e3eda931b 100644 --- a/integration_test/reporting_error_index/reporting_error_index_test.go +++ b/integration_test/reporting_error_index/reporting_error_index_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/rudderlabs/rudder-server/processor/types" + _ "github.com/marcboeker/go-duckdb" "github.com/ory/dockertest/v3" "github.com/samber/lo" @@ -28,7 +30,6 @@ import ( "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/postgres" "github.com/rudderlabs/rudder-go-kit/testhelper/rand" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" "github.com/rudderlabs/rudder-server/runner" "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" "github.com/rudderlabs/rudder-server/testhelper/health" @@ -62,7 +63,7 @@ func TestReportingErrorIndex(t *testing.T) { transformertest.ViolationErrorTransformerHandler( http.StatusBadRequest, "tracking plan validation failed", - []transformer.ValidationError{{Type: "Datatype-Mismatch", Message: "must be number"}}, + []types.ValidationError{{Type: "Datatype-Mismatch", Message: "must be number"}}, ), ). Build() diff --git a/integration_test/tracing/tracing_test.go b/integration_test/tracing/tracing_test.go index ed580b488a..dbbfe78144 100644 --- a/integration_test/tracing/tracing_test.go +++ b/integration_test/tracing/tracing_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/rudderlabs/rudder-server/processor/types" + _ "github.com/marcboeker/go-duckdb" "github.com/ory/dockertest/v3" "github.com/samber/lo" @@ -33,7 +35,6 @@ import ( "github.com/rudderlabs/rudder-server/app" "github.com/rudderlabs/rudder-server/gateway/response" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" "github.com/rudderlabs/rudder-server/runner" "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" "github.com/rudderlabs/rudder-server/testhelper/health" @@ -359,15 +360,15 @@ func TestTracing(t *testing.T) { defer bcServer.Close() trServer := transformertest.NewBuilder(). - WithUserTransformHandler(func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { + WithUserTransformHandler(func(request []types.TransformerEvent) (response []types.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, types.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: http.StatusOK, }) - response = append(response, transformer.TransformerResponse{ + response = append(response, types.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: http.StatusOK, diff --git a/integration_test/transformer_contract/transformer_contract_test.go b/integration_test/transformer_contract/transformer_contract_test.go index b15474fabf..e9fb53ed71 100644 --- a/integration_test/transformer_contract/transformer_contract_test.go +++ b/integration_test/transformer_contract/transformer_contract_test.go @@ -28,7 +28,7 @@ import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/runner" "github.com/rudderlabs/rudder-server/testhelper/health" "github.com/rudderlabs/rudder-server/testhelper/transformertest" @@ -107,7 +107,7 @@ func TestTransformerContract(t *testing.T) { trServer := transformertest.NewBuilder(). WithUserTransformHandler( - func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { + func(request []types.TransformerEvent) (response []types.TransformerResponse) { for i := range request { req := request[i] @@ -123,7 +123,7 @@ func TestTransformerContract(t *testing.T) { require.Equal(t, req.Metadata.TransformationID, "transformation-1") require.Equal(t, req.Metadata.TransformationVersionID, "version-1") require.Equal(t, req.Metadata.EventType, "identify") - require.Equal(t, req.Credentials, []transformer.Credential{ + require.Equal(t, req.Credentials, []types.Credential{ { ID: "credential-1", Key: "key-1", @@ -131,7 +131,7 @@ func TestTransformerContract(t *testing.T) { IsSecret: false, }, }) - response = append(response, transformer.TransformerResponse{ + response = append(response, types.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: http.StatusOK, diff --git a/integration_test/warehouse/warehouse_test.go b/integration_test/warehouse/warehouse_test.go index 8043b76f4c..7eedbe7cc4 100644 --- a/integration_test/warehouse/warehouse_test.go +++ b/integration_test/warehouse/warehouse_test.go @@ -17,6 +17,8 @@ import ( "text/template" "time" + "github.com/rudderlabs/rudder-server/processor/types" + transformertest "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/transformer" "github.com/google/uuid" @@ -1589,14 +1591,14 @@ func TestDestinationTransformation(t *testing.T) { testcases := []struct { name string configOverride map[string]any - validateEvents func(t *testing.T, events []transformer.TransformerResponse) + validateEvents func(t *testing.T, events []types.TransformerResponse) }{ { name: "with allowUsersContextTraits=true", configOverride: map[string]any{ "allowUsersContextTraits": true, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1625,7 +1627,7 @@ func TestDestinationTransformation(t *testing.T) { configOverride: map[string]any{ "allowUsersContextTraits": false, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1652,7 +1654,7 @@ func TestDestinationTransformation(t *testing.T) { { name: "without allowUsersContextTraits", configOverride: map[string]any{}, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1719,7 +1721,7 @@ func TestDestinationTransformation(t *testing.T) { }) require.NoError(t, err) - var transformerEvents []transformer.TransformerEvent + var transformerEvents []types.TransformerEvent err = json.Unmarshal([]byte(b.String()), &transformerEvents) require.NoError(t, err) @@ -1735,14 +1737,14 @@ func TestDestinationTransformation(t *testing.T) { testcases := []struct { name string configOverride map[string]any - validateEvents func(t *testing.T, events []transformer.TransformerResponse) + validateEvents func(t *testing.T, events []types.TransformerResponse) }{ { name: "with underscoreDivideNumbers=true", configOverride: map[string]any{ "underscoreDivideNumbers": true, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var trackOutput output err := mapstructure.Decode(events[0].Output, &trackOutput) require.NoError(t, err) @@ -1767,7 +1769,7 @@ func TestDestinationTransformation(t *testing.T) { configOverride: map[string]any{ "underscoreDivideNumbers": false, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var trackOutput output err := mapstructure.Decode(events[0].Output, &trackOutput) require.NoError(t, err) @@ -1790,7 +1792,7 @@ func TestDestinationTransformation(t *testing.T) { { name: "without underscoreDivideNumbers", configOverride: map[string]any{}, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var trackOutput output err := mapstructure.Decode(events[0].Output, &trackOutput) require.NoError(t, err) @@ -1850,7 +1852,7 @@ func TestDestinationTransformation(t *testing.T) { }) require.NoError(t, err) - var transformerEvents []transformer.TransformerEvent + var transformerEvents []types.TransformerEvent err = json.Unmarshal([]byte(b.String()), &transformerEvents) require.NoError(t, err) @@ -1867,7 +1869,7 @@ func TestDestinationTransformation(t *testing.T) { name string destType string configOverride map[string]any - validateEvents func(t *testing.T, events []transformer.TransformerResponse) + validateEvents func(t *testing.T, events []types.TransformerResponse) }{ { name: "for non-datalake destinations should be present", @@ -1875,7 +1877,7 @@ func TestDestinationTransformation(t *testing.T) { configOverride: map[string]any{ "allowUsersContextTraits": true, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1905,7 +1907,7 @@ func TestDestinationTransformation(t *testing.T) { configOverride: map[string]any{ "allowUsersContextTraits": false, }, - validateEvents: func(t *testing.T, events []transformer.TransformerResponse) { + validateEvents: func(t *testing.T, events []types.TransformerResponse) { var identifyEvent output err := mapstructure.Decode(events[0].Output, &identifyEvent) require.NoError(t, err) @@ -1975,7 +1977,7 @@ func TestDestinationTransformation(t *testing.T) { }) require.NoError(t, err) - var transformerEvents []transformer.TransformerEvent + var transformerEvents []types.TransformerEvent err = json.Unmarshal([]byte(b.String()), &transformerEvents) require.NoError(t, err) diff --git a/internal/enricher/enricher.go b/internal/enricher/enricher.go index dd16eae785..fbe3177a93 100644 --- a/internal/enricher/enricher.go +++ b/internal/enricher/enricher.go @@ -2,7 +2,7 @@ package enricher import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/utils/types" + "github.com/rudderlabs/rudder-server/processor/types" ) // PipelineEnricher is a new paradigm under which the gateway events in diff --git a/internal/enricher/geolocation.go b/internal/enricher/geolocation.go index cece86354f..49954c8d4a 100644 --- a/internal/enricher/geolocation.go +++ b/internal/enricher/geolocation.go @@ -14,8 +14,8 @@ import ( "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/services/geolocation" - "github.com/rudderlabs/rudder-server/utils/types" ) const ( diff --git a/internal/enricher/geolocation_test.go b/internal/enricher/geolocation_test.go index 66b2fd46d5..af7762dec5 100644 --- a/internal/enricher/geolocation_test.go +++ b/internal/enricher/geolocation_test.go @@ -17,8 +17,8 @@ import ( svcMetric "github.com/rudderlabs/rudder-go-kit/stats/metric" miniodocker "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/minio" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/services/geolocation" - "github.com/rudderlabs/rudder-server/utils/types" ) func TestGeolocationEnrichment_Setup(t *testing.T) { diff --git a/mocks/processor/transformer/mock_transformer.go b/mocks/processor/transformer/mock_transformer.go index 2bbf4758b4..bd4251f0a7 100644 --- a/mocks/processor/transformer/mock_transformer.go +++ b/mocks/processor/transformer/mock_transformer.go @@ -13,7 +13,7 @@ import ( context "context" reflect "reflect" - transformer "github.com/rudderlabs/rudder-server/processor/transformer" + types "github.com/rudderlabs/rudder-server/processor/types" gomock "go.uber.org/mock/gomock" ) @@ -42,10 +42,10 @@ func (m *MockTransformer) EXPECT() *MockTransformerMockRecorder { } // Transform mocks base method. -func (m *MockTransformer) Transform(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { +func (m *MockTransformer) Transform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Transform", ctx, clientEvents, batchSize) - ret0, _ := ret[0].(transformer.Response) + ret0, _ := ret[0].(types.Response) return ret0 } @@ -56,10 +56,10 @@ func (mr *MockTransformerMockRecorder) Transform(ctx, clientEvents, batchSize an } // UserTransform mocks base method. -func (m *MockTransformer) UserTransform(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { +func (m *MockTransformer) UserTransform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UserTransform", ctx, clientEvents, batchSize) - ret0, _ := ret[0].(transformer.Response) + ret0, _ := ret[0].(types.Response) return ret0 } @@ -70,10 +70,10 @@ func (mr *MockTransformerMockRecorder) UserTransform(ctx, clientEvents, batchSiz } // Validate mocks base method. -func (m *MockTransformer) Validate(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { +func (m *MockTransformer) Validate(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Validate", ctx, clientEvents, batchSize) - ret0, _ := ret[0].(transformer.Response) + ret0, _ := ret[0].(types.Response) return ret0 } diff --git a/processor/consent.go b/processor/consent.go index 1d5af5bf5c..671e77b5ac 100644 --- a/processor/consent.go +++ b/processor/consent.go @@ -6,8 +6,8 @@ import ( "github.com/samber/lo" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" ) type ConsentManagementInfo struct { diff --git a/processor/consent_test.go b/processor/consent_test.go index 11c9fc0942..af65d3feb3 100644 --- a/processor/consent_test.go +++ b/processor/consent_test.go @@ -4,13 +4,11 @@ import ( "testing" "github.com/samber/lo" + "github.com/stretchr/testify/require" "github.com/rudderlabs/rudder-go-kit/logger" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - - "github.com/stretchr/testify/require" - - "github.com/rudderlabs/rudder-server/utils/types" + "github.com/rudderlabs/rudder-server/processor/types" ) func TestGetOneTrustConsentCategories(t *testing.T) { diff --git a/processor/delayed/stats.go b/processor/delayed/stats.go index 62f8f427bd..31447eeeb5 100644 --- a/processor/delayed/stats.go +++ b/processor/delayed/stats.go @@ -4,10 +4,11 @@ import ( "strings" "time" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/stats" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" "github.com/rudderlabs/rudder-server/utils/misc" ) @@ -25,7 +26,7 @@ func NewEventStats(stats stats.Stats, config *config.Config) *eventStats { } } -func (s *eventStats) ObserveSourceEvents(source *backendconfig.SourceT, events []transformer.TransformerEvent) { +func (s *eventStats) ObserveSourceEvents(source *backendconfig.SourceT, events []types.TransformerEvent) { statusCount := map[string]map[string]int{ "missing_original_timestamp": {}, "missing_sent_at": {}, diff --git a/processor/delayed/stats_test.go b/processor/delayed/stats_test.go index 9ec6174482..2137bc4636 100644 --- a/processor/delayed/stats_test.go +++ b/processor/delayed/stats_test.go @@ -3,6 +3,8 @@ package delayed_test import ( "testing" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/stretchr/testify/require" "github.com/rudderlabs/rudder-go-kit/stats" @@ -11,7 +13,6 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-server/processor/delayed" - "github.com/rudderlabs/rudder-server/processor/transformer" ) func TestEventStats(t *testing.T) { @@ -36,7 +37,7 @@ func TestEventStats(t *testing.T) { sdkLibrary := "rudder-go" sdkVersion := "1.0.0" - es.ObserveSourceEvents(source, fromSDK(sdkLibrary, sdkVersion, []transformer.TransformerEvent{ + es.ObserveSourceEvents(source, fromSDK(sdkLibrary, sdkVersion, []types.TransformerEvent{ // missing originalTimestamp { Message: map[string]interface{}{ @@ -228,7 +229,7 @@ func TestEventStats(t *testing.T) { t.Run(tc.name, func(t *testing.T) { tc.message["originalTimestamp"] = "2020-01-01T00:00:00.000Z" tc.message["sentAt"] = "2020-01-01T00:10:00.000Z" - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { Message: tc.message, }, @@ -259,7 +260,7 @@ func TestEventStats(t *testing.T) { }) } -func fromSDK(lib, version string, events []transformer.TransformerEvent) []transformer.TransformerEvent { +func fromSDK(lib, version string, events []types.TransformerEvent) []types.TransformerEvent { for i := range events { events[i].Message["context"] = map[string]interface{}{ "library": map[string]interface{}{ diff --git a/processor/eventfilter/eventfilter.go b/processor/eventfilter/eventfilter.go index a57eff9d3e..4b434b4725 100644 --- a/processor/eventfilter/eventfilter.go +++ b/processor/eventfilter/eventfilter.go @@ -4,11 +4,12 @@ import ( "slices" "strings" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" + "github.com/rudderlabs/rudder-go-kit/logger" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" ) const ( @@ -84,7 +85,7 @@ func GetSupportedMessageEvents(destination *backendconfig.DestinationT) ([]strin } type AllowTransformerEventParams struct { - TransformerEvent *transformer.TransformerEvent + TransformerEvent *types.TransformerEvent SupportedMessageTypes []string } @@ -122,12 +123,12 @@ Currently this method supports below validations(executed in the same order): 2. Validate if the event is sendable to destination based on connectionMode, sourceType & messageType */ -func AllowEventToDestTransformation(transformerEvent *transformer.TransformerEvent, supportedMsgTypes []string) (bool, *transformer.TransformerResponse) { +func AllowEventToDestTransformation(transformerEvent *types.TransformerEvent, supportedMsgTypes []string) (bool, *types.TransformerResponse) { // MessageType filtering -- STARTS messageType := strings.TrimSpace(strings.ToLower(getMessageType(&transformerEvent.Message))) if messageType == "" { // We will abort the event - return false, &transformer.TransformerResponse{ + return false, &types.TransformerResponse{ Output: transformerEvent.Message, StatusCode: 400, Metadata: transformerEvent.Metadata, Error: "Invalid message type. Type assertion failed", @@ -140,8 +141,8 @@ func AllowEventToDestTransformation(transformerEvent *transformer.TransformerEve "supportedMsgTypes", supportedMsgTypes, "messageType", messageType, ) // We will not allow the event - return false, &transformer.TransformerResponse{ - Output: transformerEvent.Message, StatusCode: types.FilterEventCode, + return false, &types.TransformerResponse{ + Output: transformerEvent.Message, StatusCode: reportingTypes.FilterEventCode, Metadata: transformerEvent.Metadata, Error: "Message type not supported", } @@ -164,8 +165,8 @@ func AllowEventToDestTransformation(transformerEvent *transformer.TransformerEve }) if !allow { - return allow, &transformer.TransformerResponse{ - Output: transformerEvent.Message, StatusCode: types.FilterEventCode, + return allow, &types.TransformerResponse{ + Output: transformerEvent.Message, StatusCode: reportingTypes.FilterEventCode, Metadata: transformerEvent.Metadata, Error: "Filtering event based on hybridModeFilter", } diff --git a/processor/eventfilter/eventfilter_test.go b/processor/eventfilter/eventfilter_test.go index be6d7f1145..d47d34861b 100644 --- a/processor/eventfilter/eventfilter_test.go +++ b/processor/eventfilter/eventfilter_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" ) func TestFilterEventsForHybridMode(t *testing.T) { @@ -458,19 +458,19 @@ func TestConvertToArrayOfType(t *testing.T) { func TestAllowEventToDestTransformation(t *testing.T) { type testCaseT struct { caseName string - transformerEvent *transformer.TransformerEvent + transformerEvent *types.TransformerEvent expected bool supportedMsgTypes []string - expectedResp *transformer.TransformerResponse + expectedResp *types.TransformerResponse } testCases := []testCaseT{ { caseName: "if message type is invalid, return false with statusCode 400", - transformerEvent: &transformer.TransformerEvent{Message: map[string]interface{}{"type": ""}}, + transformerEvent: &types.TransformerEvent{Message: map[string]interface{}{"type": ""}}, expected: false, supportedMsgTypes: []string{"track"}, - expectedResp: &transformer.TransformerResponse{ + expectedResp: &types.TransformerResponse{ Output: map[string]interface{}{"type": ""}, StatusCode: 400, Error: "Invalid message type. Type assertion failed", @@ -478,10 +478,10 @@ func TestAllowEventToDestTransformation(t *testing.T) { }, { caseName: "if message type is unsupported, return false with statusCode 298", - transformerEvent: &transformer.TransformerEvent{Message: map[string]interface{}{"type": "identify"}}, + transformerEvent: &types.TransformerEvent{Message: map[string]interface{}{"type": "identify"}}, expected: false, supportedMsgTypes: []string{"track"}, - expectedResp: &transformer.TransformerResponse{ + expectedResp: &types.TransformerResponse{ Output: map[string]interface{}{"type": "identify"}, StatusCode: 298, Error: "Message type not supported", @@ -489,10 +489,10 @@ func TestAllowEventToDestTransformation(t *testing.T) { }, { caseName: "if event is filtered due to FilterEventsForHybridMode, return statusCode 298", - transformerEvent: &transformer.TransformerEvent{Message: map[string]interface{}{"type": "track"}, Metadata: transformer.Metadata{}}, + transformerEvent: &types.TransformerEvent{Message: map[string]interface{}{"type": "track"}, Metadata: types.Metadata{}}, expected: false, supportedMsgTypes: []string{"track"}, - expectedResp: &transformer.TransformerResponse{ + expectedResp: &types.TransformerResponse{ Output: map[string]interface{}{"type": "track"}, StatusCode: 298, Error: "Filtering event based on hybridModeFilter", @@ -500,7 +500,7 @@ func TestAllowEventToDestTransformation(t *testing.T) { }, { caseName: "if event is legit, return true with nil response", - transformerEvent: &transformer.TransformerEvent{Message: map[string]interface{}{"type": "track"}, Destination: backendconfig.DestinationT{IsProcessorEnabled: true}}, + transformerEvent: &types.TransformerEvent{Message: map[string]interface{}{"type": "track"}, Destination: backendconfig.DestinationT{IsProcessorEnabled: true}}, expected: true, supportedMsgTypes: []string{"track"}, expectedResp: nil, diff --git a/processor/events_response.go b/processor/events_response.go index 95686dfdac..5612451305 100644 --- a/processor/events_response.go +++ b/processor/events_response.go @@ -6,24 +6,24 @@ import ( "github.com/samber/lo" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" ) -func (proc *Handle) getDroppedJobs(response transformer.Response, eventsToTransform []transformer.TransformerEvent) []*jobsdb.JobT { +func (proc *Handle) getDroppedJobs(response types.Response, eventsToTransform []types.TransformerEvent) []*jobsdb.JobT { // each messageID is one event when sending to the transformer - inputMessageIDs := lo.Map(eventsToTransform, func(e transformer.TransformerEvent, _ int) string { + inputMessageIDs := lo.Map(eventsToTransform, func(e types.TransformerEvent, _ int) string { return e.Metadata.MessageID }) // in transformer response, multiple messageIDs could be batched together successFullMessageIDs := make([]string, 0) - lo.ForEach(response.Events, func(e transformer.TransformerResponse, _ int) { + lo.ForEach(response.Events, func(e types.TransformerResponse, _ int) { successFullMessageIDs = append(successFullMessageIDs, e.Metadata.GetMessagesIDs()...) }) // for failed as well failedMessageIDs := make([]string, 0) - lo.ForEach(response.FailedEvents, func(e transformer.TransformerResponse, _ int) { + lo.ForEach(response.FailedEvents, func(e types.TransformerResponse, _ int) { failedMessageIDs = append(failedMessageIDs, e.Metadata.GetMessagesIDs()...) }) // the remainder of the messageIDs are those that are dropped diff --git a/processor/integrations/integrations.go b/processor/integrations/integrations.go index f9eaf9e736..9b64b74939 100644 --- a/processor/integrations/integrations.go +++ b/processor/integrations/integrations.go @@ -10,8 +10,8 @@ import ( "github.com/rudderlabs/rudder-go-kit/stats" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" ) var ( @@ -94,7 +94,7 @@ func ValidatePostInfo(transformRawParams PostParametersT) error { // FilterClientIntegrations parses the destination names from the // input JSON, matches them with enabled destinations from controle plane and returns the IDSs func FilterClientIntegrations(clientEvent types.SingularEventT, destNameIDMap map[string]backendconfig.DestinationDefinitionT) (retVal []string) { - clientIntgs, ok := misc.GetRudderEventVal("integrations", clientEvent) + clientIntgs, ok := types.GetRudderEventVal("integrations", clientEvent) if !ok { clientIntgs = make(map[string]interface{}) } diff --git a/processor/internal/destination_transformer/destination_transformer.go b/processor/internal/destination_transformer/destination_transformer.go index 7aa276c691..2ef88e361b 100644 --- a/processor/internal/destination_transformer/destination_transformer.go +++ b/processor/internal/destination_transformer/destination_transformer.go @@ -1,20 +1,26 @@ package destination_transformer import ( + "context" "fmt" "strings" + "time" "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" "github.com/rudderlabs/rudder-server/processor/internal/http_client" + "github.com/rudderlabs/rudder-server/processor/types" warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) type DestTransformer struct { config struct { - destTransformationURL string + destTransformationURL string + maxRetry config.ValueLoader[int] + failOnError config.ValueLoader[bool] + maxRetryBackoffInterval config.ValueLoader[time.Duration] } conf *config.Config log logger.Logger @@ -22,10 +28,9 @@ type DestTransformer struct { client http_client.HTTPDoer } -func (d *DestTransformer) SendRequest(data interface{}) (interface{}, error) { - fmt.Println("Sending request to Service A") - // Add service-specific logic - return "Response from Service A", nil +func (d *DestTransformer) SendRequest(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { + _ = d.destTransformURL(clientEvents[0].Destination.DestinationDefinition.Name) + return types.Response{} } func NewDestTransformer(conf *config.Config, log logger.Logger, stat stats.Stats) *DestTransformer { @@ -34,8 +39,10 @@ func NewDestTransformer(conf *config.Config, log logger.Logger, stat stats.Stats handle.log = log handle.stat = stat handle.client = http_client.NewHTTPClient(conf) - - handle.config.destTransformationURL = handle.conf.GetString("Warehouse.destTransformationURL", "http://localhost:9090") + handle.config.destTransformationURL = handle.conf.GetString("DEST_TRANSFORM_URL", "http://localhost:9090") + handle.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") + handle.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") + handle.config.maxRetryBackoffInterval = conf.GetReloadableDurationVar(30, time.Second, "Processor.maxRetryBackoffInterval") return handle } diff --git a/processor/internal/trackingplan_validation/trackingplan_validation.go b/processor/internal/trackingplan_validation/trackingplan_validation.go index 56caba741c..3c25237e6e 100644 --- a/processor/internal/trackingplan_validation/trackingplan_validation.go +++ b/processor/internal/trackingplan_validation/trackingplan_validation.go @@ -1,7 +1,10 @@ package trackingplan_validation import ( - "fmt" + "context" + "time" + + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" @@ -12,7 +15,10 @@ import ( type TPValidator struct { config struct { - destTransformationURL string + destTransformationURL string + maxRetry config.ValueLoader[int] + failOnError config.ValueLoader[bool] + maxRetryBackoffInterval config.ValueLoader[time.Duration] } conf *config.Config log logger.Logger @@ -20,10 +26,9 @@ type TPValidator struct { client http_client.HTTPDoer } -func (t *TPValidator) SendRequest(data interface{}) (interface{}, error) { - fmt.Println("Sending request to Service A") - // Add service-specific logic - return "Response from Service A", nil +func (t *TPValidator) SendRequest(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { + _ = t.trackingPlanValidationURL() + return types.Response{} } func NewTPValidator(conf *config.Config, log logger.Logger, stat stats.Stats) *TPValidator { @@ -33,6 +38,9 @@ func NewTPValidator(conf *config.Config, log logger.Logger, stat stats.Stats) *T handle.stat = stat handle.client = http_client.NewHTTPClient(conf) handle.config.destTransformationURL = handle.conf.GetString("Warehouse.destTransformationURL", "http://localhost:9090") + handle.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") + handle.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") + handle.config.maxRetryBackoffInterval = conf.GetReloadableDurationVar(30, time.Second, "Processor.maxRetryBackoffInterval") return handle } diff --git a/processor/internal/user_transformer/user_transformer.go b/processor/internal/user_transformer/user_transformer.go index ef7ac3fadd..8a18e3b0e0 100644 --- a/processor/internal/user_transformer/user_transformer.go +++ b/processor/internal/user_transformer/user_transformer.go @@ -1,18 +1,24 @@ package user_transformer import ( - "fmt" + "context" + "time" "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" "github.com/rudderlabs/rudder-server/processor/internal/http_client" + "github.com/rudderlabs/rudder-server/processor/types" ) type UserTransformer struct { config struct { - userTransformationURL string + userTransformationURL string + maxRetry config.ValueLoader[int] + failOnUserTransformTimeout config.ValueLoader[bool] + failOnError config.ValueLoader[bool] + maxRetryBackoffInterval config.ValueLoader[time.Duration] } conf *config.Config log logger.Logger @@ -20,10 +26,9 @@ type UserTransformer struct { client http_client.HTTPDoer } -func (u *UserTransformer) SendRequest(data interface{}) (interface{}, error) { - fmt.Println("Sending request to Service A") - // Add service-specific logic - return "Response from Service A", nil +func (u *UserTransformer) SendRequest(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { + _ = u.userTransformURL() + return types.Response{} } func NewUserTransformer(conf *config.Config, log logger.Logger, stat stats.Stats) *UserTransformer { @@ -32,7 +37,11 @@ func NewUserTransformer(conf *config.Config, log logger.Logger, stat stats.Stats handle.log = log handle.stat = stat handle.client = http_client.NewHTTPClient(conf) - handle.config.userTransformationURL = handle.conf.GetString("Warehouse.userTransformationURL", "http://localhost:9090") + handle.config.userTransformationURL = handle.conf.GetString("USER_TRANSFORM_URL", "http://localhost:9090") + handle.config.failOnUserTransformTimeout = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnUserTransformTimeout") + handle.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") + handle.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") + handle.config.maxRetryBackoffInterval = conf.GetReloadableDurationVar(30, time.Second, "Processor.maxRetryBackoffInterval") return handle } diff --git a/processor/processor.go b/processor/processor.go index f7d5d2f4c8..2de2bd2650 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -40,6 +40,7 @@ import ( "github.com/rudderlabs/rudder-server/processor/isolation" "github.com/rudderlabs/rudder-server/processor/stash" "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/router/batchrouter" "github.com/rudderlabs/rudder-server/rruntime" destinationdebugger "github.com/rudderlabs/rudder-server/services/debugger/destination" @@ -54,7 +55,7 @@ import ( "github.com/rudderlabs/rudder-server/utils/crash" "github.com/rudderlabs/rudder-server/utils/misc" . "github.com/rudderlabs/rudder-server/utils/tx" //nolint:staticcheck - "github.com/rudderlabs/rudder-server/utils/types" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" "github.com/rudderlabs/rudder-server/utils/workerpool" ) @@ -75,7 +76,7 @@ func NewHandle(c *config.Config, transformer transformer.Transformer) *Handle { } type sourceObserver interface { - ObserveSourceEvents(source *backendconfig.SourceT, events []transformer.TransformerEvent) + ObserveSourceEvents(source *backendconfig.SourceT, events []types.TransformerEvent) } type trackedUsersReporter interface { @@ -101,7 +102,7 @@ type Handle struct { logger logger.Logger enrichers []enricher.PipelineEnricher dedup dedup.Dedup - reporting types.Reporting + reporting reportingTypes.Reporting reportingEnabled bool backgroundWait func() error backgroundCancel context.CancelFunc @@ -157,7 +158,7 @@ type Handle struct { enableParallelScan bool archivalEnabled config.ValueLoader[bool] eventAuditEnabled map[string]bool - credentialsMap map[string][]transformer.Credential + credentialsMap map[string][]types.Credential nonEventStreamSources map[string]bool } @@ -260,10 +261,10 @@ type MetricMetadata struct { type NonSuccessfulTransformationMetrics struct { failedJobs []*jobsdb.JobT - failedMetrics []*types.PUReportedMetric + failedMetrics []*reportingTypes.PUReportedMetric failedCountMap map[string]int64 filteredJobs []*jobsdb.JobT - filteredMetrics []*types.PUReportedMetric + filteredMetrics []*reportingTypes.PUReportedMetric filteredCountMap map[string]int64 } @@ -376,7 +377,7 @@ func (proc *Handle) Setup( gatewayDB, routerDB, batchRouterDB, readErrorDB, writeErrorDB, eventSchemaDB, archivalDB jobsdb.JobsDB, - reporting types.Reporting, + reporting reportingTypes.Reporting, transientSources transientsource.Service, fileuploader fileuploader.Provider, rsourcesService rsources.JobService, @@ -389,7 +390,7 @@ func (proc *Handle) Setup( proc.reporting = reporting proc.destDebugger = destDebugger proc.transDebugger = transDebugger - proc.reportingEnabled = config.GetBoolVar(types.DefaultReportingEnabled, "Reporting.enabled") + proc.reportingEnabled = config.GetBoolVar(reportingTypes.DefaultReportingEnabled, "Reporting.enabled") if proc.conf == nil { proc.conf = config.Default } @@ -844,7 +845,7 @@ func (proc *Handle) backendConfigSubscriber(ctx context.Context) { sourceIdDestinationMap = make(map[string][]backendconfig.DestinationT) sourceIdSourceMap = make(map[string]backendconfig.SourceT) eventAuditEnabled = make(map[string]bool) - credentialsMap = make(map[string][]transformer.Credential) + credentialsMap = make(map[string][]types.Credential) nonEventStreamSources = make(map[string]bool) connectionConfigMap = make(map[connection]backendconfig.Connection) ) @@ -875,8 +876,8 @@ func (proc *Handle) backendConfigSubscriber(ctx context.Context) { } workspaceLibrariesMap[workspaceID] = wConfig.Libraries eventAuditEnabled[workspaceID] = wConfig.Settings.EventAuditEnabled - credentialsMap[workspaceID] = lo.MapToSlice(wConfig.Credentials, func(key string, value backendconfig.Credential) transformer.Credential { - return transformer.Credential{ + credentialsMap[workspaceID] = lo.MapToSlice(wConfig.Credentials, func(key string, value backendconfig.Credential) types.Credential { + return types.Credential{ ID: key, Key: value.Key, Value: value.Value, @@ -968,7 +969,7 @@ func getTimestampFromEvent(event types.SingularEventT, field string, defaultTime return timestamp } -func enhanceWithTimeFields(event *transformer.TransformerEvent, singularEvent types.SingularEventT, receivedAt time.Time) { +func enhanceWithTimeFields(event *types.TransformerEvent, singularEvent types.SingularEventT, receivedAt time.Time) { // set timestamp skew based on timestamp fields from SDKs originalTimestamp := getTimestampFromEvent(singularEvent, "originalTimestamp", receivedAt) sentAt := getTimestampFromEvent(singularEvent, "sentAt", receivedAt) @@ -989,8 +990,8 @@ func enhanceWithTimeFields(event *transformer.TransformerEvent, singularEvent ty event.Message["timestamp"] = timestamp.Format(misc.RFC3339Milli) } -func (proc *Handle) makeCommonMetadataFromSingularEvent(singularEvent types.SingularEventT, userID string, jobId int64, receivedAt time.Time, source *backendconfig.SourceT, eventParams types.EventParams) *transformer.Metadata { - commonMetadata := transformer.Metadata{} +func (proc *Handle) makeCommonMetadataFromSingularEvent(singularEvent types.SingularEventT, userID string, jobId int64, receivedAt time.Time, source *backendconfig.SourceT, eventParams types.EventParams) *types.Metadata { + commonMetadata := types.Metadata{} commonMetadata.SourceID = source.ID commonMetadata.SourceName = source.Name commonMetadata.OriginalSourceID = source.OriginalID @@ -1020,8 +1021,8 @@ func (proc *Handle) makeCommonMetadataFromSingularEvent(singularEvent types.Sing } // add metadata to each singularEvent which will be returned by transformer in response -func enhanceWithMetadata(commonMetadata *transformer.Metadata, event *transformer.TransformerEvent, destination *backendconfig.DestinationT) { - metadata := transformer.Metadata{} +func enhanceWithMetadata(commonMetadata *types.Metadata, event *types.TransformerEvent, destination *backendconfig.DestinationT) { + metadata := types.Metadata{} metadata.SourceType = commonMetadata.SourceType metadata.SourceCategory = commonMetadata.SourceCategory metadata.SourceID = commonMetadata.SourceID @@ -1113,24 +1114,24 @@ func (proc *Handle) recordEventDeliveryStatus(jobsByDestID map[string][]*jobsdb. } func (proc *Handle) getTransformerEvents( - response transformer.Response, - commonMetaData *transformer.Metadata, + response types.Response, + commonMetaData *types.Metadata, eventsByMessageID map[string]types.SingularEventWithReceivedAt, destination *backendconfig.DestinationT, connection backendconfig.Connection, inPU, pu string, ) ( - []transformer.TransformerEvent, - []*types.PUReportedMetric, + []types.TransformerEvent, + []*reportingTypes.PUReportedMetric, map[string]int64, map[string]MetricMetadata, ) { - successMetrics := make([]*types.PUReportedMetric, 0) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + successMetrics := make([]*reportingTypes.PUReportedMetric, 0) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) successCountMap := make(map[string]int64) successCountMetadataMap := make(map[string]MetricMetadata) - var eventsToTransform []transformer.TransformerEvent + var eventsToTransform []types.TransformerEvent for i := range response.Events { // Update metrics maps userTransformedEvent := &response.Events[i] @@ -1143,7 +1144,7 @@ func (proc *Handle) getTransformerEvents( for _, message := range messages { proc.updateMetricMaps(successCountMetadataMap, successCountMap, connectionDetailsMap, statusDetailsMap, userTransformedEvent, jobsdb.Succeeded.State, pu, func() json.RawMessage { - if pu != types.TRACKINGPLAN_VALIDATOR { + if pu != reportingTypes.TRACKINGPLAN_VALIDATOR { return []byte(`{}`) } if proc.transientSources.Apply(commonMetaData.SourceID) { @@ -1176,7 +1177,7 @@ func (proc *Handle) getTransformerEvents( eventMetadata.DestinationDefinitionID = userTransformedEvent.Metadata.DestinationDefinitionID eventMetadata.SourceCategory = userTransformedEvent.Metadata.SourceCategory eventMetadata.TraceParent = userTransformedEvent.Metadata.TraceParent - updatedEvent := transformer.TransformerEvent{ + updatedEvent := types.TransformerEvent{ Message: userTransformedEvent.Output, Metadata: *eventMetadata, Destination: *destination, @@ -1188,13 +1189,13 @@ func (proc *Handle) getTransformerEvents( // REPORTING - START if proc.isReportingEnabled() { - types.AssertSameKeys(connectionDetailsMap, statusDetailsMap) + reportingTypes.AssertSameKeys(connectionDetailsMap, statusDetailsMap) for k, cd := range connectionDetailsMap { for _, sd := range statusDetailsMap[k] { - m := &types.PUReportedMetric{ + m := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails(inPU, pu, false, false), + PUDetails: *reportingTypes.CreatePUDetails(inPU, pu, false, false), StatusDetail: sd, } successMetrics = append(successMetrics, m) @@ -1209,9 +1210,9 @@ func (proc *Handle) getTransformerEvents( func (proc *Handle) updateMetricMaps( countMetadataMap map[string]MetricMetadata, countMap map[string]int64, - connectionDetailsMap map[string]*types.ConnectionDetails, - statusDetailsMap map[string]map[string]*types.StatusDetail, - event *transformer.TransformerResponse, + connectionDetailsMap map[string]*reportingTypes.ConnectionDetails, + statusDetailsMap map[string]map[string]*reportingTypes.StatusDetail, + event *types.TransformerResponse, status, stage string, payload func() json.RawMessage, eventsByMessageID map[string]types.SingularEventWithReceivedAt, @@ -1264,7 +1265,7 @@ func (proc *Handle) updateMetricMaps( ) if _, ok := connectionDetailsMap[key]; !ok { - connectionDetailsMap[key] = &types.ConnectionDetails{ + connectionDetailsMap[key] = &reportingTypes.ConnectionDetails{ SourceID: event.Metadata.SourceID, SourceTaskRunID: event.Metadata.SourceTaskRunID, SourceJobID: event.Metadata.SourceJobID, @@ -1281,16 +1282,16 @@ func (proc *Handle) updateMetricMaps( } if _, ok := statusDetailsMap[key]; !ok { - statusDetailsMap[key] = make(map[string]*types.StatusDetail) + statusDetailsMap[key] = make(map[string]*reportingTypes.StatusDetail) } // create status details for each validation error // single event can have multiple validation errors of same type veCount := len(event.ValidationErrors) - if stage == types.TRACKINGPLAN_VALIDATOR && status == jobsdb.Succeeded.State { + if stage == reportingTypes.TRACKINGPLAN_VALIDATOR && status == jobsdb.Succeeded.State { if veCount > 0 { - status = types.SUCCEEDED_WITH_VIOLATIONS + status = reportingTypes.SUCCEEDED_WITH_VIOLATIONS } else { - status = types.SUCCEEDED_WITHOUT_VIOLATIONS + status = reportingTypes.SUCCEEDED_WITHOUT_VIOLATIONS } } sdkeySet := map[string]struct{}{} @@ -1300,7 +1301,7 @@ func (proc *Handle) updateMetricMaps( sd, ok := statusDetailsMap[key][sdkey] if !ok { - sd = &types.StatusDetail{ + sd = &reportingTypes.StatusDetail{ Status: status, StatusCode: event.StatusCode, SampleResponse: event.Error, @@ -1322,7 +1323,7 @@ func (proc *Handle) updateMetricMaps( sdkey := fmt.Sprintf("%s:%d:%s:%s:%s", status, event.StatusCode, eventName, eventType, "") sd, ok := statusDetailsMap[key][sdkey] if !ok { - sd = &types.StatusDetail{ + sd = &reportingTypes.StatusDetail{ Status: status, StatusCode: event.StatusCode, SampleResponse: event.Error, @@ -1339,15 +1340,15 @@ func (proc *Handle) updateMetricMaps( messageIDs := lo.Uniq(event.Metadata.GetMessagesIDs()) // this is called defensive programming... :( for _, messageID := range messageIDs { receivedAt := eventsByMessageID[messageID].ReceivedAt - sd.FailedMessages = append(sd.FailedMessages, &types.FailedMessage{MessageID: messageID, ReceivedAt: receivedAt}) + sd.FailedMessages = append(sd.FailedMessages, &reportingTypes.FailedMessage{MessageID: messageID, ReceivedAt: receivedAt}) } } sd.ViolationCount += int64(veCount) } func (proc *Handle) getNonSuccessfulMetrics( - response transformer.Response, - commonMetaData *transformer.Metadata, + response types.Response, + commonMetaData *types.Metadata, eventsByMessageID map[string]types.SingularEventWithReceivedAt, inPU, pu string, ) *NonSuccessfulTransformationMetrics { @@ -1355,8 +1356,8 @@ func (proc *Handle) getNonSuccessfulMetrics( grouped := lo.GroupBy( response.FailedEvents, - func(event transformer.TransformerResponse) bool { - return event.StatusCode == types.FilterEventCode + func(event types.TransformerResponse) bool { + return event.StatusCode == reportingTypes.FilterEventCode }, ) filtered, failed := grouped[true], grouped[false] @@ -1407,15 +1408,15 @@ func procErrorCountsStat(destType, pu, statusCode string) { } func (proc *Handle) getTransformationMetrics( - transformerResponses []transformer.TransformerResponse, + transformerResponses []types.TransformerResponse, state string, - commonMetaData *transformer.Metadata, + commonMetaData *types.Metadata, eventsByMessageID map[string]types.SingularEventWithReceivedAt, inPU, pu string, -) ([]*jobsdb.JobT, []*types.PUReportedMetric, map[string]int64) { - metrics := make([]*types.PUReportedMetric, 0) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) +) ([]*jobsdb.JobT, []*reportingTypes.PUReportedMetric, map[string]int64) { + metrics := make([]*reportingTypes.PUReportedMetric, 0) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) countMap := make(map[string]int64) var jobs []*jobsdb.JobT statFunc := procErrorCountsStat @@ -1501,12 +1502,12 @@ func (proc *Handle) getTransformationMetrics( // REPORTING - START if proc.isReportingEnabled() { - types.AssertSameKeys(connectionDetailsMap, statusDetailsMap) + reportingTypes.AssertSameKeys(connectionDetailsMap, statusDetailsMap) for k, cd := range connectionDetailsMap { for _, sd := range statusDetailsMap[k] { - m := &types.PUReportedMetric{ + m := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails(inPU, pu, false, false), + PUDetails: *reportingTypes.CreatePUDetails(inPU, pu, false, false), StatusDetail: sd, } metrics = append(metrics, m) @@ -1566,10 +1567,10 @@ func getDiffMetrics( inCountMetadataMap map[string]MetricMetadata, inCountMap, successCountMap, failedCountMap, filteredCountMap map[string]int64, statFactory stats.Stats, -) []*types.PUReportedMetric { +) []*reportingTypes.PUReportedMetric { // Calculate diff and append to reportMetrics // diff = successCount + abortCount - inCount - diffMetrics := make([]*types.PUReportedMetric, 0) + diffMetrics := make([]*reportingTypes.PUReportedMetric, 0) for key, inCount := range inCountMap { var eventName, eventType string splitKey := strings.Split(key, MetricKeyDelimiter) @@ -1586,8 +1587,8 @@ func getDiffMetrics( diff := successCount + failedCount + filteredCount - inCount if diff != 0 { metricMetadata := inCountMetadataMap[key] - metric := &types.PUReportedMetric{ - ConnectionDetails: types.ConnectionDetails{ + metric := &reportingTypes.PUReportedMetric{ + ConnectionDetails: reportingTypes.ConnectionDetails{ SourceID: metricMetadata.sourceID, DestinationID: metricMetadata.destinationID, SourceTaskRunID: metricMetadata.sourceTaskRunID, @@ -1601,12 +1602,12 @@ func getDiffMetrics( TrackingPlanID: metricMetadata.trackingPlanID, TrackingPlanVersion: metricMetadata.trackingPlanVersion, }, - PUDetails: types.PUDetails{ + PUDetails: reportingTypes.PUDetails{ InPU: inPU, PU: pu, }, - StatusDetail: &types.StatusDetail{ - Status: types.DiffStatus, + StatusDetail: &reportingTypes.StatusDetail{ + Status: reportingTypes.DiffStatus, Count: diff, SampleEvent: []byte(`{}`), EventName: eventName, @@ -1644,20 +1645,20 @@ type preTransformationMessage struct { subJobs subJob eventSchemaJobs []*jobsdb.JobT archivalJobs []*jobsdb.JobT - connectionDetailsMap map[string]*types.ConnectionDetails - statusDetailsMap map[string]map[string]*types.StatusDetail - reportMetrics []*types.PUReportedMetric - destFilterStatusDetailMap map[string]map[string]*types.StatusDetail + connectionDetailsMap map[string]*reportingTypes.ConnectionDetails + statusDetailsMap map[string]map[string]*reportingTypes.StatusDetail + reportMetrics []*reportingTypes.PUReportedMetric + destFilterStatusDetailMap map[string]map[string]*reportingTypes.StatusDetail inCountMetadataMap map[string]MetricMetadata inCountMap map[string]int64 outCountMap map[string]int64 totalEvents int marshalStart time.Time - groupedEventsBySourceId map[SourceIDT][]transformer.TransformerEvent + groupedEventsBySourceId map[SourceIDT][]types.TransformerEvent eventsByMessageID map[string]types.SingularEventWithReceivedAt procErrorJobs []*jobsdb.JobT jobIDToSpecificDestMapOnly map[int64]string - groupedEvents map[string][]transformer.TransformerEvent + groupedEvents map[string][]types.TransformerEvent uniqueMessageIdsBySrcDestKey map[string]map[string]struct{} statusList []*jobsdb.JobStatusT jobList []*jobsdb.JobT @@ -1677,8 +1678,8 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra proc.stats.statNumRequests(partition).Count(len(jobList)) var statusList []*jobsdb.JobStatusT - groupedEvents := make(map[string][]transformer.TransformerEvent) - groupedEventsBySourceId := make(map[SourceIDT][]transformer.TransformerEvent) + groupedEvents := make(map[string][]types.TransformerEvent) + groupedEventsBySourceId := make(map[SourceIDT][]types.TransformerEvent) eventsByMessageID := make(map[string]types.SingularEventWithReceivedAt) var procErrorJobs []*jobsdb.JobT eventSchemaJobs := make([]*jobsdb.JobT, 0) @@ -1701,14 +1702,14 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra uniqueMessageIdsBySrcDestKey := make(map[string]map[string]struct{}) sourceDupStats := make(map[dupStatKey]int) - reportMetrics := make([]*types.PUReportedMetric, 0) + reportMetrics := make([]*reportingTypes.PUReportedMetric, 0) inCountMap := make(map[string]int64) inCountMetadataMap := make(map[string]MetricMetadata) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) outCountMap := make(map[string]int64) // destinations enabled - destFilterStatusDetailMap := make(map[string]map[string]*types.StatusDetail) + destFilterStatusDetailMap := make(map[string]map[string]*reportingTypes.StatusDetail) // map of jobID to destinationID: for messages that needs to be delivered to a specific destinations only jobIDToSpecificDestMapOnly := make(map[int64]string) @@ -1922,7 +1923,7 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra } // REPORTING - GATEWAY metrics - START // dummy event for metrics purposes only - transformerEvent := &transformer.TransformerResponse{} + transformerEvent := &types.TransformerResponse{} if proc.isReportingEnabled() { transformerEvent.Metadata = *commonMetadataFromSingularEvent proc.updateMetricMaps( @@ -1932,7 +1933,7 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra statusDetailsMap, transformerEvent, jobsdb.Succeeded.State, - types.GATEWAY, + reportingTypes.GATEWAY, func() json.RawMessage { if sourceIsTransient { return []byte(`{}`) @@ -1955,9 +1956,9 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra } if _, ok := groupedEventsBySourceId[SourceIDT(sourceId)]; !ok { - groupedEventsBySourceId[SourceIDT(sourceId)] = make([]transformer.TransformerEvent, 0) + groupedEventsBySourceId[SourceIDT(sourceId)] = make([]types.TransformerEvent, 0) } - shallowEventCopy := transformer.TransformerEvent{} + shallowEventCopy := types.TransformerEvent{} shallowEventCopy.Message = event.singularEvent shallowEventCopy.Message["request_ip"] = event.requestIP enhanceWithTimeFields(&shallowEventCopy, event.singularEvent, event.recievedAt) @@ -1983,7 +1984,7 @@ func (proc *Handle) processJobsForDestV2(partition string, subJobs subJob) (*tra groupedEventsBySourceId[SourceIDT(sourceId)] = append(groupedEventsBySourceId[SourceIDT(sourceId)], shallowEventCopy) if proc.isReportingEnabled() { - proc.updateMetricMaps(inCountMetadataMap, outCountMap, connectionDetailsMap, destFilterStatusDetailMap, transformerEvent, jobsdb.Succeeded.State, types.DESTINATION_FILTER, func() json.RawMessage { return []byte(`{}`) }, nil) + proc.updateMetricMaps(inCountMetadataMap, outCountMap, connectionDetailsMap, destFilterStatusDetailMap, transformerEvent, jobsdb.Succeeded.State, reportingTypes.DESTINATION_FILTER, func() json.RawMessage { return []byte(`{}`) }, nil) } } @@ -2075,21 +2076,21 @@ func (proc *Handle) generateTransformationMessage(preTrans *preTransformationMes // REPORTING - GATEWAY metrics - START if proc.isReportingEnabled() { - types.AssertSameKeys(preTrans.connectionDetailsMap, preTrans.statusDetailsMap) + reportingTypes.AssertSameKeys(preTrans.connectionDetailsMap, preTrans.statusDetailsMap) for k, cd := range preTrans.connectionDetailsMap { for _, sd := range preTrans.statusDetailsMap[k] { - m := &types.PUReportedMetric{ + m := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails("", types.GATEWAY, false, true), + PUDetails: *reportingTypes.CreatePUDetails("", reportingTypes.GATEWAY, false, true), StatusDetail: sd, } preTrans.reportMetrics = append(preTrans.reportMetrics, m) } for _, dsd := range preTrans.destFilterStatusDetailMap[k] { - destFilterMetric := &types.PUReportedMetric{ + destFilterMetric := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails(types.GATEWAY, types.DESTINATION_FILTER, false, false), + PUDetails: *reportingTypes.CreatePUDetails(reportingTypes.GATEWAY, reportingTypes.DESTINATION_FILTER, false, false), StatusDetail: dsd, } preTrans.reportMetrics = append(preTrans.reportMetrics, destFilterMetric) @@ -2098,8 +2099,8 @@ func (proc *Handle) generateTransformationMessage(preTrans *preTransformationMes // empty failedCountMap because no failures, // events are just dropped at this point if no destination is found to route the events diffMetrics := getDiffMetrics( - types.GATEWAY, - types.DESTINATION_FILTER, + reportingTypes.GATEWAY, + reportingTypes.DESTINATION_FILTER, preTrans.inCountMetadataMap, preTrans.inCountMap, preTrans.outCountMap, @@ -2171,7 +2172,7 @@ func (proc *Handle) generateTransformationMessage(preTrans *preTransformationMes // Adding a singular event multiple times if there are multiple destinations of same type for idx := range enabledDestinationsList { destination := &enabledDestinationsList[idx] - shallowEventCopy := transformer.TransformerEvent{} + shallowEventCopy := types.TransformerEvent{} shallowEventCopy.Connection = proc.getConnectionConfig(connection{sourceID: sourceId, destinationID: destination.ID}) shallowEventCopy.Message = singularEvent shallowEventCopy.Destination = *destination @@ -2197,7 +2198,7 @@ func (proc *Handle) generateTransformationMessage(preTrans *preTransformationMes // We have at-least one event so marking it good _, ok := preTrans.groupedEvents[srcAndDestKey] if !ok { - preTrans.groupedEvents[srcAndDestKey] = make([]transformer.TransformerEvent, 0) + preTrans.groupedEvents[srcAndDestKey] = make([]types.TransformerEvent, 0) } preTrans.groupedEvents[srcAndDestKey] = append(preTrans.groupedEvents[srcAndDestKey], shallowEventCopy) @@ -2249,8 +2250,8 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans proc.stats.statNumRequests(partition).Count(len(jobList)) var statusList []*jobsdb.JobStatusT - groupedEvents := make(map[string][]transformer.TransformerEvent) - groupedEventsBySourceId := make(map[SourceIDT][]transformer.TransformerEvent) + groupedEvents := make(map[string][]types.TransformerEvent) + groupedEventsBySourceId := make(map[SourceIDT][]types.TransformerEvent) eventsByMessageID := make(map[string]types.SingularEventWithReceivedAt) var procErrorJobs []*jobsdb.JobT eventSchemaJobs := make([]*jobsdb.JobT, 0) @@ -2273,14 +2274,14 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans uniqueMessageIdsBySrcDestKey := make(map[string]map[string]struct{}) sourceDupStats := make(map[dupStatKey]int) - reportMetrics := make([]*types.PUReportedMetric, 0) + reportMetrics := make([]*reportingTypes.PUReportedMetric, 0) inCountMap := make(map[string]int64) inCountMetadataMap := make(map[string]MetricMetadata) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) outCountMap := make(map[string]int64) // destinations enabled - destFilterStatusDetailMap := make(map[string]map[string]*types.StatusDetail) + destFilterStatusDetailMap := make(map[string]map[string]*reportingTypes.StatusDetail) // map of jobID to destinationID: for messages that needs to be delivered to a specific destinations only jobIDToSpecificDestMapOnly := make(map[int64]string) @@ -2448,7 +2449,7 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans // REPORTING - GATEWAY metrics - START // dummy event for metrics purposes only - event := &transformer.TransformerResponse{} + event := &types.TransformerResponse{} if proc.isReportingEnabled() { event.Metadata = *commonMetadataFromSingularEvent proc.updateMetricMaps( @@ -2458,7 +2459,7 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans statusDetailsMap, event, jobsdb.Succeeded.State, - types.GATEWAY, + reportingTypes.GATEWAY, func() json.RawMessage { if sourceIsTransient { return []byte(`{}`) @@ -2482,9 +2483,9 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans } if _, ok := groupedEventsBySourceId[SourceIDT(sourceID)]; !ok { - groupedEventsBySourceId[SourceIDT(sourceID)] = make([]transformer.TransformerEvent, 0) + groupedEventsBySourceId[SourceIDT(sourceID)] = make([]types.TransformerEvent, 0) } - shallowEventCopy := transformer.TransformerEvent{} + shallowEventCopy := types.TransformerEvent{} shallowEventCopy.Message = singularEvent shallowEventCopy.Message["request_ip"] = requestIP enhanceWithTimeFields(&shallowEventCopy, singularEvent, receivedAt) @@ -2511,7 +2512,7 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans groupedEventsBySourceId[SourceIDT(sourceID)] = append(groupedEventsBySourceId[SourceIDT(sourceID)], shallowEventCopy) if proc.isReportingEnabled() { - proc.updateMetricMaps(inCountMetadataMap, outCountMap, connectionDetailsMap, destFilterStatusDetailMap, event, jobsdb.Succeeded.State, types.DESTINATION_FILTER, func() json.RawMessage { return []byte(`{}`) }, nil) + proc.updateMetricMaps(inCountMetadataMap, outCountMap, connectionDetailsMap, destFilterStatusDetailMap, event, jobsdb.Succeeded.State, reportingTypes.DESTINATION_FILTER, func() json.RawMessage { return []byte(`{}`) }, nil) } } } @@ -2550,12 +2551,12 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) (*trans } type transformationMessage struct { - groupedEvents map[string][]transformer.TransformerEvent + groupedEvents map[string][]types.TransformerEvent trackingPlanEnabledMap map[SourceIDT]bool eventsByMessageID map[string]types.SingularEventWithReceivedAt uniqueMessageIdsBySrcDestKey map[string]map[string]struct{} - reportMetrics []*types.PUReportedMetric + reportMetrics []*reportingTypes.PUReportedMetric statusList []*jobsdb.JobStatusT procErrorJobs []*jobsdb.JobT sourceDupStats map[dupStatKey]int @@ -2694,7 +2695,7 @@ type storeMessage struct { procErrorJobs []*jobsdb.JobT routerDestIDs []string - reportMetrics []*types.PUReportedMetric + reportMetrics []*reportingTypes.PUReportedMetric sourceDupStats map[dupStatKey]int dedupKeys map[string]struct{} @@ -2950,7 +2951,7 @@ func getJobCountsByWorkspaceDestType(jobs []*jobsdb.JobT) map[string]map[string] } type transformSrcDestOutput struct { - reportMetrics []*types.PUReportedMetric + reportMetrics []*reportingTypes.PUReportedMetric destJobs []*jobsdb.JobT batchDestJobs []*jobsdb.JobT errorsPerDestID map[string][]*jobsdb.JobT @@ -2962,7 +2963,7 @@ func (proc *Handle) transformSrcDest( ctx context.Context, partition string, // main inputs - srcAndDestKey string, eventList []transformer.TransformerEvent, + srcAndDestKey string, eventList []types.TransformerEvent, // helpers trackingPlanEnabledMap map[SourceIDT]bool, @@ -2977,7 +2978,7 @@ func (proc *Handle) transformSrcDest( connection := eventList[0].Connection workspaceID := eventList[0].Metadata.WorkspaceID destType := destination.DestinationDefinition.Name - commonMetaData := &transformer.Metadata{ + commonMetaData := &types.Metadata{ SourceID: sourceID, SourceName: sourceName, OriginalSourceID: eventList[0].Metadata.OriginalSourceID, @@ -2991,7 +2992,7 @@ func (proc *Handle) transformSrcDest( SourceDefinitionType: eventList[0].Metadata.SourceDefinitionType, } - reportMetrics := make([]*types.PUReportedMetric, 0) + reportMetrics := make([]*reportingTypes.PUReportedMetric, 0) batchDestJobs := make([]*jobsdb.JobT, 0) destJobs := make([]*jobsdb.JobT, 0) routerDestIDs := make(map[string]struct{}) @@ -3045,13 +3046,13 @@ func (proc *Handle) transformSrcDest( } // REPORTING - END - var response transformer.Response - var eventsToTransform []transformer.TransformerEvent + var response types.Response + var eventsToTransform []types.TransformerEvent var inPU string if trackingPlanEnabled { - inPU = types.TRACKINGPLAN_VALIDATOR + inPU = reportingTypes.TRACKINGPLAN_VALIDATOR } else { - inPU = types.DESTINATION_FILTER + inPU = reportingTypes.DESTINATION_FILTER } // Send to custom transformer only if the destination has a transformer enabled if transformationEnabled { @@ -3065,11 +3066,11 @@ func (proc *Handle) transformSrcDest( d := time.Since(startedAt) userTransformationStat.transformTime.SendTiming(d) - var successMetrics []*types.PUReportedMetric + var successMetrics []*reportingTypes.PUReportedMetric var successCountMap map[string]int64 var successCountMetadataMap map[string]MetricMetadata - eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, types.USER_TRANSFORMER) - nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, types.USER_TRANSFORMER) + eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, reportingTypes.USER_TRANSFORMER) + nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, reportingTypes.USER_TRANSFORMER) droppedJobs = append(droppedJobs, append(proc.getDroppedJobs(response, eventList), append(nonSuccessMetrics.failedJobs, nonSuccessMetrics.filteredJobs...)...)...) if _, ok := procErrorJobsByDestID[destID]; !ok { procErrorJobsByDestID[destID] = make([]*jobsdb.JobT, 0) @@ -3086,8 +3087,8 @@ func (proc *Handle) transformSrcDest( // REPORTING - START if proc.isReportingEnabled() { diffMetrics := getDiffMetrics( - types.DESTINATION_FILTER, - types.USER_TRANSFORMER, + reportingTypes.DESTINATION_FILTER, + reportingTypes.USER_TRANSFORMER, inCountMetadataMap, inCountMap, successCountMap, @@ -3105,7 +3106,7 @@ func (proc *Handle) transformSrcDest( inCountMetadataMap = successCountMetadataMap } // REPORTING - END - inPU = types.USER_TRANSFORMER // for the next step in the pipeline + inPU = reportingTypes.USER_TRANSFORMER // for the next step in the pipeline }) } else { proc.logger.Debug("No custom transformation") @@ -3141,7 +3142,7 @@ func (proc *Handle) transformSrcDest( response = ConvertToFilteredTransformerResponse( eventsToTransform, transformAt != "none", - func(event transformer.TransformerEvent) (bool, string) { + func(event types.TransformerEvent) (bool, string) { if event.Metadata.SourceJobRunID != "" && slices.Contains( proc.drainConfig.jobRunIDs.Load(), @@ -3156,23 +3157,23 @@ func (proc *Handle) transformSrcDest( return false, "" }, ) - var successMetrics []*types.PUReportedMetric + var successMetrics []*reportingTypes.PUReportedMetric var successCountMap map[string]int64 var successCountMetadataMap map[string]MetricMetadata - nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, types.EVENT_FILTER) + nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, reportingTypes.EVENT_FILTER) droppedJobs = append(droppedJobs, append(proc.getDroppedJobs(response, eventsToTransform), append(nonSuccessMetrics.failedJobs, nonSuccessMetrics.filteredJobs...)...)...) if _, ok := procErrorJobsByDestID[destID]; !ok { procErrorJobsByDestID[destID] = make([]*jobsdb.JobT, 0) } procErrorJobsByDestID[destID] = append(procErrorJobsByDestID[destID], nonSuccessMetrics.failedJobs...) - eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, types.EVENT_FILTER) + eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, reportingTypes.EVENT_FILTER) proc.logger.Debug("Supported messages filtering output size", len(eventsToTransform)) // REPORTING - START if proc.isReportingEnabled() { diffMetrics := getDiffMetrics( inPU, - types.EVENT_FILTER, + reportingTypes.EVENT_FILTER, inCountMetadataMap, inCountMap, successCountMap, @@ -3231,7 +3232,7 @@ func (proc *Handle) transformSrcDest( nonSuccessMetrics := proc.getNonSuccessfulMetrics( response, commonMetaData, eventsByMessageID, - types.EVENT_FILTER, types.DEST_TRANSFORMER, + reportingTypes.EVENT_FILTER, reportingTypes.DEST_TRANSFORMER, ) destTransformationStat.numEvents.Count(len(eventsToTransform)) destTransformationStat.numOutputSuccessEvents.Count(len(response.Events)) @@ -3246,21 +3247,21 @@ func (proc *Handle) transformSrcDest( // REPORTING - PROCESSOR metrics - START if proc.isReportingEnabled() { - successMetrics := make([]*types.PUReportedMetric, 0) - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + successMetrics := make([]*reportingTypes.PUReportedMetric, 0) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) successCountMap := make(map[string]int64) for i := range response.Events { // Update metrics maps - proc.updateMetricMaps(nil, successCountMap, connectionDetailsMap, statusDetailsMap, &response.Events[i], jobsdb.Succeeded.State, types.DEST_TRANSFORMER, func() json.RawMessage { return []byte(`{}`) }, nil) + proc.updateMetricMaps(nil, successCountMap, connectionDetailsMap, statusDetailsMap, &response.Events[i], jobsdb.Succeeded.State, reportingTypes.DEST_TRANSFORMER, func() json.RawMessage { return []byte(`{}`) }, nil) } - types.AssertSameKeys(connectionDetailsMap, statusDetailsMap) + reportingTypes.AssertSameKeys(connectionDetailsMap, statusDetailsMap) for k, cd := range connectionDetailsMap { for _, sd := range statusDetailsMap[k] { - m := &types.PUReportedMetric{ + m := &reportingTypes.PUReportedMetric{ ConnectionDetails: *cd, - PUDetails: *types.CreatePUDetails(types.EVENT_FILTER, types.DEST_TRANSFORMER, false, false), + PUDetails: *reportingTypes.CreatePUDetails(reportingTypes.EVENT_FILTER, reportingTypes.DEST_TRANSFORMER, false, false), StatusDetail: sd, } successMetrics = append(successMetrics, m) @@ -3268,8 +3269,8 @@ func (proc *Handle) transformSrcDest( } diffMetrics := getDiffMetrics( - types.EVENT_FILTER, - types.DEST_TRANSFORMER, + reportingTypes.EVENT_FILTER, + reportingTypes.DEST_TRANSFORMER, inCountMetadataMap, inCountMap, successCountMap, @@ -3395,12 +3396,12 @@ func (proc *Handle) saveDroppedJobs(ctx context.Context, droppedJobs []*jobsdb.J } func ConvertToFilteredTransformerResponse( - events []transformer.TransformerEvent, + events []types.TransformerEvent, filter bool, - drainFunc func(transformer.TransformerEvent) (bool, string), -) transformer.Response { - var responses []transformer.TransformerResponse - var failedEvents []transformer.TransformerResponse + drainFunc func(types.TransformerEvent) (bool, string), +) types.Response { + var responses []types.TransformerResponse + var failedEvents []types.TransformerResponse type cacheValue struct { values []string @@ -3417,9 +3418,9 @@ func ConvertToFilteredTransformerResponse( if drain, reason := drainFunc(*event); drain { failedEvents = append( failedEvents, - transformer.TransformerResponse{ + types.TransformerResponse{ Output: event.Message, - StatusCode: types.DrainEventCode, + StatusCode: reportingTypes.DrainEventCode, Metadata: event.Metadata, Error: reason, }, @@ -3458,7 +3459,7 @@ func ConvertToFilteredTransformerResponse( // add to FailedEvents failedEvents = append( failedEvents, - transformer.TransformerResponse{ + types.TransformerResponse{ Output: event.Message, StatusCode: 400, Metadata: event.Metadata, @@ -3470,9 +3471,9 @@ func ConvertToFilteredTransformerResponse( if !slices.Contains(supportedEvents.values, messageEvent) { failedEvents = append( failedEvents, - transformer.TransformerResponse{ + types.TransformerResponse{ Output: event.Message, - StatusCode: types.FilterEventCode, + StatusCode: reportingTypes.FilterEventCode, Metadata: event.Metadata, Error: "Event not supported", }, @@ -3485,7 +3486,7 @@ func ConvertToFilteredTransformerResponse( // allow event responses = append( responses, - transformer.TransformerResponse{ + types.TransformerResponse{ Output: event.Message, StatusCode: 200, Metadata: event.Metadata, @@ -3493,7 +3494,7 @@ func ConvertToFilteredTransformerResponse( ) } - return transformer.Response{Events: responses, FailedEvents: failedEvents} + return types.Response{Events: responses, FailedEvents: failedEvents} } func (proc *Handle) getJobs(partition string) jobsdb.JobsResult { @@ -3680,7 +3681,7 @@ func (proc *Handle) updateRudderSourcesStats(ctx context.Context, tx jobsdb.Stor return err } -func filterConfig(eventCopy *transformer.TransformerEvent) { +func filterConfig(eventCopy *types.TransformerEvent) { if configsToFilterI, ok := eventCopy.Destination.DestinationDefinition.Config["configFilters"]; ok { if configsToFilter, ok := configsToFilterI.([]interface{}); ok { omitKeys := lo.FilterMap(configsToFilter, func(configKey interface{}, _ int) (string, bool) { diff --git a/processor/processorBenchmark_test.go b/processor/processorBenchmark_test.go index e3c7bcdd14..9a614ec695 100644 --- a/processor/processorBenchmark_test.go +++ b/processor/processorBenchmark_test.go @@ -6,10 +6,9 @@ import ( "github.com/google/uuid" - "github.com/rudderlabs/rudder-server/utils/types" - backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" + "github.com/rudderlabs/rudder-server/processor/types" ) func Benchmark_makeCommonMetadataFromSingularEvent(b *testing.B) { diff --git a/processor/processor_test.go b/processor/processor_test.go index 5aca374841..c11e5cebff 100644 --- a/processor/processor_test.go +++ b/processor/processor_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" + "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -39,7 +41,7 @@ import ( mock_features "github.com/rudderlabs/rudder-server/mocks/services/transformer" mockReportingTypes "github.com/rudderlabs/rudder-server/mocks/utils/types" "github.com/rudderlabs/rudder-server/processor/isolation" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" destinationdebugger "github.com/rudderlabs/rudder-server/services/debugger/destination" transformationdebugger "github.com/rudderlabs/rudder-server/services/debugger/transformation" dedupTypes "github.com/rudderlabs/rudder-server/services/dedup/types" @@ -51,20 +53,19 @@ import ( "github.com/rudderlabs/rudder-server/utils/pubsub" testutils "github.com/rudderlabs/rudder-server/utils/tests" . "github.com/rudderlabs/rudder-server/utils/tx" //nolint:staticcheck - "github.com/rudderlabs/rudder-server/utils/types" ) type mockObserver struct { calls []struct { source *backendconfig.SourceT - events []transformer.TransformerEvent + events []types.TransformerEvent } } -func (m *mockObserver) ObserveSourceEvents(source *backendconfig.SourceT, events []transformer.TransformerEvent) { +func (m *mockObserver) ObserveSourceEvents(source *backendconfig.SourceT, events []types.TransformerEvent) { m.calls = append(m.calls, struct { source *backendconfig.SourceT - events []transformer.TransformerEvent + events []types.TransformerEvent }{source: source, events: events}) } @@ -955,7 +956,7 @@ var _ = Describe("Tracking Plan Validation", Ordered, func() { Context("RudderTyper", func() { It("Tracking plan id and version from DgSourceTrackingPlanConfig", func() { mockTransformer := mocksTransformer.NewMockTransformer(c.mockCtrl) - mockTransformer.EXPECT().Validate(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(transformer.Response{}) + mockTransformer.EXPECT().Validate(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(types.Response{}) isolationStrategy, err := isolation.GetStrategy(isolation.ModeNone) Expect(err).To(BeNil()) @@ -1030,7 +1031,7 @@ var _ = Describe("Tracking Plan Validation", Ordered, func() { }) It("Tracking plan version override from context.ruddertyper", func() { mockTransformer := mocksTransformer.NewMockTransformer(c.mockCtrl) - mockTransformer.EXPECT().Validate(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(transformer.Response{}) + mockTransformer.EXPECT().Validate(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(types.Response{}) isolationStrategy, err := isolation.GetStrategy(isolation.ModeNone) Expect(err).To(BeNil()) @@ -3104,19 +3105,19 @@ var _ = Describe("Processor", Ordered, func() { // We expect one call to user transform for destination B callUserTransform := mockTransformer.EXPECT().UserTransform(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).After(callUnprocessed). - DoAndReturn(func(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { + DoAndReturn(func(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { defer GinkgoRecover() - outputEvents := make([]transformer.TransformerResponse, 0) + outputEvents := make([]types.TransformerResponse, 0) for _, event := range clientEvents { event.Message["user-transform"] = "value" - outputEvents = append(outputEvents, transformer.TransformerResponse{ + outputEvents = append(outputEvents, types.TransformerResponse{ Output: event.Message, }) } - return transformer.Response{ + return types.Response{ Events: outputEvents, } }) @@ -3809,19 +3810,19 @@ var _ = Describe("Processor", Ordered, func() { // We expect one call to user transform for destination B callUserTransform := mockTransformer.EXPECT().UserTransform(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).After(callUnprocessed). - DoAndReturn(func(ctx context.Context, clientEvents []transformer.TransformerEvent, batchSize int) transformer.Response { + DoAndReturn(func(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { defer GinkgoRecover() - outputEvents := make([]transformer.TransformerResponse, 0) + outputEvents := make([]types.TransformerResponse, 0) for _, event := range clientEvents { event.Message["user-transform"] = "value" - outputEvents = append(outputEvents, transformer.TransformerResponse{ + outputEvents = append(outputEvents, types.TransformerResponse{ Output: event.Message, }) } - return transformer.Response{ + return types.Response{ Events: outputEvents, } }) @@ -3937,16 +3938,16 @@ var _ = Describe("Processor", Ordered, func() { }, } - transformerResponses := []transformer.TransformerResponse{ + transformerResponses := []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, StatusCode: 400, Error: "error-1", }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, StatusCode: 400, @@ -3996,8 +3997,8 @@ var _ = Describe("Processor", Ordered, func() { }).Return(jobsdb.JobsResult{Jobs: unprocessedJobsList}, nil).Times(1) // Test transformer failure mockTransformer.EXPECT().Transform(gomock.Any(), gomock.Any(), gomock.Any()).Times(1). - Return(transformer.Response{ - Events: []transformer.TransformerResponse{}, + Return(types.Response{ + Events: []types.TransformerResponse{}, FailedEvents: transformerResponses, }) @@ -4075,9 +4076,9 @@ var _ = Describe("Processor", Ordered, func() { }, } - transformerResponses := []transformer.TransformerResponse{ + transformerResponses := []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageIDs: []string{"message-1", "message-2"}, }, StatusCode: 400, @@ -4134,8 +4135,8 @@ var _ = Describe("Processor", Ordered, func() { // Test transformer failure mockTransformer.EXPECT().UserTransform(gomock.Any(), gomock.Any(), gomock.Any()).Times(1). - Return(transformer.Response{ - Events: []transformer.TransformerResponse{}, + Return(types.Response{ + Events: []types.TransformerResponse{}, FailedEvents: transformerResponses, }) @@ -4895,7 +4896,7 @@ var _ = Describe("Processor", Ordered, func() { defer cancel() Expect(processor.config.asyncInit.WaitContext(ctx)).To(BeNil()) - commonMetadata := transformer.Metadata{SourceID: SourceIDEnabled, DestinationID: DestinationIDEnabledA} + commonMetadata := types.Metadata{SourceID: SourceIDEnabled, DestinationID: DestinationIDEnabledA} singularEventWithReceivedAt1 := types.SingularEventWithReceivedAt{ SingularEvent: event1, ReceivedAt: time.Now(), @@ -4920,22 +4921,22 @@ var _ = Describe("Processor", Ordered, func() { metadata3 := commonMetadata metadata3.MessageID = "msg3" - FailedEvents := []transformer.TransformerResponse{ + FailedEvents := []types.TransformerResponse{ {StatusCode: 400, Metadata: metadata1, Output: event1}, {StatusCode: 298, Metadata: metadata2, Output: event2}, {StatusCode: 299, Metadata: metadata3, Output: event2}, } - transformerResponse := transformer.Response{ - Events: []transformer.TransformerResponse{}, + transformerResponse := types.Response{ + Events: []types.TransformerResponse{}, FailedEvents: FailedEvents, } m := processor.getNonSuccessfulMetrics(transformerResponse, &commonMetadata, eventsByMessageID, - types.EVENT_FILTER, - types.DEST_TRANSFORMER, + reportingTypes.EVENT_FILTER, + reportingTypes.DEST_TRANSFORMER, ) key := strings.Join([]string{ @@ -4948,7 +4949,7 @@ var _ = Describe("Processor", Ordered, func() { Expect(len(m.failedJobs)).To(Equal(2)) Expect(len(m.failedMetrics)).To(Equal(2)) - slices.SortFunc(m.failedMetrics, func(a, b *types.PUReportedMetric) int { + slices.SortFunc(m.failedMetrics, func(a, b *reportingTypes.PUReportedMetric) int { return a.StatusDetail.StatusCode - b.StatusDetail.StatusCode }) Expect(m.failedMetrics[0].StatusDetail.StatusCode).To(Equal(299)) @@ -4967,9 +4968,9 @@ var _ = Describe("Static Function Tests", func() { Context("TransformerFormatResponse Tests", func() { It("Should match ConvertToTransformerResponse without filtering", func() { - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -4977,7 +4978,7 @@ var _ = Describe("Static Function Tests", func() { }, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -4985,14 +4986,14 @@ var _ = Describe("Static Function Tests", func() { }, }, } - expectedResponses := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponses := types.Response{ + Events: []types.TransformerResponse{ { Output: map[string]interface{}{ "some-key-1": "some-value-1", }, StatusCode: 200, - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, }, @@ -5001,13 +5002,13 @@ var _ = Describe("Static Function Tests", func() { "some-key-2": "some-value-2", }, StatusCode: 200, - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, }, }, } - response := ConvertToFilteredTransformerResponse(events, false, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, false, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response.Events[0].StatusCode).To(Equal(expectedResponses.Events[0].StatusCode)) Expect(response.Events[0].Metadata.MessageID).To(Equal(expectedResponses.Events[0].Metadata.MessageID)) Expect(response.Events[0].Output["some-key-1"]).To(Equal(expectedResponses.Events[0].Output["some-key-1"])) @@ -5067,22 +5068,22 @@ var _ = Describe("Static Function Tests", func() { "some-key-2": 3, } - expectedResponse := []*types.PUReportedMetric{ + expectedResponse := []*reportingTypes.PUReportedMetric{ { - ConnectionDetails: types.ConnectionDetails{ + ConnectionDetails: reportingTypes.ConnectionDetails{ SourceID: "some-source-id-1", DestinationID: "some-destination-id-1", SourceTaskRunID: "some-source-task-run-id-1", SourceJobID: "some-source-job-id-1", SourceJobRunID: "some-source-job-run-id-1", }, - PUDetails: types.PUDetails{ + PUDetails: reportingTypes.PUDetails{ InPU: "some-string-1", PU: "some-string-2", TerminalPU: false, InitialPU: false, }, - StatusDetail: &types.StatusDetail{ + StatusDetail: &reportingTypes.StatusDetail{ Status: "diff", Count: 5, StatusCode: 0, @@ -5091,20 +5092,20 @@ var _ = Describe("Static Function Tests", func() { }, }, { - ConnectionDetails: types.ConnectionDetails{ + ConnectionDetails: reportingTypes.ConnectionDetails{ SourceID: "some-source-id-2", DestinationID: "some-destination-id-2", SourceTaskRunID: "some-source-task-run-id-2", SourceJobID: "some-source-job-id-2", SourceJobRunID: "some-source-job-run-id-2", }, - PUDetails: types.PUDetails{ + PUDetails: reportingTypes.PUDetails{ InPU: "some-string-1", PU: "some-string-2", TerminalPU: false, InitialPU: false, }, - StatusDetail: &types.StatusDetail{ + StatusDetail: &reportingTypes.StatusDetail{ Status: "diff", Count: 7, StatusCode: 0, @@ -5146,8 +5147,8 @@ var _ = Describe("Static Function Tests", func() { proc.reportingEnabled = true proc.reporting = &mockReportingTypes.MockReporting{} - inputEvent := &transformer.TransformerResponse{ - Metadata: transformer.Metadata{ + inputEvent := &types.TransformerResponse{ + Metadata: types.Metadata{ SourceID: "source-id-1", DestinationID: "destination-id-1", TransformationID: "transformation-id-1", @@ -5157,7 +5158,7 @@ var _ = Describe("Static Function Tests", func() { }, StatusCode: 200, Error: "", - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "type-1", Message: "message-1", @@ -5186,7 +5187,7 @@ var _ = Describe("Static Function Tests", func() { trackingPlanID: inputEvent.Metadata.TrackingPlanID, trackingPlanVersion: inputEvent.Metadata.TrackingPlanVersion, } - expectedConnectionDetails := &types.ConnectionDetails{ + expectedConnectionDetails := &reportingTypes.ConnectionDetails{ SourceID: inputEvent.Metadata.SourceID, DestinationID: inputEvent.Metadata.DestinationID, SourceJobRunID: inputEvent.Metadata.SourceJobRunID, @@ -5200,12 +5201,12 @@ var _ = Describe("Static Function Tests", func() { TrackingPlanID: inputEvent.Metadata.TrackingPlanID, TrackingPlanVersion: inputEvent.Metadata.TrackingPlanVersion, } - connectionDetailsMap := make(map[string]*types.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*types.StatusDetail) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) countMap := make(map[string]int64) countMetadataMap := make(map[string]MetricMetadata) // update metric maps - proc.updateMetricMaps(countMetadataMap, countMap, connectionDetailsMap, statusDetailsMap, inputEvent, jobsdb.Succeeded.State, types.TRACKINGPLAN_VALIDATOR, func() json.RawMessage { return []byte(`{}`) }, nil) + proc.updateMetricMaps(countMetadataMap, countMap, connectionDetailsMap, statusDetailsMap, inputEvent, jobsdb.Succeeded.State, reportingTypes.TRACKINGPLAN_VALIDATOR, func() json.RawMessage { return []byte(`{}`) }, nil) Expect(len(countMetadataMap)).To(Equal(1)) Expect(len(countMap)).To(Equal(1)) @@ -5234,9 +5235,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5246,7 +5247,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5256,7 +5257,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5266,15 +5267,15 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, Metadata: events[0].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[1].Message, StatusCode: 298, @@ -5289,7 +5290,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(_ transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(_ types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5307,9 +5308,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5319,7 +5320,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5329,7 +5330,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5339,15 +5340,15 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[1].Message, StatusCode: 200, Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 298, @@ -5362,7 +5363,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5373,9 +5374,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5385,7 +5386,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5395,7 +5396,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5405,8 +5406,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5425,7 +5426,7 @@ var _ = Describe("Static Function Tests", func() { }, FailedEvents: nil, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) It("Should allow all events when supportedMessageTypes is not an array", func() { @@ -5437,9 +5438,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5449,7 +5450,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5459,7 +5460,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5469,8 +5470,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5489,7 +5490,7 @@ var _ = Describe("Static Function Tests", func() { }, FailedEvents: nil, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5505,9 +5506,9 @@ var _ = Describe("Static Function Tests", func() { DestinationDefinition: backendconfig.DestinationDefinitionT{}, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5517,7 +5518,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5527,7 +5528,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5537,15 +5538,15 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[1].Message, StatusCode: 200, Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 298, @@ -5560,7 +5561,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5578,9 +5579,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", }, Message: map[string]interface{}{ @@ -5590,7 +5591,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5600,7 +5601,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", }, Message: map[string]interface{}{ @@ -5610,8 +5611,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5630,7 +5631,7 @@ var _ = Describe("Static Function Tests", func() { }, FailedEvents: nil, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5668,9 +5669,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -5681,7 +5682,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5692,7 +5693,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5703,15 +5704,15 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, Metadata: events[0].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[1].Message, StatusCode: 298, @@ -5726,7 +5727,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5764,9 +5765,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -5777,7 +5778,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5788,7 +5789,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5799,9 +5800,9 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ + expectedResponse := types.Response{ Events: nil, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 298, @@ -5822,7 +5823,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5860,9 +5861,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -5873,7 +5874,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5884,7 +5885,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5895,8 +5896,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5908,7 +5909,7 @@ var _ = Describe("Static Function Tests", func() { Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[2].Message, StatusCode: 298, @@ -5917,7 +5918,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -5950,9 +5951,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -5963,7 +5964,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5974,7 +5975,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -5985,8 +5986,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -5998,7 +5999,7 @@ var _ = Describe("Static Function Tests", func() { Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[2].Message, StatusCode: 298, @@ -6007,7 +6008,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -6043,9 +6044,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "web", }, @@ -6056,7 +6057,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -6067,7 +6068,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "web", }, @@ -6078,8 +6079,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -6091,7 +6092,7 @@ var _ = Describe("Static Function Tests", func() { Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[2].Message, StatusCode: 298, @@ -6100,7 +6101,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) @@ -6139,9 +6140,9 @@ var _ = Describe("Static Function Tests", func() { }, } - events := []transformer.TransformerEvent{ + events := []types.TransformerEvent{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-1", SourceDefinitionType: "", }, @@ -6152,7 +6153,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "", }, @@ -6163,7 +6164,7 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MessageID: "message-2", SourceDefinitionType: "", }, @@ -6174,8 +6175,8 @@ var _ = Describe("Static Function Tests", func() { Destination: destinationConfig, }, } - expectedResponse := transformer.Response{ - Events: []transformer.TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: events[0].Message, StatusCode: 200, @@ -6187,7 +6188,7 @@ var _ = Describe("Static Function Tests", func() { Metadata: events[1].Metadata, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: events[2].Message, StatusCode: 298, @@ -6196,7 +6197,7 @@ var _ = Describe("Static Function Tests", func() { }, }, } - response := ConvertToFilteredTransformerResponse(events, true, func(event transformer.TransformerEvent) (bool, string) { return false, "" }) + response := ConvertToFilteredTransformerResponse(events, true, func(event types.TransformerEvent) (bool, string) { return false, "" }) Expect(response).To(Equal(expectedResponse)) }) }) @@ -6275,7 +6276,7 @@ func assertJobStatus(job *jobsdb.JobT, status *jobsdb.JobStatusT, expectedState Expect(status.ExecTime).To(BeTemporally("~", time.Now(), 200*time.Millisecond)) } -func assertReportMetric(expectedMetric, actualMetric []*types.PUReportedMetric) { +func assertReportMetric(expectedMetric, actualMetric []*reportingTypes.PUReportedMetric) { sort.Slice(expectedMetric, func(i, j int) bool { return expectedMetric[i].ConnectionDetails.SourceID < expectedMetric[j].ConnectionDetails.SourceID }) @@ -6308,14 +6309,14 @@ func assertDestinationTransform( expectations transformExpectation, ) func( ctx context.Context, - clientEvents []transformer.TransformerEvent, + clientEvents []types.TransformerEvent, batchSize int, -) transformer.Response { +) types.Response { return func( ctx context.Context, - clientEvents []transformer.TransformerEvent, + clientEvents []types.TransformerEvent, batchSize int, - ) transformer.Response { + ) types.Response { defer GinkgoRecover() Expect(clientEvents).To(HaveLen(expectations.events)) @@ -6395,14 +6396,14 @@ func assertDestinationTransform( Expect(strings.Join(messageIDs, ",")).To(Equal(expectations.messageIds)) - return transformer.Response{ - Events: []transformer.TransformerResponse{ + return types.Response{ + Events: []types.TransformerResponse{ { Output: map[string]interface{}{ "int-value": 0, "string-value": fmt.Sprintf("value-%s", destinationID), }, - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ SourceID: "source-from-transformer", // transformer should replay source id SourceName: "source-from-transformer-name", DestinationID: "destination-from-transformer", // transformer should replay destination id @@ -6413,7 +6414,7 @@ func assertDestinationTransform( "int-value": 1, "string-value": fmt.Sprintf("value-%s", destinationID), }, - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ SourceID: "source-from-transformer", // transformer should replay source id SourceName: "source-from-transformer-name", DestinationID: "destination-from-transformer", // transformer should replay destination id @@ -6617,7 +6618,7 @@ var _ = Describe("TestConfigFilter", func() { Enabled: true, IsProcessorEnabled: true, } - expectedEvent := transformer.TransformerEvent{ + expectedEvent := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6636,7 +6637,7 @@ var _ = Describe("TestConfigFilter", func() { IsProcessorEnabled: true, }, } - event := transformer.TransformerEvent{ + event := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6668,7 +6669,7 @@ var _ = Describe("TestConfigFilter", func() { Expect(json.Unmarshal([]byte(intgConfigStr), &intgConfig)).To(BeNil()) Expect(json.Unmarshal([]byte(destDefStr), &destDef)).To(BeNil()) intgConfig.DestinationDefinition = destDef - expectedEvent := transformer.TransformerEvent{ + expectedEvent := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6687,7 +6688,7 @@ var _ = Describe("TestConfigFilter", func() { IsProcessorEnabled: true, }, } - event := transformer.TransformerEvent{ + event := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6714,7 +6715,7 @@ var _ = Describe("TestConfigFilter", func() { Enabled: true, IsProcessorEnabled: true, } - expectedEvent := transformer.TransformerEvent{ + expectedEvent := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6735,7 +6736,7 @@ var _ = Describe("TestConfigFilter", func() { IsProcessorEnabled: true, }, } - event := transformer.TransformerEvent{ + event := types.TransformerEvent{ Message: types.SingularEventT{ "MessageID": "messageId-1", }, @@ -6749,12 +6750,12 @@ var _ = Describe("TestConfigFilter", func() { func Test_GetTimestampFromEvent(t *testing.T) { input := []struct { - event transformer.TransformerEvent + event types.TransformerEvent timestamp time.Time expectedTimeStamp time.Time }{ { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{ "timestamp": "2021-06-09T09:00:00.000Z", }, @@ -6763,7 +6764,7 @@ func Test_GetTimestampFromEvent(t *testing.T) { expectedTimeStamp: time.Date(2021, 6, 9, 9, 0, 0, 0, time.UTC), }, { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{}, }, timestamp: time.Now(), @@ -6781,7 +6782,7 @@ func Test_GetTimestampFromEvent(t *testing.T) { func Test_EnhanceWithTimeFields(t *testing.T) { input := []struct { - event transformer.TransformerEvent + event types.TransformerEvent singularEvent types.SingularEventT recievedAt time.Time expectedSentAt string @@ -6789,7 +6790,7 @@ func Test_EnhanceWithTimeFields(t *testing.T) { expectedOriginalTimestamp string }{ { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{ "timestamp": "2021-06-09T09:00:00.000Z", }, @@ -6804,7 +6805,7 @@ func Test_EnhanceWithTimeFields(t *testing.T) { expectedOriginalTimestamp: "2021-06-09T09:00:00.000Z", }, { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{ "timestamp": "2021-06-09T09:00:00.000Z", }, @@ -6816,7 +6817,7 @@ func Test_EnhanceWithTimeFields(t *testing.T) { expectedOriginalTimestamp: "2021-06-09T09:00:00.000Z", }, { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{}, }, singularEvent: types.SingularEventT{}, @@ -6826,7 +6827,7 @@ func Test_EnhanceWithTimeFields(t *testing.T) { expectedOriginalTimestamp: "2021-06-09T09:00:00.000Z", }, { - event: transformer.TransformerEvent{ + event: types.TransformerEvent{ Message: types.SingularEventT{}, }, singularEvent: types.SingularEventT{ @@ -6860,7 +6861,7 @@ func TestStoreMessageMerge(t *testing.T) { }, procErrorJobs: []*jobsdb.JobT{{JobID: 1}}, routerDestIDs: []string{"1"}, - reportMetrics: []*types.PUReportedMetric{{}}, + reportMetrics: []*reportingTypes.PUReportedMetric{{}}, sourceDupStats: map[dupStatKey]int{{sourceID: "1"}: 1}, dedupKeys: map[string]struct{}{"1": {}}, totalEvents: 1, @@ -6876,7 +6877,7 @@ func TestStoreMessageMerge(t *testing.T) { }, procErrorJobs: []*jobsdb.JobT{{JobID: 2}}, routerDestIDs: []string{"2"}, - reportMetrics: []*types.PUReportedMetric{{}}, + reportMetrics: []*reportingTypes.PUReportedMetric{{}}, sourceDupStats: map[dupStatKey]int{{sourceID: "1"}: 2}, dedupKeys: map[string]struct{}{"2": {}}, totalEvents: 1, @@ -6894,7 +6895,7 @@ func TestStoreMessageMerge(t *testing.T) { }, []*jobsdb.JobT{{JobID: 3}}, []string{"3"}, - []*types.PUReportedMetric{{}}, + []*reportingTypes.PUReportedMetric{{}}, map[dupStatKey]int{{sourceID: "1"}: 3}, map[string]struct{}{"3": {}}, 1, @@ -6958,7 +6959,7 @@ func TestStoreMessageMerge(t *testing.T) { }, procErrorJobs: []*jobsdb.JobT{{JobID: 1}, {JobID: 2}, {JobID: 3}}, routerDestIDs: []string{"1", "2", "3"}, - reportMetrics: []*types.PUReportedMetric{{}, {}, {}}, + reportMetrics: []*reportingTypes.PUReportedMetric{{}, {}, {}}, sourceDupStats: map[dupStatKey]int{{sourceID: "1"}: 6}, dedupKeys: map[string]struct{}{"1": {}, "2": {}, "3": {}}, totalEvents: 3, diff --git a/processor/trackingplan.go b/processor/trackingplan.go index a085517b6f..0322277e6a 100644 --- a/processor/trackingplan.go +++ b/processor/trackingplan.go @@ -7,11 +7,12 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/stats" + backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" ) type TrackingPlanStatT struct { @@ -24,7 +25,7 @@ type TrackingPlanStatT struct { // reportViolations It is going add violationErrors in context depending upon certain criteria: // 1. sourceSchemaConfig in Metadata.MergedTpConfig should be true -func reportViolations(validateEvent *transformer.TransformerResponse, trackingPlanId string, trackingPlanVersion int) { +func reportViolations(validateEvent *types.TransformerResponse, trackingPlanId string, trackingPlanVersion int) { if validateEvent.Metadata.MergedTpConfig["propagateValidationErrors"] == "false" { return } @@ -52,7 +53,7 @@ func reportViolations(validateEvent *transformer.TransformerResponse, trackingPl // enhanceWithViolation It enhances extra information of ValidationErrors in context for: // 1. response.Events // 1. response.FailedEvents -func enhanceWithViolation(response transformer.Response, trackingPlanId string, trackingPlanVersion int) { +func enhanceWithViolation(response types.Response, trackingPlanId string, trackingPlanVersion int) { for i := range response.Events { validatedEvent := &response.Events[i] reportViolations(validatedEvent, trackingPlanId, trackingPlanVersion) @@ -68,9 +69,9 @@ func enhanceWithViolation(response transformer.Response, trackingPlanId string, // The Response will contain both the Events and FailedEvents // 1. eventsToTransform gets added to validatedEventsBySourceId // 2. failedJobs gets added to validatedErrorJobs -func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]transformer.TransformerEvent, eventsByMessageID map[string]types.SingularEventWithReceivedAt) (map[SourceIDT][]transformer.TransformerEvent, []*types.PUReportedMetric, []*jobsdb.JobT, map[SourceIDT]bool) { - validatedEventsBySourceId := make(map[SourceIDT][]transformer.TransformerEvent) - validatedReportMetrics := make([]*types.PUReportedMetric, 0) +func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]types.TransformerEvent, eventsByMessageID map[string]types.SingularEventWithReceivedAt) (map[SourceIDT][]types.TransformerEvent, []*reportingTypes.PUReportedMetric, []*jobsdb.JobT, map[SourceIDT]bool) { + validatedEventsBySourceId := make(map[SourceIDT][]types.TransformerEvent) + validatedReportMetrics := make([]*reportingTypes.PUReportedMetric, 0) validatedErrorJobs := make([]*jobsdb.JobT, 0) trackingPlanEnabledMap := make(map[SourceIDT]bool) @@ -84,7 +85,7 @@ func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]trans isTpExists := eventList[0].Metadata.TrackingPlanID != "" if !isTpExists { // pass on the jobs for transformation(User, Dest) - validatedEventsBySourceId[sourceId] = make([]transformer.TransformerEvent, 0) + validatedEventsBySourceId[sourceId] = make([]types.TransformerEvent, 0) validatedEventsBySourceId[sourceId] = append(validatedEventsBySourceId[sourceId], eventList...) continue } @@ -101,7 +102,7 @@ func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]trans // This is a safety check we are adding so that if something unexpected comes from transformer // We are ignoring it. if (len(response.Events) + len(response.FailedEvents)) != len(eventList) { - validatedEventsBySourceId[sourceId] = make([]transformer.TransformerEvent, 0) + validatedEventsBySourceId[sourceId] = make([]types.TransformerEvent, 0) validatedEventsBySourceId[sourceId] = append(validatedEventsBySourceId[sourceId], eventList...) continue } @@ -111,9 +112,9 @@ func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]trans // This is being used to distinguish the flows in reporting service trackingPlanEnabledMap[sourceId] = true - var successMetrics []*types.PUReportedMetric - eventsToTransform, successMetrics, _, _ := proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, backendconfig.Connection{}, types.DESTINATION_FILTER, types.TRACKINGPLAN_VALIDATOR) // Note: Sending false for usertransformation enabled is safe because this stage is before user transformation. - nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, types.DESTINATION_FILTER, types.TRACKINGPLAN_VALIDATOR) + var successMetrics []*reportingTypes.PUReportedMetric + eventsToTransform, successMetrics, _, _ := proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, backendconfig.Connection{}, reportingTypes.DESTINATION_FILTER, reportingTypes.TRACKINGPLAN_VALIDATOR) // Note: Sending false for usertransformation enabled is safe because this stage is before user transformation. + nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, reportingTypes.DESTINATION_FILTER, reportingTypes.TRACKINGPLAN_VALIDATOR) validationStat.numValidationSuccessEvents.Count(len(eventsToTransform)) validationStat.numValidationFailedEvents.Count(len(nonSuccessMetrics.failedJobs)) @@ -134,16 +135,16 @@ func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]trans if len(eventsToTransform) == 0 { continue } - validatedEventsBySourceId[sourceId] = make([]transformer.TransformerEvent, 0) + validatedEventsBySourceId[sourceId] = make([]types.TransformerEvent, 0) validatedEventsBySourceId[sourceId] = append(validatedEventsBySourceId[sourceId], eventsToTransform...) } return validatedEventsBySourceId, validatedReportMetrics, validatedErrorJobs, trackingPlanEnabledMap } // makeCommonMetadataFromTransformerEvent Creates a new Metadata instance -func makeCommonMetadataFromTransformerEvent(transformerEvent *transformer.TransformerEvent) *transformer.Metadata { +func makeCommonMetadataFromTransformerEvent(transformerEvent *types.TransformerEvent) *types.Metadata { metadata := transformerEvent.Metadata - commonMetaData := transformer.Metadata{ + commonMetaData := types.Metadata{ SourceID: metadata.SourceID, SourceName: metadata.SourceName, SourceType: metadata.SourceType, @@ -159,7 +160,7 @@ func makeCommonMetadataFromTransformerEvent(transformerEvent *transformer.Transf } // newValidationStat Creates a new TrackingPlanStatT instance -func (proc *Handle) newValidationStat(metadata *transformer.Metadata) *TrackingPlanStatT { +func (proc *Handle) newValidationStat(metadata *types.Metadata) *TrackingPlanStatT { tags := map[string]string{ "destination": metadata.DestinationID, "destType": metadata.DestinationType, diff --git a/processor/trackingplan_test.go b/processor/trackingplan_test.go index 88366df505..6ced145fac 100644 --- a/processor/trackingplan_test.go +++ b/processor/trackingplan_test.go @@ -3,13 +3,13 @@ package processor import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/rudderlabs/rudder-server/processor/types" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/stretchr/testify/assert" ) func TestReportViolations(t *testing.T) { - eventsFromTransformerResponse := func(response *transformer.Response) (events []transformer.TransformerResponse) { + eventsFromTransformerResponse := func(response *types.Response) (events []types.TransformerResponse) { events = append(events, response.Events...) events = append(events, response.FailedEvents...) return @@ -21,10 +21,10 @@ func TestReportViolations(t *testing.T) { trackingPlanVersion int ) - response := transformer.Response{ - Events: []transformer.TransformerResponse{ + response := types.Response{ + Events: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "false", }, @@ -34,9 +34,9 @@ func TestReportViolations(t *testing.T) { }, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "false", }, @@ -64,10 +64,10 @@ func TestReportViolations(t *testing.T) { trackingPlanVersion int ) - response := transformer.Response{ - Events: []transformer.TransformerResponse{ + response := types.Response{ + Events: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "false", }, @@ -77,9 +77,9 @@ func TestReportViolations(t *testing.T) { }, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, @@ -87,7 +87,7 @@ func TestReportViolations(t *testing.T) { Output: map[string]interface{}{ "context": 1234, }, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ @@ -109,10 +109,10 @@ func TestReportViolations(t *testing.T) { }) t.Run("Propagate validation errors", func(t *testing.T) { - response := transformer.Response{ - Events: []transformer.TransformerResponse{ + response := types.Response{ + Events: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, @@ -120,7 +120,7 @@ func TestReportViolations(t *testing.T) { Output: map[string]interface{}{ "context": map[string]interface{}{}, }, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ @@ -132,7 +132,7 @@ func TestReportViolations(t *testing.T) { }, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, @@ -140,7 +140,7 @@ func TestReportViolations(t *testing.T) { Output: map[string]interface{}{ "context": nil, }, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ @@ -152,9 +152,9 @@ func TestReportViolations(t *testing.T) { }, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, @@ -162,7 +162,7 @@ func TestReportViolations(t *testing.T) { Output: map[string]interface{}{ "context": map[string]interface{}{}, }, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ @@ -174,13 +174,13 @@ func TestReportViolations(t *testing.T) { }, }, { - Metadata: transformer.Metadata{ + Metadata: types.Metadata{ MergedTpConfig: map[string]interface{}{ "propagateValidationErrors": "true", }, }, Output: map[string]interface{}{}, - ValidationErrors: []transformer.ValidationError{ + ValidationErrors: []types.ValidationError{ { Type: "Datatype-Mismatch", Meta: map[string]string{ diff --git a/processor/transformer/manager.go b/processor/transformer/manager.go index 8ef742f42a..1bab858721 100644 --- a/processor/transformer/manager.go +++ b/processor/transformer/manager.go @@ -1,9 +1,14 @@ package transformer -import "errors" +import ( + "context" + "errors" + + "github.com/rudderlabs/rudder-server/processor/types" +) type ServiceClient interface { - SendRequest(data interface{}) (response interface{}, err error) + SendRequest(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response } type CommunicationManager struct { diff --git a/processor/transformer/transformer.go b/processor/transformer/transformer.go index 57d51d977f..5208f645e7 100644 --- a/processor/transformer/transformer.go +++ b/processor/transformer/transformer.go @@ -16,6 +16,8 @@ import ( "sync" "time" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" + "github.com/bufbuild/httplb" "github.com/bufbuild/httplb/resolver" "github.com/cenkalti/backoff" @@ -26,11 +28,10 @@ import ( "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" - backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/processor/integrations" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/httputil" "github.com/rudderlabs/rudder-server/utils/sysUtils" - "github.com/rudderlabs/rudder-server/utils/types" warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) @@ -48,66 +49,6 @@ const ( var json = jsoniter.ConfigCompatibleWithStandardLibrary -type Metadata struct { - SourceID string `json:"sourceId"` - SourceName string `json:"sourceName"` - OriginalSourceID string `json:"originalSourceId"` - WorkspaceID string `json:"workspaceId"` - Namespace string `json:"namespace"` - InstanceID string `json:"instanceId"` - SourceType string `json:"sourceType"` - SourceCategory string `json:"sourceCategory"` - TrackingPlanID string `json:"trackingPlanId"` - TrackingPlanVersion int `json:"trackingPlanVersion"` - SourceTpConfig map[string]map[string]interface{} `json:"sourceTpConfig"` - MergedTpConfig map[string]interface{} `json:"mergedTpConfig"` - DestinationID string `json:"destinationId"` - JobID int64 `json:"jobId"` - SourceJobID string `json:"sourceJobId"` - SourceJobRunID string `json:"sourceJobRunId"` - SourceTaskRunID string `json:"sourceTaskRunId"` - RecordID interface{} `json:"recordId"` - DestinationType string `json:"destinationType"` - DestinationName string `json:"destinationName"` - MessageID string `json:"messageId"` - OAuthAccessToken string `json:"oauthAccessToken"` - TraceParent string `json:"traceparent"` - // set by user_transformer to indicate transformed event is part of group indicated by messageIDs - MessageIDs []string `json:"messageIds"` - RudderID string `json:"rudderId"` - ReceivedAt string `json:"receivedAt"` - EventName string `json:"eventName"` - EventType string `json:"eventType"` - SourceDefinitionID string `json:"sourceDefinitionId"` - DestinationDefinitionID string `json:"destinationDefinitionId"` - TransformationID string `json:"transformationId"` - TransformationVersionID string `json:"transformationVersionId"` - SourceDefinitionType string `json:"-"` -} - -func (m Metadata) GetMessagesIDs() []string { - if len(m.MessageIDs) > 0 { - return m.MessageIDs - } - return []string{m.MessageID} -} - -type TransformerEvent struct { - Message types.SingularEventT `json:"message"` - Metadata Metadata `json:"metadata"` - Destination backendconfig.DestinationT `json:"destination"` - Connection backendconfig.Connection `json:"connection"` - Libraries []backendconfig.LibraryT `json:"libraries"` - Credentials []Credential `json:"credentials"` -} - -type Credential struct { - ID string `json:"id"` - Key string `json:"key"` - Value string `json:"value"` - IsSecret bool `json:"isSecret"` -} - func isJobTerminated(status int) bool { if status == http.StatusTooManyRequests || status == http.StatusRequestTimeout { return false @@ -115,29 +56,6 @@ func isJobTerminated(status int) bool { return status >= http.StatusOK && status < http.StatusInternalServerError } -type TransformerResponse struct { - // Not marking this Singular Event, since this not a RudderEvent - Output map[string]interface{} `json:"output"` - Metadata Metadata `json:"metadata"` - StatusCode int `json:"statusCode"` - Error string `json:"error"` - ValidationErrors []ValidationError `json:"validationErrors"` - StatTags map[string]string `json:"statTags"` -} - -type ValidationError struct { - Type string `json:"type"` - Message string `json:"message"` - Meta map[string]string `json:"meta"` - Property string `json:"property"` -} - -// Response represents a Transformer response -type Response struct { - Events []TransformerResponse - FailedEvents []TransformerResponse -} - type Opt func(*handle) func WithClient(client HTTPDoer) Opt { @@ -148,9 +66,9 @@ func WithClient(client HTTPDoer) Opt { // Transformer provides methods to transform events type Transformer interface { - Transform(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response - UserTransform(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response - Validate(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response + Transform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response + UserTransform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response + Validate(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response } type HTTPDoer interface { @@ -264,17 +182,17 @@ func NewTransformer(conf *config.Config, log logger.Logger, stat stats.Stats, op } // Transform function is used to invoke destination transformer API -func (trans *handle) Transform(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response { +func (trans *handle) Transform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { return trans.transform(ctx, clientEvents, trans.destTransformURL(clientEvents[0].Destination.DestinationDefinition.Name), batchSize, destTransformerStage) } // UserTransform function is used to invoke user transformer API -func (trans *handle) UserTransform(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response { +func (trans *handle) UserTransform(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { return trans.transform(ctx, clientEvents, trans.userTransformURL(), batchSize, userTransformerStage) } // Validate function is used to invoke tracking plan validation API -func (trans *handle) Validate(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response { +func (trans *handle) Validate(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { return trans.transform(ctx, clientEvents, trans.trackingPlanValidationURL(), batchSize, trackingPlanValidationStage) } @@ -288,13 +206,13 @@ func (t *HTTPLBTransport) NewRoundTripper(scheme, target string, config httplb.T func (trans *handle) transform( ctx context.Context, - clientEvents []TransformerEvent, + clientEvents []types.TransformerEvent, url string, batchSize int, stage string, -) Response { +) types.Response { if len(clientEvents) == 0 { - return Response{} + return types.Response{} } // flip sourceID and originalSourceID if it's a replay source for the purpose of any user transformation // flip back afterwards @@ -334,14 +252,14 @@ func (trans *handle) transform( ).Observe(float64(len(batches))) trace.Logf(ctx, "request", "batch_count: %d", len(batches)) - transformResponse := make([][]TransformerResponse, len(batches)) + transformResponse := make([][]types.TransformerResponse, len(batches)) var wg sync.WaitGroup wg.Add(len(batches)) lo.ForEach( batches, - func(batch []TransformerEvent, i int) { + func(batch []types.TransformerEvent, i int) { trans.guardConcurrency <- struct{}{} go func() { trace.WithRegion(ctx, "request", func() { @@ -354,8 +272,8 @@ func (trans *handle) transform( ) wg.Wait() - var outClientEvents []TransformerResponse - var failedEvents []TransformerResponse + var outClientEvents []types.TransformerResponse + var failedEvents []types.TransformerResponse for _, batch := range transformResponse { // Transform is one to many mapping so returned @@ -376,13 +294,13 @@ func (trans *handle) transform( trans.sentStat.Count(len(clientEvents)) trans.receivedStat.Count(len(outClientEvents)) - return Response{ + return types.Response{ Events: outClientEvents, FailedEvents: failedEvents, } } -func (trans *handle) request(ctx context.Context, url, stage string, data []TransformerEvent) []TransformerResponse { +func (trans *handle) request(ctx context.Context, url, stage string, data []types.TransformerEvent) []types.TransformerResponse { // Call remote transformation var ( rawJSON []byte @@ -462,7 +380,7 @@ func (trans *handle) request(ctx context.Context, url, stage string, data []Tran trans.logger.Errorf("Transformer returned status code: %v", statusCode) } - var transformerResponses []TransformerResponse + var transformerResponses []types.TransformerResponse switch statusCode { case http.StatusOK: integrations.CollectIntgTransformErrorStats(respData) @@ -481,7 +399,7 @@ func (trans *handle) request(ctx context.Context, url, stage string, data []Tran default: for i := range data { transformEvent := &data[i] - resp := TransformerResponse{StatusCode: statusCode, Error: string(respData), Metadata: transformEvent.Metadata} + resp := types.TransformerResponse{StatusCode: statusCode, Error: string(respData), Metadata: transformEvent.Metadata} transformerResponses = append(transformerResponses, resp) } } @@ -554,8 +472,8 @@ func (trans *handle) doPost(ctx context.Context, rawJSON []byte, url, stage stri // perform version compatibility check only on success if resp.StatusCode == http.StatusOK { transformerAPIVersion, _ := strconv.Atoi(resp.Header.Get("apiVersion")) - if types.SupportedTransformerApiVersion != transformerAPIVersion { - unexpectedVersionError := fmt.Errorf("incompatible transformer version: Expected: %d Received: %s, URL: %v", types.SupportedTransformerApiVersion, resp.Header.Get("apiVersion"), url) + if reportingTypes.SupportedTransformerApiVersion != transformerAPIVersion { + unexpectedVersionError := fmt.Errorf("incompatible transformer version: Expected: %d Received: %s, URL: %v", reportingTypes.SupportedTransformerApiVersion, resp.Header.Get("apiVersion"), url) trans.logger.Error(unexpectedVersionError) panic(unexpectedVersionError) } diff --git a/processor/transformer/transformer_test.go b/processor/transformer/transformer_test.go index a920831a99..47007fefc2 100644 --- a/processor/transformer/transformer_test.go +++ b/processor/transformer/transformer_test.go @@ -13,46 +13,45 @@ import ( "testing" "time" - "go.uber.org/mock/gomock" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" - "github.com/rudderlabs/rudder-go-kit/testhelper/rand" - "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" - warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" - - "github.com/rudderlabs/rudder-server/utils/types" + "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" + "github.com/rudderlabs/rudder-go-kit/config" + "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/logger/mock_logger" "github.com/rudderlabs/rudder-go-kit/stats" "github.com/rudderlabs/rudder-go-kit/stats/memstats" + "github.com/rudderlabs/rudder-go-kit/testhelper/rand" backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/gateway/response" - - "github.com/rudderlabs/rudder-go-kit/config" - "github.com/rudderlabs/rudder-go-kit/logger" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/rudderlabs/rudder-server/testhelper/backendconfigtest" + warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) type fakeTransformer struct { - requests [][]TransformerEvent + requests [][]types.TransformerEvent t testing.TB } func (t *fakeTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var reqBody []TransformerEvent + var reqBody []types.TransformerEvent require.NoError(t.t, json.NewDecoder(r.Body).Decode(&reqBody)) t.requests = append(t.requests, reqBody) - responses := make([]TransformerResponse, len(reqBody)) + responses := make([]types.TransformerResponse, len(reqBody)) for i := range reqBody { statusCode := int(reqBody[i].Message["forceStatusCode"].(float64)) delete(reqBody[i].Message, "forceStatusCode") reqBody[i].Message["echo-key-1"] = reqBody[i].Message["src-key-1"] - responses[i] = TransformerResponse{ + responses[i] = types.TransformerResponse{ Output: reqBody[i].Message, Metadata: reqBody[i].Metadata, StatusCode: statusCode, @@ -62,7 +61,7 @@ func (t *fakeTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } - w.Header().Set("apiVersion", strconv.Itoa(types.SupportedTransformerApiVersion)) + w.Header().Set("apiVersion", strconv.Itoa(reportingTypes.SupportedTransformerApiVersion)) require.NoError(t.t, json.NewEncoder(w).Encode(responses)) } @@ -83,10 +82,10 @@ type endlessLoopTransformer struct { func (elt *endlessLoopTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) { elt.retryCount++ - var reqBody []TransformerEvent + var reqBody []types.TransformerEvent require.NoError(elt.t, json.NewDecoder(r.Body).Decode(&reqBody)) - responses := make([]TransformerResponse, len(reqBody)) + responses := make([]types.TransformerResponse, len(reqBody)) if elt.retryCount < elt.maxRetryCount { http.Error(w, response.MakeResponse(elt.statusError), elt.statusCode) @@ -94,7 +93,7 @@ func (elt *endlessLoopTransformer) ServeHTTP(w http.ResponseWriter, r *http.Requ } for i := range reqBody { - responses[i] = TransformerResponse{ + responses[i] = types.TransformerResponse{ Output: reqBody[i].Message, Metadata: reqBody[i].Metadata, StatusCode: http.StatusOK, @@ -122,13 +121,13 @@ func (et *endpointTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) return } - var reqBody []TransformerEvent + var reqBody []types.TransformerEvent require.NoError(et.t, json.NewDecoder(r.Body).Decode(&reqBody)) - responses := make([]TransformerResponse, len(reqBody)) + responses := make([]types.TransformerResponse, len(reqBody)) for i := range reqBody { - responses[i] = TransformerResponse{ + responses[i] = types.TransformerResponse{ Output: reqBody[i].Message, Metadata: reqBody[i].Metadata, StatusCode: http.StatusOK, @@ -136,7 +135,7 @@ func (et *endpointTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) } } - w.Header().Set("apiVersion", strconv.Itoa(types.SupportedTransformerApiVersion)) + w.Header().Set("apiVersion", strconv.Itoa(reportingTypes.SupportedTransformerApiVersion)) require.NoError(et.t, json.NewEncoder(w).Encode(responses)) } @@ -194,15 +193,15 @@ func TestTransformer(t *testing.T) { eventsCount := tt.eventsCount failEvery := tt.failEvery - events := make([]TransformerEvent, eventsCount) - expectedResponse := Response{} + events := make([]types.TransformerEvent, eventsCount) + expectedResponse := types.Response{} transformationID := rand.String(10) destinationConfig := backendconfigtest.NewDestinationBuilder("WEBHOOK"). WithUserTransformation(transformationID, rand.String(10)).Build() - metadata := Metadata{ + Metadata := types.Metadata{ DestinationType: destinationConfig.DestinationDefinition.Name, SourceID: rand.String(10), DestinationID: destinationConfig.ID, @@ -217,17 +216,17 @@ func TestTransformer(t *testing.T) { statusCode = http.StatusBadRequest } - metadata := metadata - metadata.MessageID = msgID + Metadata := Metadata + Metadata.MessageID = msgID - events[i] = TransformerEvent{ - Metadata: metadata, + events[i] = types.TransformerEvent{ + Metadata: Metadata, Message: map[string]interface{}{ "src-key-1": msgID, "forceStatusCode": statusCode, }, Destination: destinationConfig, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -237,8 +236,8 @@ func TestTransformer(t *testing.T) { }, } - tResp := TransformerResponse{ - Metadata: metadata, + tResp := types.TransformerResponse{ + Metadata: Metadata, StatusCode: statusCode, Output: map[string]interface{}{ "src-key-1": msgID, @@ -263,7 +262,7 @@ func TestTransformer(t *testing.T) { for _, m := range metrics { require.Equal(t, stats.Tags{ "stage": "test-stage", - "sourceId": metadata.SourceID, + "sourceId": Metadata.SourceID, "destinationType": destinationConfig.DestinationDefinition.Name, "destinationId": destinationConfig.ID, "transformationId": destinationConfig.Transformations[0].ID, @@ -271,7 +270,7 @@ func TestTransformer(t *testing.T) { // Legacy tags: to be removed "dest_type": destinationConfig.DestinationDefinition.Name, "dest_id": destinationConfig.ID, - "src_id": metadata.SourceID, + "src_id": Metadata.SourceID, }, m.Tags) } } @@ -280,14 +279,14 @@ func TestTransformer(t *testing.T) { t.Run("timeout", func(t *testing.T) { msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ "src-key-1": msgID, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -303,7 +302,7 @@ func TestTransformer(t *testing.T) { expectedRetries int expectPanic bool stage string - expectedResponse []TransformerResponse + expectedResponse []types.TransformerResponse failOnUserTransformTimeout bool }{ { @@ -318,9 +317,9 @@ func TestTransformer(t *testing.T) { retries: 3, stage: userTransformerStage, expectPanic: false, - expectedResponse: []TransformerResponse{ + expectedResponse: []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: TransformerRequestTimeout, @@ -385,7 +384,7 @@ func TestTransformer(t *testing.T) { rsp := tr.request(context.TODO(), srv.URL, tc.stage, events) require.Len(t, rsp, 1) require.Equal(t, rsp[0].StatusCode, TransformerRequestTimeout) - require.Equal(t, rsp[0].Metadata, Metadata{ + require.Equal(t, rsp[0].Metadata, types.Metadata{ MessageID: msgID, }) require.Contains(t, rsp[0].Error, "transformer request timed out:") @@ -396,14 +395,14 @@ func TestTransformer(t *testing.T) { t.Run("endless retries in case of control plane down", func(t *testing.T) { msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ "src-key-1": msgID, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -417,7 +416,7 @@ func TestTransformer(t *testing.T) { maxRetryCount: 3, statusCode: StatusCPDown, statusError: "control plane not reachable", - apiVersion: types.SupportedTransformerApiVersion, + apiVersion: reportingTypes.SupportedTransformerApiVersion, t: t, } @@ -436,9 +435,9 @@ func TestTransformer(t *testing.T) { tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) rsp := tr.request(context.TODO(), srv.URL, "test-stage", events) - require.Equal(t, rsp, []TransformerResponse{ + require.Equal(t, rsp, []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: http.StatusOK, @@ -452,8 +451,8 @@ func TestTransformer(t *testing.T) { t.Run("retries", func(t *testing.T) { msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ @@ -467,7 +466,7 @@ func TestTransformer(t *testing.T) { }, }, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -485,7 +484,7 @@ func TestTransformer(t *testing.T) { statusError string expectedRetries int expectPanic bool - expectedResponse []TransformerResponse + expectedResponse []types.TransformerResponse failOnError bool }{ { @@ -506,9 +505,9 @@ func TestTransformer(t *testing.T) { statusError: "too many requests", expectedRetries: 4, expectPanic: false, - expectedResponse: []TransformerResponse{ + expectedResponse: []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: TransformerRequestFailure, @@ -525,9 +524,9 @@ func TestTransformer(t *testing.T) { statusError: "control plane not reachable", expectedRetries: 3, expectPanic: false, - expectedResponse: []TransformerResponse{ + expectedResponse: []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: http.StatusOK, @@ -548,7 +547,7 @@ func TestTransformer(t *testing.T) { maxRetryCount: tc.maxRetryCount, statusCode: tc.statusCode, statusError: tc.statusError, - apiVersion: types.SupportedTransformerApiVersion, + apiVersion: reportingTypes.SupportedTransformerApiVersion, t: t, } @@ -584,8 +583,8 @@ func TestTransformer(t *testing.T) { t.Run("version compatibility", func(t *testing.T) { msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ @@ -599,7 +598,7 @@ func TestTransformer(t *testing.T) { }, }, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -614,15 +613,15 @@ func TestTransformer(t *testing.T) { apiVersion int skipApiVersion bool expectPanic bool - expectedResponse []TransformerResponse + expectedResponse []types.TransformerResponse }{ { name: "compatible api version", - apiVersion: types.SupportedTransformerApiVersion, + apiVersion: reportingTypes.SupportedTransformerApiVersion, expectPanic: false, - expectedResponse: []TransformerResponse{ + expectedResponse: []types.TransformerResponse{ { - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: http.StatusOK, @@ -683,21 +682,21 @@ func TestTransformer(t *testing.T) { t.Run("endpoints", func(t *testing.T) { msgID := "messageID-0" - expectedResponse := Response{ - Events: []TransformerResponse{ + expectedResponse := types.Response{ + Events: []types.TransformerResponse{ { Output: map[string]interface{}{ "src-key-1": msgID, }, - Metadata: Metadata{ + Metadata: types.Metadata{ MessageID: msgID, }, StatusCode: http.StatusOK, }, }, } - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ @@ -714,7 +713,7 @@ func TestTransformer(t *testing.T) { }, }, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", @@ -779,8 +778,8 @@ func TestTransformer(t *testing.T) { tr := NewTransformer(c, logger.NOP, stats.Default, WithClient(srv.Client())) - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ + events := append([]types.TransformerEvent{}, types.TransformerEvent{ + Metadata: types.Metadata{ MessageID: msgID, }, Message: map[string]interface{}{ @@ -797,7 +796,7 @@ func TestTransformer(t *testing.T) { }, }, }, - Credentials: []Credential{ + Credentials: []types.Credential{ { ID: "test-credential", Key: "test-key", diff --git a/processor/types/types.go b/processor/types/types.go new file mode 100644 index 0000000000..e73d95aace --- /dev/null +++ b/processor/types/types.go @@ -0,0 +1,122 @@ +package types + +import ( + "time" + + backendconfig "github.com/rudderlabs/rudder-server/backend-config" +) + +// SingularEventT single event structure +type SingularEventT map[string]interface{} + +// GetRudderEventVal returns the value corresponding to the key in the message structure +func GetRudderEventVal(key string, rudderEvent SingularEventT) (interface{}, bool) { + rudderVal, ok := rudderEvent[key] + if !ok { + return nil, false + } + return rudderVal, true +} + +type SingularEventWithReceivedAt struct { + SingularEvent SingularEventT + ReceivedAt time.Time +} + +// GatewayBatchRequest batch request structure +type GatewayBatchRequest struct { + Batch []SingularEventT `json:"batch"` + RequestIP string `json:"requestIP"` + ReceivedAt time.Time `json:"receivedAt"` +} + +type TransformerEvent struct { + Message SingularEventT `json:"message"` + Metadata Metadata `json:"metadata"` + Destination backendconfig.DestinationT `json:"destination"` + Connection backendconfig.Connection `json:"connection"` + Libraries []backendconfig.LibraryT `json:"libraries"` + Credentials []Credential `json:"credentials"` +} + +type Credential struct { + ID string `json:"id"` + Key string `json:"key"` + Value string `json:"value"` + IsSecret bool `json:"isSecret"` +} + +type Metadata struct { + SourceID string `json:"sourceId"` + SourceName string `json:"sourceName"` + OriginalSourceID string `json:"originalSourceId"` + WorkspaceID string `json:"workspaceId"` + Namespace string `json:"namespace"` + InstanceID string `json:"instanceId"` + SourceType string `json:"sourceType"` + SourceCategory string `json:"sourceCategory"` + TrackingPlanID string `json:"trackingPlanId"` + TrackingPlanVersion int `json:"trackingPlanVersion"` + SourceTpConfig map[string]map[string]interface{} `json:"sourceTpConfig"` + MergedTpConfig map[string]interface{} `json:"mergedTpConfig"` + DestinationID string `json:"destinationId"` + JobID int64 `json:"jobId"` + SourceJobID string `json:"sourceJobId"` + SourceJobRunID string `json:"sourceJobRunId"` + SourceTaskRunID string `json:"sourceTaskRunId"` + RecordID interface{} `json:"recordId"` + DestinationType string `json:"destinationType"` + DestinationName string `json:"destinationName"` + MessageID string `json:"messageId"` + OAuthAccessToken string `json:"oauthAccessToken"` + TraceParent string `json:"traceparent"` + // set by user_transformer to indicate transformed event is part of group indicated by messageIDs + MessageIDs []string `json:"messageIds"` + RudderID string `json:"rudderId"` + ReceivedAt string `json:"receivedAt"` + EventName string `json:"eventName"` + EventType string `json:"eventType"` + SourceDefinitionID string `json:"sourceDefinitionId"` + DestinationDefinitionID string `json:"destinationDefinitionId"` + TransformationID string `json:"transformationId"` + TransformationVersionID string `json:"transformationVersionId"` + SourceDefinitionType string `json:"-"` +} + +func (m Metadata) GetMessagesIDs() []string { + if len(m.MessageIDs) > 0 { + return m.MessageIDs + } + return []string{m.MessageID} +} + +type TransformerResponse struct { + // Not marking this Singular Event, since this not a RudderEvent + Output map[string]interface{} `json:"output"` + Metadata Metadata `json:"metadata"` + StatusCode int `json:"statusCode"` + Error string `json:"error"` + ValidationErrors []ValidationError `json:"validationErrors"` + StatTags map[string]string `json:"statTags"` +} + +type ValidationError struct { + Type string `json:"type"` + Message string `json:"message"` + Meta map[string]string `json:"meta"` + Property string `json:"property"` +} + +// Response represents a Transformer response +type Response struct { + Events []TransformerResponse + FailedEvents []TransformerResponse +} + +type EventParams struct { + SourceJobRunId string `json:"source_job_run_id"` + SourceId string `json:"source_id"` + SourceTaskRunId string `json:"source_task_run_id"` + TraceParent string `json:"traceparent"` + DestinationID string `json:"destination_id"` +} diff --git a/services/debugger/transformation/transformationStatusUploader.go b/services/debugger/transformation/transformationStatusUploader.go index ad79da6426..6cb63312c3 100644 --- a/services/debugger/transformation/transformationStatusUploader.go +++ b/services/debugger/transformation/transformationStatusUploader.go @@ -6,6 +6,8 @@ import ( "sync" "time" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" + jsoniter "github.com/json-iterator/go" "github.com/samber/lo" @@ -13,21 +15,20 @@ import ( "github.com/rudderlabs/rudder-go-kit/logger" backendconfig "github.com/rudderlabs/rudder-server/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/rruntime" "github.com/rudderlabs/rudder-server/services/debugger" "github.com/rudderlabs/rudder-server/services/debugger/cache" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/types" ) type TransformationStatusT struct { SourceID string DestID string Destination *backendconfig.DestinationT - UserTransformedEvents []transformer.TransformerEvent + UserTransformedEvents []types.TransformerEvent EventsByMessageID map[string]types.SingularEventWithReceivedAt - FailedEvents []transformer.TransformerResponse + FailedEvents []types.TransformerResponse UniqueMessageIds map[string]struct{} } @@ -233,13 +234,13 @@ func (ts *TransformationStatusT) Limit( append( lo.Map( ts.UserTransformedEvents, - func(event transformer.TransformerEvent, _ int) string { + func(event types.TransformerEvent, _ int) string { return event.Metadata.MessageID }, ), lo.Map( ts.FailedEvents, - func(event transformer.TransformerResponse, _ int) string { + func(event types.TransformerResponse, _ int) string { return event.Metadata.MessageID }, )..., @@ -383,7 +384,7 @@ func (h *Handle) processRecordTransformationStatus(tStatus *TransformationStatus } var isError bool switch failedEvent.StatusCode { - case types.FilterEventCode: + case reportingTypes.FilterEventCode: eventAfter.IsDropped = true isError = false default: diff --git a/services/debugger/transformation/transformationStatusUploader_test.go b/services/debugger/transformation/transformationStatusUploader_test.go index 9767e8e916..4c96bbbd8d 100644 --- a/services/debugger/transformation/transformationStatusUploader_test.go +++ b/services/debugger/transformation/transformationStatusUploader_test.go @@ -18,10 +18,9 @@ import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" mocksBackendConfig "github.com/rudderlabs/rudder-server/mocks/backend-config" - "github.com/rudderlabs/rudder-server/processor/transformer" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-server/utils/pubsub" testutils "github.com/rudderlabs/rudder-server/utils/tests" - "github.com/rudderlabs/rudder-server/utils/types" ) const ( @@ -324,13 +323,13 @@ var _ = Describe("eventDeliveryStatusUploader", func() { func TestLimit(t *testing.T) { var ( singularEvent1 = types.SingularEventT{"payload": "event-1"} - metadata1 = transformer.Metadata{MessageID: "message-id-1"} + metadata1 = types.Metadata{MessageID: "message-id-1"} singularEvent2 = types.SingularEventT{"payload": "event-2"} - metadata2 = transformer.Metadata{MessageID: "message-id-2"} + metadata2 = types.Metadata{MessageID: "message-id-2"} singularEvent3 = types.SingularEventT{"payload": "event-3"} - metadata3 = transformer.Metadata{MessageID: "message-id-3"} + metadata3 = types.Metadata{MessageID: "message-id-3"} singularEvent4 = types.SingularEventT{"payload": "event-4"} - metadata4 = transformer.Metadata{MessageID: "message-id-4"} + metadata4 = types.Metadata{MessageID: "message-id-4"} now = time.Now() limit = 1 ) @@ -339,7 +338,7 @@ func TestLimit(t *testing.T) { Destination: &sampleBackendConfig.Sources[1].Destinations[1], DestID: sampleBackendConfig.Sources[1].Destinations[1].ID, SourceID: sampleBackendConfig.Sources[1].ID, - UserTransformedEvents: []transformer.TransformerEvent{ + UserTransformedEvents: []types.TransformerEvent{ { Message: singularEvent1, Metadata: metadata1, @@ -367,7 +366,7 @@ func TestLimit(t *testing.T) { ReceivedAt: now, }, }, - FailedEvents: []transformer.TransformerResponse{ + FailedEvents: []types.TransformerResponse{ { Output: singularEvent1, Metadata: metadata3, @@ -401,7 +400,7 @@ func TestLimit(t *testing.T) { ) require.Equal( t, - []transformer.TransformerEvent{ + []types.TransformerEvent{ { Message: singularEvent1, Metadata: metadata1, @@ -411,7 +410,7 @@ func TestLimit(t *testing.T) { ) require.Equal( t, - []transformer.TransformerResponse{ + []types.TransformerResponse{ { Output: singularEvent1, Metadata: metadata3, diff --git a/testhelper/transformertest/builder.go b/testhelper/transformertest/builder.go index cdad3bb214..acd59c638c 100644 --- a/testhelper/transformertest/builder.go +++ b/testhelper/transformertest/builder.go @@ -7,11 +7,10 @@ import ( "net/http/httptest" "strings" - "github.com/rudderlabs/rudder-server/router/types" - "github.com/tidwall/sjson" - "github.com/rudderlabs/rudder-server/processor/transformer" + processorTypes "github.com/rudderlabs/rudder-server/processor/types" + "github.com/rudderlabs/rudder-server/router/types" ) // NewBuilder returns a new test transformer Builder @@ -151,7 +150,7 @@ func transformerFunc(h TransformerHandler) http.HandlerFunc { w.WriteHeader(http.StatusInternalServerError) return } - var request []transformer.TransformerEvent + var request []processorTypes.TransformerEvent if err := json.Unmarshal(data, &request); err != nil { w.WriteHeader(http.StatusBadRequest) return diff --git a/testhelper/transformertest/handler_funcs.go b/testhelper/transformertest/handler_funcs.go index 563c46fdd1..efa0804e8c 100644 --- a/testhelper/transformertest/handler_funcs.go +++ b/testhelper/transformertest/handler_funcs.go @@ -7,20 +7,20 @@ import ( "github.com/rudderlabs/rudder-server/router/types" "github.com/rudderlabs/rudder-server/processor/integrations" - "github.com/rudderlabs/rudder-server/processor/transformer" + processorTypes "github.com/rudderlabs/rudder-server/processor/types" ) // TransformerHandler is a function that takes a transformer request and returns a response -type TransformerHandler func(request []transformer.TransformerEvent) []transformer.TransformerResponse +type TransformerHandler func(request []processorTypes.TransformerEvent) []processorTypes.TransformerResponse // RouterTransformerHandler is a function that takes a router transformer request and returns a response type RouterTransformerHandler func(request types.TransformMessageT) types.DestinationJobs // MirroringTransformerHandler mirrors the request payload in the response -var MirroringTransformerHandler TransformerHandler = func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { +var MirroringTransformerHandler TransformerHandler = func(request []processorTypes.TransformerEvent) (response []processorTypes.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: http.StatusOK, @@ -46,10 +46,10 @@ var MirroringRouterTransformerHandler RouterTransformerHandler = func(request ty // ErrorTransformerHandler mirrors the request payload in the response but uses an error status code func ErrorTransformerHandler(code int, err string) TransformerHandler { - return func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { + return func(request []processorTypes.TransformerEvent) (response []processorTypes.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: code, @@ -61,11 +61,11 @@ func ErrorTransformerHandler(code int, err string) TransformerHandler { } // ViolationErrorTransformerHandler mirrors the request payload in the response but uses an error status code along with the provided validation errors -func ViolationErrorTransformerHandler(code int, err string, validationErrors []transformer.ValidationError) TransformerHandler { - return func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { +func ViolationErrorTransformerHandler(code int, err string, validationErrors []processorTypes.ValidationError) TransformerHandler { + return func(request []processorTypes.TransformerEvent) (response []processorTypes.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: req.Message, StatusCode: code, @@ -78,19 +78,19 @@ func ViolationErrorTransformerHandler(code int, err string, validationErrors []t } // EmptyTransformerHandler returns an empty response -var EmptyTransformerHandler TransformerHandler = func(request []transformer.TransformerEvent) []transformer.TransformerResponse { - return []transformer.TransformerResponse{} +var EmptyTransformerHandler TransformerHandler = func(request []processorTypes.TransformerEvent) []processorTypes.TransformerResponse { + return []processorTypes.TransformerResponse{} } // DestTransformerHandler returns an empty response -func DestTransformerHandler(f func(event transformer.TransformerEvent) integrations.PostParametersT) func(request []transformer.TransformerEvent) []transformer.TransformerResponse { - return func(request []transformer.TransformerEvent) (res []transformer.TransformerResponse) { +func DestTransformerHandler(f func(event processorTypes.TransformerEvent) integrations.PostParametersT) func(request []processorTypes.TransformerEvent) []processorTypes.TransformerResponse { + return func(request []processorTypes.TransformerEvent) (res []processorTypes.TransformerResponse) { for _, req := range request { postParameters := f(req) jsonString, _ := json.Marshal(postParameters) var output map[string]interface{} _ = json.Unmarshal(jsonString, &output) - res = append(res, transformer.TransformerResponse{ + res = append(res, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: output, StatusCode: http.StatusOK, @@ -101,8 +101,8 @@ func DestTransformerHandler(f func(event transformer.TransformerEvent) integrati } // RESTJSONDestTransformerHandler transforms the request payload into a REST JSON destination request using the original message as the payload -func RESTJSONDestTransformerHandler(method, url string) func(request []transformer.TransformerEvent) []transformer.TransformerResponse { - return DestTransformerHandler(func(event transformer.TransformerEvent) integrations.PostParametersT { +func RESTJSONDestTransformerHandler(method, url string) func(request []processorTypes.TransformerEvent) []processorTypes.TransformerResponse { + return DestTransformerHandler(func(event processorTypes.TransformerEvent) integrations.PostParametersT { return integrations.PostParametersT{ Type: "REST", URL: url, @@ -116,10 +116,10 @@ func RESTJSONDestTransformerHandler(method, url string) func(request []transform // WarehouseTransformerHandler mirrors the request payload in the response but uses an error, status code along with warehouse compatible output func WarehouseTransformerHandler(tableName string, code int, err string) TransformerHandler { - return func(request []transformer.TransformerEvent) (response []transformer.TransformerResponse) { + return func(request []processorTypes.TransformerEvent) (response []processorTypes.TransformerResponse) { for i := range request { req := request[i] - response = append(response, transformer.TransformerResponse{ + response = append(response, processorTypes.TransformerResponse{ Metadata: req.Metadata, Output: map[string]interface{}{ "table": tableName, diff --git a/utils/misc/misc.go b/utils/misc/misc.go index 0c0677e114..6f69496816 100644 --- a/utils/misc/misc.go +++ b/utils/misc/misc.go @@ -34,7 +34,6 @@ import ( "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-server/utils/httputil" - "github.com/rudderlabs/rudder-server/utils/types" ) var ( @@ -112,15 +111,6 @@ func GetMD5Hash(input string) string { return hex.EncodeToString(hash[:]) } -// GetRudderEventVal returns the value corresponding to the key in the message structure -func GetRudderEventVal(key string, rudderEvent types.SingularEventT) (interface{}, bool) { - rudderVal, ok := rudderEvent[key] - if !ok { - return nil, false - } - return rudderVal, true -} - // RemoveFilePaths removes filePaths as well as cleans up the empty folder structure. func RemoveFilePaths(filePaths ...string) { for _, fp := range filePaths { diff --git a/utils/types/types.go b/utils/types/types.go index 6e2b0ce3cc..b82a54dd90 100644 --- a/utils/types/types.go +++ b/utils/types/types.go @@ -4,7 +4,6 @@ package types import ( "context" - "time" "github.com/rudderlabs/rudder-server/enterprise/suppress-user/model" . "github.com/rudderlabs/rudder-server/utils/tx" //nolint:staticcheck @@ -16,29 +15,6 @@ const ( DrainEventCode = 410 ) -// SingularEventT single event structure -type SingularEventT map[string]interface{} - -type SingularEventWithReceivedAt struct { - SingularEvent SingularEventT - ReceivedAt time.Time -} - -// GatewayBatchRequest batch request structure -type GatewayBatchRequest struct { - Batch []SingularEventT `json:"batch"` - RequestIP string `json:"requestIP"` - ReceivedAt time.Time `json:"receivedAt"` -} - -type EventParams struct { - SourceJobRunId string `json:"source_job_run_id"` - SourceId string `json:"source_id"` - SourceTaskRunId string `json:"source_task_run_id"` - TraceParent string `json:"traceparent"` - DestinationID string `json:"destination_id"` -} - // UserSuppression is interface to access Suppress user feature type UserSuppression interface { GetSuppressedUser(workspaceID, userID, sourceID string) *model.Metadata diff --git a/warehouse/integrations/testhelper/staging.go b/warehouse/integrations/testhelper/staging.go index b9daf20614..84e07b08c3 100644 --- a/warehouse/integrations/testhelper/staging.go +++ b/warehouse/integrations/testhelper/staging.go @@ -13,6 +13,8 @@ import ( "text/template" "time" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/google/uuid" "github.com/samber/lo" "github.com/stretchr/testify/require" @@ -128,14 +130,14 @@ func prepareStagingFilePathUsingEventsFile(t testing.TB, testConfig *TestConfig) }) require.NoError(t, err) - var transformerEvents []transformer.TransformerEvent + var transformerEvents []types.TransformerEvent err = json.Unmarshal([]byte(b.String()), &transformerEvents) require.NoError(t, err) tr := transformer.NewTransformer(c, logger.NOP, stats.Default) response := tr.Transform(context.Background(), transformerEvents, 100) require.Zero(t, len(response.FailedEvents)) - responseOutputs := lo.Map(response.Events, func(r transformer.TransformerResponse, index int) map[string]interface{} { + responseOutputs := lo.Map(response.Events, func(r types.TransformerResponse, index int) map[string]interface{} { return r.Output }) From eea1385fce1e869e400a1ac16d423a5f98a4a1a1 Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Wed, 15 Jan 2025 17:28:04 +0530 Subject: [PATCH 4/8] chore: populate ut module --- .../transformer_utils/transformer_utils.go | 38 +++ .../user_transformer/user_transformer.go | 319 +++++++++++++++++- 2 files changed, 350 insertions(+), 7 deletions(-) create mode 100644 processor/internal/transformer_utils/transformer_utils.go diff --git a/processor/internal/transformer_utils/transformer_utils.go b/processor/internal/transformer_utils/transformer_utils.go new file mode 100644 index 0000000000..9508d1579f --- /dev/null +++ b/processor/internal/transformer_utils/transformer_utils.go @@ -0,0 +1,38 @@ +package transformer_utils + +import ( + "context" + "net/http" + "time" + + "github.com/rudderlabs/rudder-go-kit/logger" +) + +const ( + StatusCPDown = 809 + TransformerRequestFailure = 909 + TransformerRequestTimeout = 919 +) + +func IsJobTerminated(status int) bool { + if status == http.StatusTooManyRequests || status == http.StatusRequestTimeout { + return false + } + return status >= http.StatusOK && status < http.StatusInternalServerError +} + +func TrackLongRunningTransformation(ctx context.Context, stage string, timeout time.Duration, log logger.Logger) { + start := time.Now() + t := time.NewTimer(timeout) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + log.Errorw("Long running transformation detected", + "stage", stage, + "duration", time.Since(start).String()) + } + } +} diff --git a/processor/internal/user_transformer/user_transformer.go b/processor/internal/user_transformer/user_transformer.go index 8a18e3b0e0..64e135c6aa 100644 --- a/processor/internal/user_transformer/user_transformer.go +++ b/processor/internal/user_transformer/user_transformer.go @@ -1,17 +1,35 @@ package user_transformer import ( + "bytes" "context" + "fmt" + "io" + "net/http" + "os" + "runtime/trace" + "strconv" + "sync" "time" + "github.com/cenkalti/backoff/v4" + jsoniter "github.com/json-iterator/go" + "github.com/samber/lo" + "github.com/rudderlabs/rudder-go-kit/config" + "github.com/rudderlabs/rudder-go-kit/httputil" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" + "github.com/rudderlabs/rudder-server/processor/integrations" "github.com/rudderlabs/rudder-server/processor/internal/http_client" + "github.com/rudderlabs/rudder-server/processor/internal/transformer_utils" "github.com/rudderlabs/rudder-server/processor/types" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" ) +var json = jsoniter.ConfigCompatibleWithStandardLibrary + type UserTransformer struct { config struct { userTransformationURL string @@ -19,25 +37,35 @@ type UserTransformer struct { failOnUserTransformTimeout config.ValueLoader[bool] failOnError config.ValueLoader[bool] maxRetryBackoffInterval config.ValueLoader[time.Duration] + timeoutDuration time.Duration + maxConcurrency int } - conf *config.Config - log logger.Logger - stat stats.Stats - client http_client.HTTPDoer + conf *config.Config + log logger.Logger + stat stats.Stats + client http_client.HTTPDoer + guardConcurrency chan struct{} } func (u *UserTransformer) SendRequest(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { - _ = u.userTransformURL() - return types.Response{} + var dehydratedClientEvents []types.TransformerEvent + for _, clientEvent := range clientEvents { + dehydratedClientEvent := clientEvent.GetVersionsOnly() + dehydratedClientEvents = append(dehydratedClientEvents, *dehydratedClientEvent) + } + return u.transform(ctx, dehydratedClientEvents, u.userTransformURL(), batchSize) } func NewUserTransformer(conf *config.Config, log logger.Logger, stat stats.Stats) *UserTransformer { handle := &UserTransformer{} handle.conf = conf - handle.log = log + handle.log = log.Child("user_transformer") handle.stat = stat handle.client = http_client.NewHTTPClient(conf) + handle.config.maxConcurrency = conf.GetInt("Processor.maxConcurrency", 200) + handle.guardConcurrency = make(chan struct{}, handle.config.maxConcurrency) handle.config.userTransformationURL = handle.conf.GetString("USER_TRANSFORM_URL", "http://localhost:9090") + handle.config.timeoutDuration = conf.GetDuration("HttpClient.procTransformer.timeout", 600, time.Second) handle.config.failOnUserTransformTimeout = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnUserTransformTimeout") handle.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") handle.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") @@ -45,6 +73,283 @@ func NewUserTransformer(conf *config.Config, log logger.Logger, stat stats.Stats return handle } +func (u *UserTransformer) transform( + ctx context.Context, + clientEvents []types.TransformerEvent, + url string, + batchSize int, +) types.Response { + if len(clientEvents) == 0 { + return types.Response{} + } + // flip sourceID and originalSourceID if it's a replay source for the purpose of any user transformation + // flip back afterward + for i := range clientEvents { + if clientEvents[i].Metadata.OriginalSourceID != "" { + clientEvents[i].Metadata.OriginalSourceID, clientEvents[i].Metadata.SourceID = clientEvents[i].Metadata.SourceID, clientEvents[i].Metadata.OriginalSourceID + } + } + sTags := stats.Tags{ + "dest_type": clientEvents[0].Destination.DestinationDefinition.Name, + "dest_id": clientEvents[0].Destination.ID, + "src_id": clientEvents[0].Metadata.SourceID, + "stage": "user_transformer", + } + + var trackWg sync.WaitGroup + defer trackWg.Wait() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + trackWg.Add(1) + go func() { + var loggerCtx []interface{} + for k, v := range sTags { + loggerCtx = append(loggerCtx, k, v) + } + transformer_utils.TrackLongRunningTransformation(ctx, "user_transformer", u.config.timeoutDuration, u.log.With(loggerCtx...)) + trackWg.Done() + }() + + batches := lo.Chunk(clientEvents, batchSize) + + u.stat.NewTaggedStat( + "processor.transformer_request_batch_count", + stats.HistogramType, + sTags, + ).Observe(float64(len(batches))) + trace.Logf(ctx, "request", "batch_count: %d", len(batches)) + + transformResponse := make([][]types.TransformerResponse, len(batches)) + + var wg sync.WaitGroup + wg.Add(len(batches)) + + lo.ForEach( + batches, + func(batch []types.TransformerEvent, i int) { + u.guardConcurrency <- struct{}{} + go func() { + trace.WithRegion(ctx, "request", func() { + transformResponse[i] = u.request(ctx, url, "user_transformer", batch) + }) + <-u.guardConcurrency + wg.Done() + }() + }, + ) + wg.Wait() + + var outClientEvents []types.TransformerResponse + var failedEvents []types.TransformerResponse + + for _, batch := range transformResponse { + // Transform is one to many mapping so returned + // response for each is an array. We flatten it out + for _, transformerResponse := range batch { + if transformerResponse.Metadata.OriginalSourceID != "" { + transformerResponse.Metadata.SourceID, transformerResponse.Metadata.OriginalSourceID = transformerResponse.Metadata.OriginalSourceID, transformerResponse.Metadata.SourceID + } + switch transformerResponse.StatusCode { + case http.StatusOK: + outClientEvents = append(outClientEvents, transformerResponse) + default: + failedEvents = append(failedEvents, transformerResponse) + } + } + } + + u.stat.NewStat("processor.transformer_sent", stats.CountType).Count(len(clientEvents)) + u.stat.NewStat("processor.transformer_received", stats.CountType).Count(len(outClientEvents)) + + return types.Response{ + Events: outClientEvents, + FailedEvents: failedEvents, + } +} + +func (u *UserTransformer) request(ctx context.Context, url, stage string, data []types.TransformerEvent) []types.TransformerResponse { + // Call remote transformation + var ( + rawJSON []byte + err error + ) + + trace.WithRegion(ctx, "marshal", func() { + rawJSON, err = json.Marshal(data) + }) + trace.Logf(ctx, "marshal", "request raw body size: %d", len(rawJSON)) + if err != nil { + panic(err) + } + + if len(data) == 0 { + return nil + } + + var ( + respData []byte + statusCode int + ) + + // endless retry if transformer-control plane connection is down + endlessBackoff := backoff.NewExponentialBackOff() + endlessBackoff.MaxElapsedTime = 0 // no max time -> ends only when no error + endlessBackoff.MaxInterval = u.config.maxRetryBackoffInterval.Load() + + // endless backoff loop, only nil error or panics inside + _ = backoff.RetryNotify( + func() error { + transformationID := "" + if len(data[0].Destination.Transformations) > 0 { + transformationID = data[0].Destination.Transformations[0].ID + } + + respData, statusCode = u.doPost(ctx, rawJSON, url, stats.Tags{ + "destinationType": data[0].Destination.DestinationDefinition.Name, + "destinationId": data[0].Destination.ID, + "sourceId": data[0].Metadata.SourceID, + "transformationId": transformationID, + "stage": stage, + + // Legacy tags: to be removed + "dest_type": data[0].Destination.DestinationDefinition.Name, + "dest_id": data[0].Destination.ID, + "src_id": data[0].Metadata.SourceID, + }) + if statusCode == transformer_utils.StatusCPDown { + u.stat.NewStat("processor.control_plane_down", stats.GaugeType).Gauge(1) + return fmt.Errorf("control plane not reachable") + } + u.stat.NewStat("processor.control_plane_down", stats.GaugeType).Gauge(0) + return nil + }, + endlessBackoff, + func(err error, t time.Duration) { + var transformationID, transformationVersionID string + if len(data[0].Destination.Transformations) > 0 { + transformationID = data[0].Destination.Transformations[0].ID + transformationVersionID = data[0].Destination.Transformations[0].VersionID + } + u.log.Errorf("JS HTTP connection error: URL: %v Error: %+v. WorkspaceID: %s, sourceID: %s, destinationID: %s, transformationID: %s, transformationVersionID: %s", + url, err, data[0].Metadata.WorkspaceID, data[0].Metadata.SourceID, data[0].Metadata.DestinationID, + transformationID, transformationVersionID, + ) + }, + ) + // control plane back up + + switch statusCode { + case http.StatusOK, + http.StatusBadRequest, + http.StatusNotFound, + http.StatusRequestEntityTooLarge: + default: + u.log.Errorf("Transformer returned status code: %v", statusCode) + } + + var transformerResponses []types.TransformerResponse + switch statusCode { + case http.StatusOK: + integrations.CollectIntgTransformErrorStats(respData) + + trace.Logf(ctx, "Unmarshal", "response raw size: %d", len(respData)) + trace.WithRegion(ctx, "Unmarshal", func() { + err = json.Unmarshal(respData, &transformerResponses) + }) + // This is returned by our JS engine so should be parsable + // Panic the processor to avoid replays + if err != nil { + u.log.Errorf("Data sent to transformer : %v", string(rawJSON)) + u.log.Errorf("Transformer returned : %v", string(respData)) + panic(err) + } + default: + for i := range data { + transformEvent := &data[i] + resp := types.TransformerResponse{StatusCode: statusCode, Error: string(respData), Metadata: transformEvent.Metadata} + transformerResponses = append(transformerResponses, resp) + } + } + return transformerResponses +} + +func (u *UserTransformer) doPost(ctx context.Context, rawJSON []byte, url string, tags stats.Tags) ([]byte, int) { + var ( + retryCount int + resp *http.Response + respData []byte + ) + retryStrategy := backoff.NewExponentialBackOff() + // MaxInterval caps the RetryInterval + retryStrategy.MaxInterval = u.config.maxRetryBackoffInterval.Load() + + err := backoff.RetryNotify( + func() error { + var reqErr error + requestStartTime := time.Now() + + trace.WithRegion(ctx, "request/post", func() { + var req *http.Request + req, reqErr = http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(rawJSON)) + if reqErr != nil { + return + } + + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("X-Feature-Gzip-Support", "?1") + // Header to let transformer know that the client understands event filter code + req.Header.Set("X-Feature-Filter-Code", "?1") + + resp, reqErr = u.client.Do(req) + }) + u.stat.NewTaggedStat("processor.transformer_request_time", stats.TimerType, tags).SendTiming(time.Since(requestStartTime)) + if reqErr != nil { + return reqErr + } + + defer func() { httputil.CloseResponse(resp) }() + + if !transformer_utils.IsJobTerminated(resp.StatusCode) && resp.StatusCode != transformer_utils.StatusCPDown { + return fmt.Errorf("transformer returned status code: %v", resp.StatusCode) + } + + respData, reqErr = io.ReadAll(resp.Body) + return reqErr + }, + backoff.WithMaxRetries(retryStrategy, uint64(u.config.maxRetry.Load())), + func(err error, t time.Duration) { + retryCount++ + u.log.Warnn( + "JS HTTP connection error", + logger.NewErrorField(err), + logger.NewIntField("attempts", int64(retryCount)), + ) + }, + ) + if err != nil { + if u.config.failOnUserTransformTimeout.Load() && os.IsTimeout(err) { + return []byte(fmt.Sprintf("transformer request timed out: %s", err)), transformer_utils.TransformerRequestTimeout + } else if u.config.failOnError.Load() { + return []byte(fmt.Sprintf("transformer request failed: %s", err)), transformer_utils.TransformerRequestFailure + } else { + panic(err) + } + } + + // perform version compatibility check only on success + if resp.StatusCode == http.StatusOK { + transformerAPIVersion, _ := strconv.Atoi(resp.Header.Get("apiVersion")) + if reportingTypes.SupportedTransformerApiVersion != transformerAPIVersion { + unexpectedVersionError := fmt.Errorf("incompatible transformer version: Expected: %d Received: %s, URL: %v", reportingTypes.SupportedTransformerApiVersion, resp.Header.Get("apiVersion"), url) + u.log.Error(unexpectedVersionError) + panic(unexpectedVersionError) + } + } + + return respData, resp.StatusCode +} + func (u *UserTransformer) userTransformURL() string { return u.config.userTransformationURL + "/customTransform" } From fdebb34571cca7b37444b4c304e71d6c57ffcb39 Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Mon, 20 Jan 2025 14:38:34 +0530 Subject: [PATCH 5/8] chore: add module dest transform --- .../destination_transformer.go | 305 +++++++++++++++++- .../user_transformer/user_transformer.go | 6 +- 2 files changed, 302 insertions(+), 9 deletions(-) diff --git a/processor/internal/destination_transformer/destination_transformer.go b/processor/internal/destination_transformer/destination_transformer.go index 2ef88e361b..520301a0d1 100644 --- a/processor/internal/destination_transformer/destination_transformer.go +++ b/processor/internal/destination_transformer/destination_transformer.go @@ -1,17 +1,31 @@ package destination_transformer import ( + "bytes" "context" + "encoding/json" "fmt" + "io" + "net/http" + "runtime/trace" + "strconv" "strings" + "sync" "time" + "github.com/cenkalti/backoff" + "github.com/samber/lo" + "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" + "github.com/rudderlabs/rudder-server/processor/integrations" "github.com/rudderlabs/rudder-server/processor/internal/http_client" + "github.com/rudderlabs/rudder-server/processor/internal/transformer_utils" "github.com/rudderlabs/rudder-server/processor/types" + "github.com/rudderlabs/rudder-server/utils/httputil" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) @@ -20,17 +34,19 @@ type DestTransformer struct { destTransformationURL string maxRetry config.ValueLoader[int] failOnError config.ValueLoader[bool] + maxConcurrency int maxRetryBackoffInterval config.ValueLoader[time.Duration] + timeoutDuration time.Duration } - conf *config.Config - log logger.Logger - stat stats.Stats - client http_client.HTTPDoer + guardConcurrency chan struct{} + conf *config.Config + log logger.Logger + stat stats.Stats + client http_client.HTTPDoer } func (d *DestTransformer) SendRequest(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { - _ = d.destTransformURL(clientEvents[0].Destination.DestinationDefinition.Name) - return types.Response{} + return d.transform(ctx, clientEvents, d.destTransformURL(clientEvents[0].Destination.DestinationDefinition.Name), batchSize) } func NewDestTransformer(conf *config.Config, log logger.Logger, stat stats.Stats) *DestTransformer { @@ -39,13 +55,290 @@ func NewDestTransformer(conf *config.Config, log logger.Logger, stat stats.Stats handle.log = log handle.stat = stat handle.client = http_client.NewHTTPClient(conf) + handle.config.maxConcurrency = conf.GetInt("Processor.maxConcurrency", 200) + handle.guardConcurrency = make(chan struct{}, handle.config.maxConcurrency) handle.config.destTransformationURL = handle.conf.GetString("DEST_TRANSFORM_URL", "http://localhost:9090") + handle.config.timeoutDuration = conf.GetDuration("HttpClient.procTransformer.timeout", 600, time.Second) handle.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") handle.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") handle.config.maxRetryBackoffInterval = conf.GetReloadableDurationVar(30, time.Second, "Processor.maxRetryBackoffInterval") return handle } +func (d *DestTransformer) transform( + ctx context.Context, + clientEvents []types.TransformerEvent, + url string, + batchSize int, +) types.Response { + if len(clientEvents) == 0 { + return types.Response{} + } + // flip sourceID and originalSourceID if it's a replay source for the purpose of any user transformation + // flip back afterward + for i := range clientEvents { + if clientEvents[i].Metadata.OriginalSourceID != "" { + clientEvents[i].Metadata.OriginalSourceID, clientEvents[i].Metadata.SourceID = clientEvents[i].Metadata.SourceID, clientEvents[i].Metadata.OriginalSourceID + } + } + sTags := stats.Tags{ + "dest_type": clientEvents[0].Destination.DestinationDefinition.Name, + "dest_id": clientEvents[0].Destination.ID, + "src_id": clientEvents[0].Metadata.SourceID, + "stage": "dest_transformer", + } + + var trackWg sync.WaitGroup + defer trackWg.Wait() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + trackWg.Add(1) + go func() { + var loggerCtx []interface{} + for k, v := range sTags { + loggerCtx = append(loggerCtx, k, v) + } + transformer_utils.TrackLongRunningTransformation(ctx, "dest_transformer", d.config.timeoutDuration, d.log.With(loggerCtx...)) + trackWg.Done() + }() + + batches := lo.Chunk(clientEvents, batchSize) + + d.stat.NewTaggedStat( + "processor.transformer_request_batch_count", + stats.HistogramType, + sTags, + ).Observe(float64(len(batches))) + trace.Logf(ctx, "request", "batch_count: %d", len(batches)) + + transformResponse := make([][]types.TransformerResponse, len(batches)) + + var wg sync.WaitGroup + wg.Add(len(batches)) + + lo.ForEach( + batches, + func(batch []types.TransformerEvent, i int) { + d.guardConcurrency <- struct{}{} + go func() { + trace.WithRegion(ctx, "request", func() { + transformResponse[i] = d.request(ctx, url, "dest_transformer", batch) + }) + <-d.guardConcurrency + wg.Done() + }() + }, + ) + wg.Wait() + + var outClientEvents []types.TransformerResponse + var failedEvents []types.TransformerResponse + + for _, batch := range transformResponse { + // Transform is one to many mapping so returned + // response for each is an array. We flatten it out + for _, transformerResponse := range batch { + if transformerResponse.Metadata.OriginalSourceID != "" { + transformerResponse.Metadata.SourceID, transformerResponse.Metadata.OriginalSourceID = transformerResponse.Metadata.OriginalSourceID, transformerResponse.Metadata.SourceID + } + switch transformerResponse.StatusCode { + case http.StatusOK: + outClientEvents = append(outClientEvents, transformerResponse) + default: + failedEvents = append(failedEvents, transformerResponse) + } + } + } + + d.stat.NewStat("processor.transformer_sent", stats.CountType).Count(len(clientEvents)) + d.stat.NewStat("processor.transformer_received", stats.CountType).Count(len(outClientEvents)) + + return types.Response{ + Events: outClientEvents, + FailedEvents: failedEvents, + } +} + +func (d *DestTransformer) request(ctx context.Context, url, stage string, data []types.TransformerEvent) []types.TransformerResponse { + // Call remote transformation + var ( + rawJSON []byte + err error + ) + + trace.WithRegion(ctx, "marshal", func() { + rawJSON, err = json.Marshal(data) + }) + trace.Logf(ctx, "marshal", "request raw body size: %d", len(rawJSON)) + if err != nil { + panic(err) + } + + if len(data) == 0 { + return nil + } + + var ( + respData []byte + statusCode int + ) + + // endless retry if transformer-control plane connection is down + endlessBackoff := backoff.NewExponentialBackOff() + endlessBackoff.MaxElapsedTime = 0 // no max time -> ends only when no error + endlessBackoff.MaxInterval = d.config.maxRetryBackoffInterval.Load() + + // endless backoff loop, only nil error or panics inside + _ = backoff.RetryNotify( + func() error { + transformationID := "" + if len(data[0].Destination.Transformations) > 0 { + transformationID = data[0].Destination.Transformations[0].ID + } + + respData, statusCode = d.doPost(ctx, rawJSON, url, stats.Tags{ + "destinationType": data[0].Destination.DestinationDefinition.Name, + "destinationId": data[0].Destination.ID, + "sourceId": data[0].Metadata.SourceID, + "transformationId": transformationID, + "stage": stage, + + // Legacy tags: to be removed + "dest_type": data[0].Destination.DestinationDefinition.Name, + "dest_id": data[0].Destination.ID, + "src_id": data[0].Metadata.SourceID, + }) + if statusCode == transformer_utils.StatusCPDown { + d.stat.NewStat("processor.control_plane_down", stats.GaugeType).Gauge(1) + return fmt.Errorf("control plane not reachable") + } + d.stat.NewStat("processor.control_plane_down", stats.GaugeType).Gauge(0) + return nil + }, + endlessBackoff, + func(err error, t time.Duration) { + var transformationID, transformationVersionID string + if len(data[0].Destination.Transformations) > 0 { + transformationID = data[0].Destination.Transformations[0].ID + transformationVersionID = data[0].Destination.Transformations[0].VersionID + } + d.log.Errorf("JS HTTP connection error: URL: %v Error: %+v. WorkspaceID: %s, sourceID: %s, destinationID: %s, transformationID: %s, transformationVersionID: %s", + url, err, data[0].Metadata.WorkspaceID, data[0].Metadata.SourceID, data[0].Metadata.DestinationID, + transformationID, transformationVersionID, + ) + }, + ) + // control plane back up + + switch statusCode { + case http.StatusOK, + http.StatusBadRequest, + http.StatusNotFound, + http.StatusRequestEntityTooLarge: + default: + d.log.Errorf("Transformer returned status code: %v", statusCode) + } + + var transformerResponses []types.TransformerResponse + switch statusCode { + case http.StatusOK: + integrations.CollectIntgTransformErrorStats(respData) + + trace.Logf(ctx, "Unmarshal", "response raw size: %d", len(respData)) + trace.WithRegion(ctx, "Unmarshal", func() { + err = json.Unmarshal(respData, &transformerResponses) + }) + // This is returned by our JS engine so should be parsable + // Panic the processor to avoid replays + if err != nil { + d.log.Errorf("Data sent to transformer : %v", string(rawJSON)) + d.log.Errorf("Transformer returned : %v", string(respData)) + panic(err) + } + default: + for i := range data { + transformEvent := &data[i] + resp := types.TransformerResponse{StatusCode: statusCode, Error: string(respData), Metadata: transformEvent.Metadata} + transformerResponses = append(transformerResponses, resp) + } + } + return transformerResponses +} + +func (d *DestTransformer) doPost(ctx context.Context, rawJSON []byte, url string, tags stats.Tags) ([]byte, int) { + var ( + retryCount int + resp *http.Response + respData []byte + ) + retryStrategy := backoff.NewExponentialBackOff() + // MaxInterval caps the RetryInterval + retryStrategy.MaxInterval = d.config.maxRetryBackoffInterval.Load() + + err := backoff.RetryNotify( + func() error { + var reqErr error + requestStartTime := time.Now() + + trace.WithRegion(ctx, "request/post", func() { + var req *http.Request + req, reqErr = http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(rawJSON)) + if reqErr != nil { + return + } + + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("X-Feature-Gzip-Support", "?1") + // Header to let transformer know that the client understands event filter code + req.Header.Set("X-Feature-Filter-Code", "?1") + + resp, reqErr = d.client.Do(req) + defer func() { httputil.CloseResponse(resp) }() + }) + d.stat.NewTaggedStat("processor.transformer_request_time", stats.TimerType, tags).SendTiming(time.Since(requestStartTime)) + if reqErr != nil { + return reqErr + } + + if !transformer_utils.IsJobTerminated(resp.StatusCode) && resp.StatusCode != transformer_utils.StatusCPDown { + return fmt.Errorf("transformer returned status code: %v", resp.StatusCode) + } + + respData, reqErr = io.ReadAll(resp.Body) + return reqErr + }, + backoff.WithMaxRetries(retryStrategy, uint64(d.config.maxRetry.Load())), + func(err error, t time.Duration) { + retryCount++ + d.log.Warnn( + "JS HTTP connection error", + logger.NewErrorField(err), + logger.NewIntField("attempts", int64(retryCount)), + ) + }, + ) + if err != nil { + if d.config.failOnError.Load() { + return []byte(fmt.Sprintf("transformer request failed: %s", err)), transformer_utils.TransformerRequestFailure + } else { + panic(err) + } + } + + // perform version compatibility check only on success + if resp.StatusCode == http.StatusOK { + transformerAPIVersion, _ := strconv.Atoi(resp.Header.Get("apiVersion")) + if reportingTypes.SupportedTransformerApiVersion != transformerAPIVersion { + unexpectedVersionError := fmt.Errorf("incompatible transformer version: Expected: %d Received: %s, URL: %v", reportingTypes.SupportedTransformerApiVersion, resp.Header.Get("apiVersion"), url) + d.log.Error(unexpectedVersionError) + panic(unexpectedVersionError) + } + } + + return respData, resp.StatusCode +} + func (d *DestTransformer) destTransformURL(destType string) string { destinationEndPoint := fmt.Sprintf("%s/v0/destinations/%s", d.config.destTransformationURL, strings.ToLower(destType)) diff --git a/processor/internal/user_transformer/user_transformer.go b/processor/internal/user_transformer/user_transformer.go index 64e135c6aa..2855d06acd 100644 --- a/processor/internal/user_transformer/user_transformer.go +++ b/processor/internal/user_transformer/user_transformer.go @@ -12,12 +12,13 @@ import ( "sync" "time" + "github.com/rudderlabs/rudder-server/utils/httputil" + "github.com/cenkalti/backoff/v4" jsoniter "github.com/json-iterator/go" "github.com/samber/lo" "github.com/rudderlabs/rudder-go-kit/config" - "github.com/rudderlabs/rudder-go-kit/httputil" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" @@ -302,14 +303,13 @@ func (u *UserTransformer) doPost(ctx context.Context, rawJSON []byte, url string req.Header.Set("X-Feature-Filter-Code", "?1") resp, reqErr = u.client.Do(req) + defer func() { httputil.CloseResponse(resp) }() }) u.stat.NewTaggedStat("processor.transformer_request_time", stats.TimerType, tags).SendTiming(time.Since(requestStartTime)) if reqErr != nil { return reqErr } - defer func() { httputil.CloseResponse(resp) }() - if !transformer_utils.IsJobTerminated(resp.StatusCode) && resp.StatusCode != transformer_utils.StatusCPDown { return fmt.Errorf("transformer returned status code: %v", resp.StatusCode) } From da374131e8323a8e88718c2e0bbebdb366718fac Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Mon, 20 Jan 2025 14:54:18 +0530 Subject: [PATCH 6/8] chore: add module tp validator --- .../destination_transformer.go | 5 - .../trackingplan_validation.go | 302 +++++++++++++++++- 2 files changed, 296 insertions(+), 11 deletions(-) diff --git a/processor/internal/destination_transformer/destination_transformer.go b/processor/internal/destination_transformer/destination_transformer.go index 520301a0d1..5e46b45908 100644 --- a/processor/internal/destination_transformer/destination_transformer.go +++ b/processor/internal/destination_transformer/destination_transformer.go @@ -209,11 +209,6 @@ func (d *DestTransformer) request(ctx context.Context, url, stage string, data [ "dest_id": data[0].Destination.ID, "src_id": data[0].Metadata.SourceID, }) - if statusCode == transformer_utils.StatusCPDown { - d.stat.NewStat("processor.control_plane_down", stats.GaugeType).Gauge(1) - return fmt.Errorf("control plane not reachable") - } - d.stat.NewStat("processor.control_plane_down", stats.GaugeType).Gauge(0) return nil }, endlessBackoff, diff --git a/processor/internal/trackingplan_validation/trackingplan_validation.go b/processor/internal/trackingplan_validation/trackingplan_validation.go index 3c25237e6e..f966404bd5 100644 --- a/processor/internal/trackingplan_validation/trackingplan_validation.go +++ b/processor/internal/trackingplan_validation/trackingplan_validation.go @@ -1,9 +1,25 @@ package trackingplan_validation import ( + "bytes" "context" + "encoding/json" + "fmt" + "io" + "net/http" + "runtime/trace" + "strconv" + "sync" "time" + "github.com/cenkalti/backoff" + "github.com/samber/lo" + + "github.com/rudderlabs/rudder-server/processor/integrations" + "github.com/rudderlabs/rudder-server/processor/internal/transformer_utils" + "github.com/rudderlabs/rudder-server/utils/httputil" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" + "github.com/rudderlabs/rudder-server/processor/types" "github.com/rudderlabs/rudder-go-kit/config" @@ -18,17 +34,19 @@ type TPValidator struct { destTransformationURL string maxRetry config.ValueLoader[int] failOnError config.ValueLoader[bool] + maxConcurrency int maxRetryBackoffInterval config.ValueLoader[time.Duration] + timeoutDuration time.Duration } - conf *config.Config - log logger.Logger - stat stats.Stats - client http_client.HTTPDoer + conf *config.Config + log logger.Logger + stat stats.Stats + guardConcurrency chan struct{} + client http_client.HTTPDoer } func (t *TPValidator) SendRequest(ctx context.Context, clientEvents []types.TransformerEvent, batchSize int) types.Response { - _ = t.trackingPlanValidationURL() - return types.Response{} + return t.transform(ctx, clientEvents, t.trackingPlanValidationURL(), batchSize) } func NewTPValidator(conf *config.Config, log logger.Logger, stat stats.Stats) *TPValidator { @@ -38,12 +56,284 @@ func NewTPValidator(conf *config.Config, log logger.Logger, stat stats.Stats) *T handle.stat = stat handle.client = http_client.NewHTTPClient(conf) handle.config.destTransformationURL = handle.conf.GetString("Warehouse.destTransformationURL", "http://localhost:9090") + handle.config.maxConcurrency = conf.GetInt("Processor.maxConcurrency", 200) + handle.guardConcurrency = make(chan struct{}, handle.config.maxConcurrency) handle.config.maxRetry = conf.GetReloadableIntVar(30, 1, "Processor.maxRetry") + handle.config.timeoutDuration = conf.GetDuration("HttpClient.procTransformer.timeout", 600, time.Second) handle.config.failOnError = conf.GetReloadableBoolVar(false, "Processor.Transformer.failOnError") handle.config.maxRetryBackoffInterval = conf.GetReloadableDurationVar(30, time.Second, "Processor.maxRetryBackoffInterval") return handle } +func (t *TPValidator) transform( + ctx context.Context, + clientEvents []types.TransformerEvent, + url string, + batchSize int, +) types.Response { + if len(clientEvents) == 0 { + return types.Response{} + } + // flip sourceID and originalSourceID if it's a replay source for the purpose of any user transformation + // flip back afterward + for i := range clientEvents { + if clientEvents[i].Metadata.OriginalSourceID != "" { + clientEvents[i].Metadata.OriginalSourceID, clientEvents[i].Metadata.SourceID = clientEvents[i].Metadata.SourceID, clientEvents[i].Metadata.OriginalSourceID + } + } + sTags := stats.Tags{ + "dest_type": clientEvents[0].Destination.DestinationDefinition.Name, + "dest_id": clientEvents[0].Destination.ID, + "src_id": clientEvents[0].Metadata.SourceID, + "stage": "dest_transformer", + } + + var trackWg sync.WaitGroup + defer trackWg.Wait() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + trackWg.Add(1) + go func() { + var loggerCtx []interface{} + for k, v := range sTags { + loggerCtx = append(loggerCtx, k, v) + } + transformer_utils.TrackLongRunningTransformation(ctx, "dest_transformer", t.config.timeoutDuration, t.log.With(loggerCtx...)) + trackWg.Done() + }() + + batches := lo.Chunk(clientEvents, batchSize) + + t.stat.NewTaggedStat( + "processor.transformer_request_batch_count", + stats.HistogramType, + sTags, + ).Observe(float64(len(batches))) + trace.Logf(ctx, "request", "batch_count: %d", len(batches)) + + transformResponse := make([][]types.TransformerResponse, len(batches)) + + var wg sync.WaitGroup + wg.Add(len(batches)) + + lo.ForEach( + batches, + func(batch []types.TransformerEvent, i int) { + t.guardConcurrency <- struct{}{} + go func() { + trace.WithRegion(ctx, "request", func() { + transformResponse[i] = t.request(ctx, url, "dest_transformer", batch) + }) + <-t.guardConcurrency + wg.Done() + }() + }, + ) + wg.Wait() + + var outClientEvents []types.TransformerResponse + var failedEvents []types.TransformerResponse + + for _, batch := range transformResponse { + // Transform is one to many mapping so returned + // response for each is an array. We flatten it out + for _, transformerResponse := range batch { + if transformerResponse.Metadata.OriginalSourceID != "" { + transformerResponse.Metadata.SourceID, transformerResponse.Metadata.OriginalSourceID = transformerResponse.Metadata.OriginalSourceID, transformerResponse.Metadata.SourceID + } + switch transformerResponse.StatusCode { + case http.StatusOK: + outClientEvents = append(outClientEvents, transformerResponse) + default: + failedEvents = append(failedEvents, transformerResponse) + } + } + } + + t.stat.NewStat("processor.transformer_sent", stats.CountType).Count(len(clientEvents)) + t.stat.NewStat("processor.transformer_received", stats.CountType).Count(len(outClientEvents)) + + return types.Response{ + Events: outClientEvents, + FailedEvents: failedEvents, + } +} + +func (t *TPValidator) request(ctx context.Context, url, stage string, data []types.TransformerEvent) []types.TransformerResponse { + // Call remote transformation + var ( + rawJSON []byte + err error + ) + + trace.WithRegion(ctx, "marshal", func() { + rawJSON, err = json.Marshal(data) + }) + trace.Logf(ctx, "marshal", "request raw body size: %d", len(rawJSON)) + if err != nil { + panic(err) + } + + if len(data) == 0 { + return nil + } + + var ( + respData []byte + statusCode int + ) + + // endless retry if transformer-control plane connection is down + endlessBackoff := backoff.NewExponentialBackOff() + endlessBackoff.MaxElapsedTime = 0 // no max time -> ends only when no error + endlessBackoff.MaxInterval = t.config.maxRetryBackoffInterval.Load() + + // endless backoff loop, only nil error or panics inside + _ = backoff.RetryNotify( + func() error { + transformationID := "" + if len(data[0].Destination.Transformations) > 0 { + transformationID = data[0].Destination.Transformations[0].ID + } + + respData, statusCode = t.doPost(ctx, rawJSON, url, stats.Tags{ + "destinationType": data[0].Destination.DestinationDefinition.Name, + "destinationId": data[0].Destination.ID, + "sourceId": data[0].Metadata.SourceID, + "transformationId": transformationID, + "stage": stage, + + // Legacy tags: to be removed + "dest_type": data[0].Destination.DestinationDefinition.Name, + "dest_id": data[0].Destination.ID, + "src_id": data[0].Metadata.SourceID, + }) + return nil + }, + endlessBackoff, + func(err error, time time.Duration) { + var transformationID, transformationVersionID string + if len(data[0].Destination.Transformations) > 0 { + transformationID = data[0].Destination.Transformations[0].ID + transformationVersionID = data[0].Destination.Transformations[0].VersionID + } + t.log.Errorf("JS HTTP connection error: URL: %v Error: %+v. WorkspaceID: %s, sourceID: %s, destinationID: %s, transformationID: %s, transformationVersionID: %s", + url, err, data[0].Metadata.WorkspaceID, data[0].Metadata.SourceID, data[0].Metadata.DestinationID, + transformationID, transformationVersionID, + ) + }, + ) + // control plane back up + + switch statusCode { + case http.StatusOK, + http.StatusBadRequest, + http.StatusNotFound, + http.StatusRequestEntityTooLarge: + default: + t.log.Errorf("Transformer returned status code: %v", statusCode) + } + + var transformerResponses []types.TransformerResponse + switch statusCode { + case http.StatusOK: + integrations.CollectIntgTransformErrorStats(respData) + + trace.Logf(ctx, "Unmarshal", "response raw size: %d", len(respData)) + trace.WithRegion(ctx, "Unmarshal", func() { + err = json.Unmarshal(respData, &transformerResponses) + }) + // This is returned by our JS engine so should be parsable + // Panic the processor to avoid replays + if err != nil { + t.log.Errorf("Data sent to transformer : %v", string(rawJSON)) + t.log.Errorf("Transformer returned : %v", string(respData)) + panic(err) + } + default: + for i := range data { + transformEvent := &data[i] + resp := types.TransformerResponse{StatusCode: statusCode, Error: string(respData), Metadata: transformEvent.Metadata} + transformerResponses = append(transformerResponses, resp) + } + } + return transformerResponses +} + +func (t *TPValidator) doPost(ctx context.Context, rawJSON []byte, url string, tags stats.Tags) ([]byte, int) { + var ( + retryCount int + resp *http.Response + respData []byte + ) + retryStrategy := backoff.NewExponentialBackOff() + // MaxInterval caps the RetryInterval + retryStrategy.MaxInterval = t.config.maxRetryBackoffInterval.Load() + + err := backoff.RetryNotify( + func() error { + var reqErr error + requestStartTime := time.Now() + + trace.WithRegion(ctx, "request/post", func() { + var req *http.Request + req, reqErr = http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(rawJSON)) + if reqErr != nil { + return + } + + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("X-Feature-Gzip-Support", "?1") + // Header to let transformer know that the client understands event filter code + req.Header.Set("X-Feature-Filter-Code", "?1") + + resp, reqErr = t.client.Do(req) + defer func() { httputil.CloseResponse(resp) }() + }) + t.stat.NewTaggedStat("processor.transformer_request_time", stats.TimerType, tags).SendTiming(time.Since(requestStartTime)) + if reqErr != nil { + return reqErr + } + + if !transformer_utils.IsJobTerminated(resp.StatusCode) && resp.StatusCode != transformer_utils.StatusCPDown { + return fmt.Errorf("transformer returned status code: %v", resp.StatusCode) + } + + respData, reqErr = io.ReadAll(resp.Body) + return reqErr + }, + backoff.WithMaxRetries(retryStrategy, uint64(t.config.maxRetry.Load())), + func(err error, time time.Duration) { + retryCount++ + t.log.Warnn( + "JS HTTP connection error", + logger.NewErrorField(err), + logger.NewIntField("attempts", int64(retryCount)), + ) + }, + ) + if err != nil { + if t.config.failOnError.Load() { + return []byte(fmt.Sprintf("transformer request failed: %s", err)), transformer_utils.TransformerRequestFailure + } else { + panic(err) + } + } + + // perform version compatibility check only on success + if resp.StatusCode == http.StatusOK { + transformerAPIVersion, _ := strconv.Atoi(resp.Header.Get("apiVersion")) + if reportingTypes.SupportedTransformerApiVersion != transformerAPIVersion { + unexpectedVersionError := fmt.Errorf("incompatible transformer version: Expected: %d Received: %s, URL: %v", reportingTypes.SupportedTransformerApiVersion, resp.Header.Get("apiVersion"), url) + t.log.Error(unexpectedVersionError) + panic(unexpectedVersionError) + } + } + + return respData, resp.StatusCode +} + func (t *TPValidator) trackingPlanValidationURL() string { return t.config.destTransformationURL + "/v0/validate" } From 4bf68a2db6d25b3da1cb0fec0150c31f65aad487 Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Mon, 20 Jan 2025 15:24:45 +0530 Subject: [PATCH 7/8] chore: remove swap original src id and src id --- .../destination_transformer.go | 15 --------- .../trackingplan_validation.go | 33 ++++++++----------- 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/processor/internal/destination_transformer/destination_transformer.go b/processor/internal/destination_transformer/destination_transformer.go index 5e46b45908..0a333cf741 100644 --- a/processor/internal/destination_transformer/destination_transformer.go +++ b/processor/internal/destination_transformer/destination_transformer.go @@ -74,13 +74,6 @@ func (d *DestTransformer) transform( if len(clientEvents) == 0 { return types.Response{} } - // flip sourceID and originalSourceID if it's a replay source for the purpose of any user transformation - // flip back afterward - for i := range clientEvents { - if clientEvents[i].Metadata.OriginalSourceID != "" { - clientEvents[i].Metadata.OriginalSourceID, clientEvents[i].Metadata.SourceID = clientEvents[i].Metadata.SourceID, clientEvents[i].Metadata.OriginalSourceID - } - } sTags := stats.Tags{ "dest_type": clientEvents[0].Destination.DestinationDefinition.Name, "dest_id": clientEvents[0].Destination.ID, @@ -139,9 +132,6 @@ func (d *DestTransformer) transform( // Transform is one to many mapping so returned // response for each is an array. We flatten it out for _, transformerResponse := range batch { - if transformerResponse.Metadata.OriginalSourceID != "" { - transformerResponse.Metadata.SourceID, transformerResponse.Metadata.OriginalSourceID = transformerResponse.Metadata.OriginalSourceID, transformerResponse.Metadata.SourceID - } switch transformerResponse.StatusCode { case http.StatusOK: outClientEvents = append(outClientEvents, transformerResponse) @@ -295,11 +285,6 @@ func (d *DestTransformer) doPost(ctx context.Context, rawJSON []byte, url string if reqErr != nil { return reqErr } - - if !transformer_utils.IsJobTerminated(resp.StatusCode) && resp.StatusCode != transformer_utils.StatusCPDown { - return fmt.Errorf("transformer returned status code: %v", resp.StatusCode) - } - respData, reqErr = io.ReadAll(resp.Body) return reqErr }, diff --git a/processor/internal/trackingplan_validation/trackingplan_validation.go b/processor/internal/trackingplan_validation/trackingplan_validation.go index f966404bd5..0fc9d9f179 100644 --- a/processor/internal/trackingplan_validation/trackingplan_validation.go +++ b/processor/internal/trackingplan_validation/trackingplan_validation.go @@ -15,18 +15,16 @@ import ( "github.com/cenkalti/backoff" "github.com/samber/lo" - "github.com/rudderlabs/rudder-server/processor/integrations" - "github.com/rudderlabs/rudder-server/processor/internal/transformer_utils" - "github.com/rudderlabs/rudder-server/utils/httputil" - reportingTypes "github.com/rudderlabs/rudder-server/utils/types" - - "github.com/rudderlabs/rudder-server/processor/types" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" "github.com/rudderlabs/rudder-go-kit/stats" + "github.com/rudderlabs/rudder-server/processor/integrations" "github.com/rudderlabs/rudder-server/processor/internal/http_client" + "github.com/rudderlabs/rudder-server/processor/internal/transformer_utils" + "github.com/rudderlabs/rudder-server/processor/types" + "github.com/rudderlabs/rudder-server/utils/httputil" + reportingTypes "github.com/rudderlabs/rudder-server/utils/types" ) type TPValidator struct { @@ -74,18 +72,11 @@ func (t *TPValidator) transform( if len(clientEvents) == 0 { return types.Response{} } - // flip sourceID and originalSourceID if it's a replay source for the purpose of any user transformation - // flip back afterward - for i := range clientEvents { - if clientEvents[i].Metadata.OriginalSourceID != "" { - clientEvents[i].Metadata.OriginalSourceID, clientEvents[i].Metadata.SourceID = clientEvents[i].Metadata.SourceID, clientEvents[i].Metadata.OriginalSourceID - } - } sTags := stats.Tags{ "dest_type": clientEvents[0].Destination.DestinationDefinition.Name, "dest_id": clientEvents[0].Destination.ID, "src_id": clientEvents[0].Metadata.SourceID, - "stage": "dest_transformer", + "stage": "trackingPlan_validation", } var trackWg sync.WaitGroup @@ -99,7 +90,7 @@ func (t *TPValidator) transform( for k, v := range sTags { loggerCtx = append(loggerCtx, k, v) } - transformer_utils.TrackLongRunningTransformation(ctx, "dest_transformer", t.config.timeoutDuration, t.log.With(loggerCtx...)) + transformer_utils.TrackLongRunningTransformation(ctx, "trackingPlan_validation", t.config.timeoutDuration, t.log.With(loggerCtx...)) trackWg.Done() }() @@ -123,7 +114,7 @@ func (t *TPValidator) transform( t.guardConcurrency <- struct{}{} go func() { trace.WithRegion(ctx, "request", func() { - transformResponse[i] = t.request(ctx, url, "dest_transformer", batch) + transformResponse[i] = t.request(ctx, url, "trackingPlan_validation", batch) }) <-t.guardConcurrency wg.Done() @@ -139,9 +130,6 @@ func (t *TPValidator) transform( // Transform is one to many mapping so returned // response for each is an array. We flatten it out for _, transformerResponse := range batch { - if transformerResponse.Metadata.OriginalSourceID != "" { - transformerResponse.Metadata.SourceID, transformerResponse.Metadata.OriginalSourceID = transformerResponse.Metadata.OriginalSourceID, transformerResponse.Metadata.SourceID - } switch transformerResponse.StatusCode { case http.StatusOK: outClientEvents = append(outClientEvents, transformerResponse) @@ -209,6 +197,11 @@ func (t *TPValidator) request(ctx context.Context, url, stage string, data []typ "dest_id": data[0].Destination.ID, "src_id": data[0].Metadata.SourceID, }) + if statusCode == transformer_utils.StatusCPDown { + t.stat.NewStat("processor.control_plane_down", stats.GaugeType).Gauge(1) + return fmt.Errorf("control plane not reachable") + } + t.stat.NewStat("processor.control_plane_down", stats.GaugeType).Gauge(0) return nil }, endlessBackoff, From f01fe431eea6be855f03766a5ea5c7e0dd511433 Mon Sep 17 00:00:00 2001 From: Rohith BCS Date: Wed, 22 Jan 2025 14:16:58 +0530 Subject: [PATCH 8/8] chore: review comments --- .../destination_transformer.go | 93 +---- .../trackingplan_validation.go | 46 +-- .../user_transformer/user_transformer.go | 41 +- processor/processor.go | 378 +++++++++--------- processor/trackingplan.go | 12 +- 5 files changed, 256 insertions(+), 314 deletions(-) diff --git a/processor/internal/destination_transformer/destination_transformer.go b/processor/internal/destination_transformer/destination_transformer.go index 0a333cf741..484ac81fa8 100644 --- a/processor/internal/destination_transformer/destination_transformer.go +++ b/processor/internal/destination_transformer/destination_transformer.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/http" - "runtime/trace" "strconv" "strings" "sync" @@ -103,7 +102,6 @@ func (d *DestTransformer) transform( stats.HistogramType, sTags, ).Observe(float64(len(batches))) - trace.Logf(ctx, "request", "batch_count: %d", len(batches)) transformResponse := make([][]types.TransformerResponse, len(batches)) @@ -115,9 +113,7 @@ func (d *DestTransformer) transform( func(batch []types.TransformerEvent, i int) { d.guardConcurrency <- struct{}{} go func() { - trace.WithRegion(ctx, "request", func() { - transformResponse[i] = d.request(ctx, url, "dest_transformer", batch) - }) + transformResponse[i] = d.request(ctx, url, "dest_transformer", batch) <-d.guardConcurrency wg.Done() }() @@ -157,10 +153,7 @@ func (d *DestTransformer) request(ctx context.Context, url, stage string, data [ err error ) - trace.WithRegion(ctx, "marshal", func() { - rawJSON, err = json.Marshal(data) - }) - trace.Logf(ctx, "marshal", "request raw body size: %d", len(rawJSON)) + rawJSON, err = json.Marshal(data) if err != nil { panic(err) } @@ -174,47 +167,12 @@ func (d *DestTransformer) request(ctx context.Context, url, stage string, data [ statusCode int ) - // endless retry if transformer-control plane connection is down - endlessBackoff := backoff.NewExponentialBackOff() - endlessBackoff.MaxElapsedTime = 0 // no max time -> ends only when no error - endlessBackoff.MaxInterval = d.config.maxRetryBackoffInterval.Load() - - // endless backoff loop, only nil error or panics inside - _ = backoff.RetryNotify( - func() error { - transformationID := "" - if len(data[0].Destination.Transformations) > 0 { - transformationID = data[0].Destination.Transformations[0].ID - } - - respData, statusCode = d.doPost(ctx, rawJSON, url, stats.Tags{ - "destinationType": data[0].Destination.DestinationDefinition.Name, - "destinationId": data[0].Destination.ID, - "sourceId": data[0].Metadata.SourceID, - "transformationId": transformationID, - "stage": stage, - - // Legacy tags: to be removed - "dest_type": data[0].Destination.DestinationDefinition.Name, - "dest_id": data[0].Destination.ID, - "src_id": data[0].Metadata.SourceID, - }) - return nil - }, - endlessBackoff, - func(err error, t time.Duration) { - var transformationID, transformationVersionID string - if len(data[0].Destination.Transformations) > 0 { - transformationID = data[0].Destination.Transformations[0].ID - transformationVersionID = data[0].Destination.Transformations[0].VersionID - } - d.log.Errorf("JS HTTP connection error: URL: %v Error: %+v. WorkspaceID: %s, sourceID: %s, destinationID: %s, transformationID: %s, transformationVersionID: %s", - url, err, data[0].Metadata.WorkspaceID, data[0].Metadata.SourceID, data[0].Metadata.DestinationID, - transformationID, transformationVersionID, - ) - }, - ) - // control plane back up + respData, statusCode = d.doPost(ctx, rawJSON, url, stats.Tags{ + "destinationType": data[0].Destination.DestinationDefinition.Name, + "destinationId": data[0].Destination.ID, + "sourceId": data[0].Metadata.SourceID, + "stage": stage, + }) switch statusCode { case http.StatusOK, @@ -230,15 +188,10 @@ func (d *DestTransformer) request(ctx context.Context, url, stage string, data [ case http.StatusOK: integrations.CollectIntgTransformErrorStats(respData) - trace.Logf(ctx, "Unmarshal", "response raw size: %d", len(respData)) - trace.WithRegion(ctx, "Unmarshal", func() { - err = json.Unmarshal(respData, &transformerResponses) - }) + err = json.Unmarshal(respData, &transformerResponses) // This is returned by our JS engine so should be parsable // Panic the processor to avoid replays if err != nil { - d.log.Errorf("Data sent to transformer : %v", string(rawJSON)) - d.log.Errorf("Transformer returned : %v", string(respData)) panic(err) } default: @@ -266,21 +219,19 @@ func (d *DestTransformer) doPost(ctx context.Context, rawJSON []byte, url string var reqErr error requestStartTime := time.Now() - trace.WithRegion(ctx, "request/post", func() { - var req *http.Request - req, reqErr = http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(rawJSON)) - if reqErr != nil { - return - } - - req.Header.Set("Content-Type", "application/json; charset=utf-8") - req.Header.Set("X-Feature-Gzip-Support", "?1") - // Header to let transformer know that the client understands event filter code - req.Header.Set("X-Feature-Filter-Code", "?1") - - resp, reqErr = d.client.Do(req) - defer func() { httputil.CloseResponse(resp) }() - }) + var req *http.Request + req, reqErr = http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(rawJSON)) + if reqErr != nil { + return reqErr + } + + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("X-Feature-Gzip-Support", "?1") + // Header to let transformer know that the client understands event filter code + req.Header.Set("X-Feature-Filter-Code", "?1") + + resp, reqErr = d.client.Do(req) + defer func() { httputil.CloseResponse(resp) }() d.stat.NewTaggedStat("processor.transformer_request_time", stats.TimerType, tags).SendTiming(time.Since(requestStartTime)) if reqErr != nil { return reqErr diff --git a/processor/internal/trackingplan_validation/trackingplan_validation.go b/processor/internal/trackingplan_validation/trackingplan_validation.go index 0fc9d9f179..4bc8b11c06 100644 --- a/processor/internal/trackingplan_validation/trackingplan_validation.go +++ b/processor/internal/trackingplan_validation/trackingplan_validation.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/http" - "runtime/trace" "strconv" "sync" "time" @@ -101,7 +100,6 @@ func (t *TPValidator) transform( stats.HistogramType, sTags, ).Observe(float64(len(batches))) - trace.Logf(ctx, "request", "batch_count: %d", len(batches)) transformResponse := make([][]types.TransformerResponse, len(batches)) @@ -113,9 +111,7 @@ func (t *TPValidator) transform( func(batch []types.TransformerEvent, i int) { t.guardConcurrency <- struct{}{} go func() { - trace.WithRegion(ctx, "request", func() { - transformResponse[i] = t.request(ctx, url, "trackingPlan_validation", batch) - }) + transformResponse[i] = t.request(ctx, url, "trackingPlan_validation", batch) <-t.guardConcurrency wg.Done() }() @@ -155,10 +151,7 @@ func (t *TPValidator) request(ctx context.Context, url, stage string, data []typ err error ) - trace.WithRegion(ctx, "marshal", func() { - rawJSON, err = json.Marshal(data) - }) - trace.Logf(ctx, "marshal", "request raw body size: %d", len(rawJSON)) + rawJSON, err = json.Marshal(data) if err != nil { panic(err) } @@ -191,11 +184,6 @@ func (t *TPValidator) request(ctx context.Context, url, stage string, data []typ "sourceId": data[0].Metadata.SourceID, "transformationId": transformationID, "stage": stage, - - // Legacy tags: to be removed - "dest_type": data[0].Destination.DestinationDefinition.Name, - "dest_id": data[0].Destination.ID, - "src_id": data[0].Metadata.SourceID, }) if statusCode == transformer_utils.StatusCPDown { t.stat.NewStat("processor.control_plane_down", stats.GaugeType).Gauge(1) @@ -232,11 +220,7 @@ func (t *TPValidator) request(ctx context.Context, url, stage string, data []typ switch statusCode { case http.StatusOK: integrations.CollectIntgTransformErrorStats(respData) - - trace.Logf(ctx, "Unmarshal", "response raw size: %d", len(respData)) - trace.WithRegion(ctx, "Unmarshal", func() { - err = json.Unmarshal(respData, &transformerResponses) - }) + err = json.Unmarshal(respData, &transformerResponses) // This is returned by our JS engine so should be parsable // Panic the processor to avoid replays if err != nil { @@ -269,21 +253,19 @@ func (t *TPValidator) doPost(ctx context.Context, rawJSON []byte, url string, ta var reqErr error requestStartTime := time.Now() - trace.WithRegion(ctx, "request/post", func() { - var req *http.Request - req, reqErr = http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(rawJSON)) - if reqErr != nil { - return - } + var req *http.Request + req, reqErr = http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(rawJSON)) + if reqErr != nil { + return reqErr + } - req.Header.Set("Content-Type", "application/json; charset=utf-8") - req.Header.Set("X-Feature-Gzip-Support", "?1") - // Header to let transformer know that the client understands event filter code - req.Header.Set("X-Feature-Filter-Code", "?1") + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("X-Feature-Gzip-Support", "?1") + // Header to let transformer know that the client understands event filter code + req.Header.Set("X-Feature-Filter-Code", "?1") - resp, reqErr = t.client.Do(req) - defer func() { httputil.CloseResponse(resp) }() - }) + resp, reqErr = t.client.Do(req) + defer func() { httputil.CloseResponse(resp) }() t.stat.NewTaggedStat("processor.transformer_request_time", stats.TimerType, tags).SendTiming(time.Since(requestStartTime)) if reqErr != nil { return reqErr diff --git a/processor/internal/user_transformer/user_transformer.go b/processor/internal/user_transformer/user_transformer.go index 2855d06acd..32e8461c1a 100644 --- a/processor/internal/user_transformer/user_transformer.go +++ b/processor/internal/user_transformer/user_transformer.go @@ -7,7 +7,6 @@ import ( "io" "net/http" "os" - "runtime/trace" "strconv" "sync" "time" @@ -119,7 +118,6 @@ func (u *UserTransformer) transform( stats.HistogramType, sTags, ).Observe(float64(len(batches))) - trace.Logf(ctx, "request", "batch_count: %d", len(batches)) transformResponse := make([][]types.TransformerResponse, len(batches)) @@ -131,9 +129,7 @@ func (u *UserTransformer) transform( func(batch []types.TransformerEvent, i int) { u.guardConcurrency <- struct{}{} go func() { - trace.WithRegion(ctx, "request", func() { - transformResponse[i] = u.request(ctx, url, "user_transformer", batch) - }) + transformResponse[i] = u.request(ctx, url, "user_transformer", batch) <-u.guardConcurrency wg.Done() }() @@ -176,10 +172,7 @@ func (u *UserTransformer) request(ctx context.Context, url, stage string, data [ err error ) - trace.WithRegion(ctx, "marshal", func() { - rawJSON, err = json.Marshal(data) - }) - trace.Logf(ctx, "marshal", "request raw body size: %d", len(rawJSON)) + rawJSON, err = json.Marshal(data) if err != nil { panic(err) } @@ -253,11 +246,7 @@ func (u *UserTransformer) request(ctx context.Context, url, stage string, data [ switch statusCode { case http.StatusOK: integrations.CollectIntgTransformErrorStats(respData) - - trace.Logf(ctx, "Unmarshal", "response raw size: %d", len(respData)) - trace.WithRegion(ctx, "Unmarshal", func() { - err = json.Unmarshal(respData, &transformerResponses) - }) + err = json.Unmarshal(respData, &transformerResponses) // This is returned by our JS engine so should be parsable // Panic the processor to avoid replays if err != nil { @@ -290,21 +279,19 @@ func (u *UserTransformer) doPost(ctx context.Context, rawJSON []byte, url string var reqErr error requestStartTime := time.Now() - trace.WithRegion(ctx, "request/post", func() { - var req *http.Request - req, reqErr = http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(rawJSON)) - if reqErr != nil { - return - } + var req *http.Request + req, reqErr = http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(rawJSON)) + if reqErr != nil { + return reqErr + } - req.Header.Set("Content-Type", "application/json; charset=utf-8") - req.Header.Set("X-Feature-Gzip-Support", "?1") - // Header to let transformer know that the client understands event filter code - req.Header.Set("X-Feature-Filter-Code", "?1") + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("X-Feature-Gzip-Support", "?1") + // Header to let transformer know that the client understands event filter code + req.Header.Set("X-Feature-Filter-Code", "?1") - resp, reqErr = u.client.Do(req) - defer func() { httputil.CloseResponse(resp) }() - }) + resp, reqErr = u.client.Do(req) + defer func() { httputil.CloseResponse(resp) }() u.stat.NewTaggedStat("processor.transformer_request_time", stats.TimerType, tags).SendTiming(time.Since(requestStartTime)) if reqErr != nil { return reqErr diff --git a/processor/processor.go b/processor/processor.go index 2de2bd2650..31c41f161c 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -160,6 +160,7 @@ type Handle struct { eventAuditEnabled map[string]bool credentialsMap map[string][]types.Credential nonEventStreamSources map[string]bool + enableTransformationV2 bool } drainConfig struct { @@ -809,6 +810,7 @@ func (proc *Handle) loadConfig() { // GWCustomVal is used as a key in the jobsDB customval column proc.config.GWCustomVal = config.GetStringVar("GW", "Gateway.CustomVal") proc.config.enableParallelScan = config.GetBoolVar(false, "Dedup.enableParallelScan") + proc.config.enableTransformationV2 = config.GetBoolVar(false, "Processor.enableTransformationV2") proc.loadReloadableConfig(defaultPayloadLimit, defaultMaxEventsToProcess) } @@ -3060,54 +3062,60 @@ func (proc *Handle) transformSrcDest( userTransformationStat.numEvents.Count(len(eventList)) proc.logger.Debug("Custom Transform input size", len(eventList)) - trace.WithRegion(ctx, "UserTransform", func() { - startedAt := time.Now() + startedAt := time.Now() + if !proc.config.enableTransformationV2 { response = proc.transformer.UserTransform(ctx, eventList, proc.config.userTransformBatchSize.Load()) - d := time.Since(startedAt) - userTransformationStat.transformTime.SendTiming(d) - - var successMetrics []*reportingTypes.PUReportedMetric - var successCountMap map[string]int64 - var successCountMetadataMap map[string]MetricMetadata - eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, reportingTypes.USER_TRANSFORMER) - nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, reportingTypes.USER_TRANSFORMER) - droppedJobs = append(droppedJobs, append(proc.getDroppedJobs(response, eventList), append(nonSuccessMetrics.failedJobs, nonSuccessMetrics.filteredJobs...)...)...) - if _, ok := procErrorJobsByDestID[destID]; !ok { - procErrorJobsByDestID[destID] = make([]*jobsdb.JobT, 0) + } else { + client, err := proc.transformerManager.GetServiceClient("user_transformer") + if err != nil { + proc.logger.Error("Error getting destination transformer client", err) + panic(err) } - procErrorJobsByDestID[destID] = append(procErrorJobsByDestID[destID], nonSuccessMetrics.failedJobs...) - userTransformationStat.numOutputSuccessEvents.Count(len(eventsToTransform)) - userTransformationStat.numOutputFailedEvents.Count(len(nonSuccessMetrics.failedJobs)) - userTransformationStat.numOutputFilteredEvents.Count(len(nonSuccessMetrics.filteredJobs)) - proc.logger.Debug("Custom Transform output size", len(eventsToTransform)) - trace.Logf(ctx, "UserTransform", "User Transform output size: %d", len(eventsToTransform)) + response = client.SendRequest(ctx, eventsToTransform, proc.config.transformBatchSize.Load()) + } + d := time.Since(startedAt) + userTransformationStat.transformTime.SendTiming(d) - proc.transDebugger.UploadTransformationStatus(&transformationdebugger.TransformationStatusT{SourceID: sourceID, DestID: destID, Destination: destination, UserTransformedEvents: eventsToTransform, EventsByMessageID: eventsByMessageID, FailedEvents: response.FailedEvents, UniqueMessageIds: uniqueMessageIdsBySrcDestKey[srcAndDestKey]}) + var successMetrics []*reportingTypes.PUReportedMetric + var successCountMap map[string]int64 + var successCountMetadataMap map[string]MetricMetadata + eventsToTransform, successMetrics, successCountMap, successCountMetadataMap = proc.getTransformerEvents(response, commonMetaData, eventsByMessageID, destination, connection, inPU, reportingTypes.USER_TRANSFORMER) + nonSuccessMetrics := proc.getNonSuccessfulMetrics(response, commonMetaData, eventsByMessageID, inPU, reportingTypes.USER_TRANSFORMER) + droppedJobs = append(droppedJobs, append(proc.getDroppedJobs(response, eventList), append(nonSuccessMetrics.failedJobs, nonSuccessMetrics.filteredJobs...)...)...) + if _, ok := procErrorJobsByDestID[destID]; !ok { + procErrorJobsByDestID[destID] = make([]*jobsdb.JobT, 0) + } + procErrorJobsByDestID[destID] = append(procErrorJobsByDestID[destID], nonSuccessMetrics.failedJobs...) + userTransformationStat.numOutputSuccessEvents.Count(len(eventsToTransform)) + userTransformationStat.numOutputFailedEvents.Count(len(nonSuccessMetrics.failedJobs)) + userTransformationStat.numOutputFilteredEvents.Count(len(nonSuccessMetrics.filteredJobs)) + proc.logger.Debug("Custom Transform output size", len(eventsToTransform)) - // REPORTING - START - if proc.isReportingEnabled() { - diffMetrics := getDiffMetrics( - reportingTypes.DESTINATION_FILTER, - reportingTypes.USER_TRANSFORMER, - inCountMetadataMap, - inCountMap, - successCountMap, - nonSuccessMetrics.failedCountMap, - nonSuccessMetrics.filteredCountMap, - proc.statsFactory, - ) - reportMetrics = append(reportMetrics, successMetrics...) - reportMetrics = append(reportMetrics, nonSuccessMetrics.failedMetrics...) - reportMetrics = append(reportMetrics, nonSuccessMetrics.filteredMetrics...) - reportMetrics = append(reportMetrics, diffMetrics...) - - // successCountMap will be inCountMap for filtering events based on supported event types - inCountMap = successCountMap - inCountMetadataMap = successCountMetadataMap - } - // REPORTING - END - inPU = reportingTypes.USER_TRANSFORMER // for the next step in the pipeline - }) + proc.transDebugger.UploadTransformationStatus(&transformationdebugger.TransformationStatusT{SourceID: sourceID, DestID: destID, Destination: destination, UserTransformedEvents: eventsToTransform, EventsByMessageID: eventsByMessageID, FailedEvents: response.FailedEvents, UniqueMessageIds: uniqueMessageIdsBySrcDestKey[srcAndDestKey]}) + + // REPORTING - START + if proc.isReportingEnabled() { + diffMetrics := getDiffMetrics( + reportingTypes.DESTINATION_FILTER, + reportingTypes.USER_TRANSFORMER, + inCountMetadataMap, + inCountMap, + successCountMap, + nonSuccessMetrics.failedCountMap, + nonSuccessMetrics.filteredCountMap, + proc.statsFactory, + ) + reportMetrics = append(reportMetrics, successMetrics...) + reportMetrics = append(reportMetrics, nonSuccessMetrics.failedMetrics...) + reportMetrics = append(reportMetrics, nonSuccessMetrics.filteredMetrics...) + reportMetrics = append(reportMetrics, diffMetrics...) + + // successCountMap will be inCountMap for filtering events based on supported event types + inCountMap = successCountMap + inCountMetadataMap = successCountMetadataMap + } + // REPORTING - END + inPU = reportingTypes.USER_TRANSFORMER // for the next step in the pipeline } else { proc.logger.Debug("No custom transformation") eventsToTransform = eventList @@ -3217,159 +3225,163 @@ func (proc *Handle) transformSrcDest( // OR // b. transformAt is router and transformer doesn't support router transform if transformAt == "processor" || (transformAt == "router" && !transformAtFromFeaturesFile) { - trace.WithRegion(ctx, "Dest Transform", func() { - trace.Logf(ctx, "Dest Transform", "input size %d", len(eventsToTransform)) - proc.logger.Debug("Dest Transform input size", len(eventsToTransform)) - s := time.Now() + proc.logger.Debug("Dest Transform input size", len(eventsToTransform)) + s := time.Now() + var response types.Response + if !proc.config.enableTransformationV2 { response = proc.transformer.Transform(ctx, eventsToTransform, proc.config.transformBatchSize.Load()) + } else { + client, err := proc.transformerManager.GetServiceClient("destination_transformer") + if err != nil { + proc.logger.Error("Error getting destination transformer client", err) + panic(err) + } + response = client.SendRequest(ctx, eventsToTransform, proc.config.transformBatchSize.Load()) + } - destTransformationStat := proc.newDestinationTransformationStat(sourceID, workspaceID, transformAt, destination) - destTransformationStat.transformTime.Since(s) - transformAt = "processor" + destTransformationStat := proc.newDestinationTransformationStat(sourceID, workspaceID, transformAt, destination) + destTransformationStat.transformTime.Since(s) + transformAt = "processor" - proc.logger.Debugf("Dest Transform output size %d", len(response.Events)) - trace.Logf(ctx, "DestTransform", "output size %d", len(response.Events)) + proc.logger.Debugf("Dest Transform output size %d", len(response.Events)) - nonSuccessMetrics := proc.getNonSuccessfulMetrics( - response, commonMetaData, eventsByMessageID, - reportingTypes.EVENT_FILTER, reportingTypes.DEST_TRANSFORMER, - ) - destTransformationStat.numEvents.Count(len(eventsToTransform)) - destTransformationStat.numOutputSuccessEvents.Count(len(response.Events)) - destTransformationStat.numOutputFailedEvents.Count(len(nonSuccessMetrics.failedJobs)) - destTransformationStat.numOutputFilteredEvents.Count(len(nonSuccessMetrics.filteredJobs)) - droppedJobs = append(droppedJobs, append(proc.getDroppedJobs(response, eventsToTransform), append(nonSuccessMetrics.failedJobs, nonSuccessMetrics.filteredJobs...)...)...) - - if _, ok := procErrorJobsByDestID[destID]; !ok { - procErrorJobsByDestID[destID] = make([]*jobsdb.JobT, 0) - } - procErrorJobsByDestID[destID] = append(procErrorJobsByDestID[destID], nonSuccessMetrics.failedJobs...) + nonSuccessMetrics := proc.getNonSuccessfulMetrics( + response, commonMetaData, eventsByMessageID, + reportingTypes.EVENT_FILTER, reportingTypes.DEST_TRANSFORMER, + ) + destTransformationStat.numEvents.Count(len(eventsToTransform)) + destTransformationStat.numOutputSuccessEvents.Count(len(response.Events)) + destTransformationStat.numOutputFailedEvents.Count(len(nonSuccessMetrics.failedJobs)) + destTransformationStat.numOutputFilteredEvents.Count(len(nonSuccessMetrics.filteredJobs)) + droppedJobs = append(droppedJobs, append(proc.getDroppedJobs(response, eventsToTransform), append(nonSuccessMetrics.failedJobs, nonSuccessMetrics.filteredJobs...)...)...) - // REPORTING - PROCESSOR metrics - START - if proc.isReportingEnabled() { - successMetrics := make([]*reportingTypes.PUReportedMetric, 0) - connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) - statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) - successCountMap := make(map[string]int64) - for i := range response.Events { - // Update metrics maps - proc.updateMetricMaps(nil, successCountMap, connectionDetailsMap, statusDetailsMap, &response.Events[i], jobsdb.Succeeded.State, reportingTypes.DEST_TRANSFORMER, func() json.RawMessage { return []byte(`{}`) }, nil) - } - reportingTypes.AssertSameKeys(connectionDetailsMap, statusDetailsMap) - - for k, cd := range connectionDetailsMap { - for _, sd := range statusDetailsMap[k] { - m := &reportingTypes.PUReportedMetric{ - ConnectionDetails: *cd, - PUDetails: *reportingTypes.CreatePUDetails(reportingTypes.EVENT_FILTER, reportingTypes.DEST_TRANSFORMER, false, false), - StatusDetail: sd, - } - successMetrics = append(successMetrics, m) + if _, ok := procErrorJobsByDestID[destID]; !ok { + procErrorJobsByDestID[destID] = make([]*jobsdb.JobT, 0) + } + procErrorJobsByDestID[destID] = append(procErrorJobsByDestID[destID], nonSuccessMetrics.failedJobs...) + + // REPORTING - PROCESSOR metrics - START + if proc.isReportingEnabled() { + successMetrics := make([]*reportingTypes.PUReportedMetric, 0) + connectionDetailsMap := make(map[string]*reportingTypes.ConnectionDetails) + statusDetailsMap := make(map[string]map[string]*reportingTypes.StatusDetail) + successCountMap := make(map[string]int64) + for i := range response.Events { + // Update metrics maps + proc.updateMetricMaps(nil, successCountMap, connectionDetailsMap, statusDetailsMap, &response.Events[i], jobsdb.Succeeded.State, reportingTypes.DEST_TRANSFORMER, func() json.RawMessage { return []byte(`{}`) }, nil) + } + reportingTypes.AssertSameKeys(connectionDetailsMap, statusDetailsMap) + + for k, cd := range connectionDetailsMap { + for _, sd := range statusDetailsMap[k] { + m := &reportingTypes.PUReportedMetric{ + ConnectionDetails: *cd, + PUDetails: *reportingTypes.CreatePUDetails(reportingTypes.EVENT_FILTER, reportingTypes.DEST_TRANSFORMER, false, false), + StatusDetail: sd, } + successMetrics = append(successMetrics, m) } + } - diffMetrics := getDiffMetrics( - reportingTypes.EVENT_FILTER, - reportingTypes.DEST_TRANSFORMER, - inCountMetadataMap, - inCountMap, - successCountMap, - nonSuccessMetrics.failedCountMap, - nonSuccessMetrics.filteredCountMap, - proc.statsFactory, - ) + diffMetrics := getDiffMetrics( + reportingTypes.EVENT_FILTER, + reportingTypes.DEST_TRANSFORMER, + inCountMetadataMap, + inCountMap, + successCountMap, + nonSuccessMetrics.failedCountMap, + nonSuccessMetrics.filteredCountMap, + proc.statsFactory, + ) - reportMetrics = append(reportMetrics, nonSuccessMetrics.failedMetrics...) - reportMetrics = append(reportMetrics, nonSuccessMetrics.filteredMetrics...) - reportMetrics = append(reportMetrics, successMetrics...) - reportMetrics = append(reportMetrics, diffMetrics...) - } - // REPORTING - PROCESSOR metrics - END - }) + reportMetrics = append(reportMetrics, nonSuccessMetrics.failedMetrics...) + reportMetrics = append(reportMetrics, nonSuccessMetrics.filteredMetrics...) + reportMetrics = append(reportMetrics, successMetrics...) + reportMetrics = append(reportMetrics, diffMetrics...) + } + // REPORTING - PROCESSOR metrics - END } - trace.WithRegion(ctx, "MarshalForDB", func() { - // Save the JSON in DB. This is what the router uses - for i := range response.Events { - destEventJSON, err := jsonfast.Marshal(response.Events[i].Output) - // Should be a valid JSON since it's our transformation, but we handle it anyway - if err != nil { - continue - } - - // Need to replace UUID his with messageID from client - id := misc.FastUUID() - // read source_id from metadata that is replayed back from transformer - // in case of custom transformations metadata of first event is returned along with all events in session - // source_id will be same for all events belong to same user in a session - metadata := response.Events[i].Metadata - - sourceID := metadata.SourceID - destID := metadata.DestinationID - rudderID := metadata.RudderID - receivedAt := metadata.ReceivedAt - messageId := metadata.MessageID - jobId := metadata.JobID - sourceTaskRunId := metadata.SourceTaskRunID - recordId := metadata.RecordID - sourceJobId := metadata.SourceJobID - sourceJobRunId := metadata.SourceJobRunID - eventName := metadata.EventName - eventType := metadata.EventType - sourceDefID := metadata.SourceDefinitionID - destDefID := metadata.DestinationDefinitionID - sourceCategory := metadata.SourceCategory - workspaceId := metadata.WorkspaceID - // If the response from the transformer does not have userID in metadata, setting userID to random-uuid. - // This is done to respect findWorker logic in router. - if rudderID == "" { - rudderID = "random-" + id.String() - } + // Save the JSON in DB. This is what the router uses + for i := range response.Events { + destEventJSON, err := jsonfast.Marshal(response.Events[i].Output) + // Should be a valid JSON since it's our transformation, but we handle it anyway + if err != nil { + continue + } - params := ParametersT{ - SourceID: sourceID, - SourceName: sourceName, - DestinationID: destID, - ReceivedAt: receivedAt, - TransformAt: transformAt, - MessageID: messageId, - GatewayJobID: jobId, - SourceTaskRunID: sourceTaskRunId, - SourceJobID: sourceJobId, - SourceJobRunID: sourceJobRunId, - EventName: eventName, - EventType: eventType, - SourceCategory: sourceCategory, - SourceDefinitionID: sourceDefID, - DestinationDefinitionID: destDefID, - RecordID: recordId, - WorkspaceId: workspaceId, - TraceParent: metadata.TraceParent, - } - marshalledParams, err := jsonfast.Marshal(params) - if err != nil { - proc.logger.Errorf("[Processor] Failed to marshal parameters object. Parameters: %v", params) - panic(err) - } + // Need to replace UUID his with messageID from client + id := misc.FastUUID() + // read source_id from metadata that is replayed back from transformer + // in case of custom transformations metadata of first event is returned along with all events in session + // source_id will be same for all events belong to same user in a session + metadata := response.Events[i].Metadata + + sourceID := metadata.SourceID + destID := metadata.DestinationID + rudderID := metadata.RudderID + receivedAt := metadata.ReceivedAt + messageId := metadata.MessageID + jobId := metadata.JobID + sourceTaskRunId := metadata.SourceTaskRunID + recordId := metadata.RecordID + sourceJobId := metadata.SourceJobID + sourceJobRunId := metadata.SourceJobRunID + eventName := metadata.EventName + eventType := metadata.EventType + sourceDefID := metadata.SourceDefinitionID + destDefID := metadata.DestinationDefinitionID + sourceCategory := metadata.SourceCategory + workspaceId := metadata.WorkspaceID + // If the response from the transformer does not have userID in metadata, setting userID to random-uuid. + // This is done to respect findWorker logic in router. + if rudderID == "" { + rudderID = "random-" + id.String() + } + + params := ParametersT{ + SourceID: sourceID, + SourceName: sourceName, + DestinationID: destID, + ReceivedAt: receivedAt, + TransformAt: transformAt, + MessageID: messageId, + GatewayJobID: jobId, + SourceTaskRunID: sourceTaskRunId, + SourceJobID: sourceJobId, + SourceJobRunID: sourceJobRunId, + EventName: eventName, + EventType: eventType, + SourceCategory: sourceCategory, + SourceDefinitionID: sourceDefID, + DestinationDefinitionID: destDefID, + RecordID: recordId, + WorkspaceId: workspaceId, + TraceParent: metadata.TraceParent, + } + marshalledParams, err := jsonfast.Marshal(params) + if err != nil { + proc.logger.Errorf("[Processor] Failed to marshal parameters object. Parameters: %v", params) + panic(err) + } - newJob := jobsdb.JobT{ - UUID: id, - UserID: rudderID, - Parameters: marshalledParams, - CreatedAt: time.Now(), - ExpireAt: time.Now(), - CustomVal: destType, - EventPayload: destEventJSON, - WorkspaceId: workspaceId, - } - if slices.Contains(proc.config.batchDestinations, newJob.CustomVal) { - batchDestJobs = append(batchDestJobs, &newJob) - } else { - destJobs = append(destJobs, &newJob) - routerDestIDs[destID] = struct{}{} - } + newJob := jobsdb.JobT{ + UUID: id, + UserID: rudderID, + Parameters: marshalledParams, + CreatedAt: time.Now(), + ExpireAt: time.Now(), + CustomVal: destType, + EventPayload: destEventJSON, + WorkspaceId: workspaceId, } - }) + if slices.Contains(proc.config.batchDestinations, newJob.CustomVal) { + batchDestJobs = append(batchDestJobs, &newJob) + } else { + destJobs = append(destJobs, &newJob) + routerDestIDs[destID] = struct{}{} + } + } return transformSrcDestOutput{ destJobs: destJobs, batchDestJobs: batchDestJobs, diff --git a/processor/trackingplan.go b/processor/trackingplan.go index 0322277e6a..bd968149d8 100644 --- a/processor/trackingplan.go +++ b/processor/trackingplan.go @@ -95,7 +95,17 @@ func (proc *Handle) validateEvents(groupedEventsBySourceId map[SourceIDT][]types commonMetaData := makeCommonMetadataFromTransformerEvent(&transformerEvent) validationStart := time.Now() - response := proc.transformer.Validate(context.TODO(), eventList, proc.config.userTransformBatchSize.Load()) + var response types.Response + if proc.config.enableTransformationV2 { + response = proc.transformer.Validate(context.TODO(), eventList, proc.config.userTransformBatchSize.Load()) + } else { + client, err := proc.transformerManager.GetServiceClient("trackingplan_validation") + if err != nil { + proc.logger.Error("Error getting trackingplan_validation client", err.Error()) + panic(err) + } + response = client.SendRequest(context.TODO(), eventList, proc.config.userTransformBatchSize.Load()) + } validationStat.tpValidationTime.Since(validationStart) // If transformerInput does not match with transformerOutput then we do not consider transformerOutput