diff --git a/build.gradle b/build.gradle index 7bb8cb063..6f03712aa 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,7 @@ lombok { } group 'com.gotocompany' -version '0.10.5' +version '0.10.6' def projName = "firehose" diff --git a/docs/docs/sinks/grpc-sink.md b/docs/docs/sinks/grpc-sink.md index 25910ac28..f82ce2ca1 100644 --- a/docs/docs/sinks/grpc-sink.md +++ b/docs/docs/sinks/grpc-sink.md @@ -42,14 +42,23 @@ Note - final metadata will be generated with merging metadata and the kafka reco message GenericPayload { string field = "field_name"; - string field_two = "field_two"; + string field_two = "FIELD_two"; string id = "123"; int code = 400; } ``` - Example config : `$com.goto.company.GenericPayload.field: $(com.goto.company.GenericPayload.field_two + '_' + string(com.goto.company.GenericPayload.code))` + Example config : `$com.goto.company.GenericPayload.field: $(com.goto.company.GenericPayload.field_two.lowerAscii() + '_' + string(com.goto.company.GenericPayload.code))` This would result in : `field_name:field_two_400` - +- CEL Extended Libraries : + - String operations (https://github.com/google/cel-java/blob/main/extensions/src/main/java/dev/cel/extensions/README.md#strings) + - Add the capabilities of performing typical string operations such as up/lower casting, trimming, replacement, etc. + - Example : `com.goto.company.GenericPayload.field_two.lowerAscii()`, `com.goto.company.GenericPayload.field_two.upperAscii()` + - Math Operations (https://github.com/google/cel-java/blob/main/extensions/src/main/java/dev/cel/extensions/README.md#math) + - Currently only support finding greatest and least of numerical inputs + - Example : `math.greatest(com.goto.company.GenericPayload.code, 999), math.least(com.goto.company.GenericPayload.code, 999)` + - Binding Operations (https://github.com/google/cel-java/blob/main/extensions/src/main/java/dev/cel/extensions/README.md#celbind) + - Add the capabilities of binding a CEL Expression into variables and reusing them in the subsequent expression + - Example : `cel.bind(stringifiedCode, string(com.goto.company.GenericPayload.code), stringifiedCode + '_' + com.goto.company.GenericPayload.id)` ### `SINK_GRPC_RESPONSE_SCHEMA_PROTO_CLASS` diff --git a/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java b/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java index b234455a1..d05a8a5b0 100644 --- a/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java +++ b/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java @@ -6,7 +6,6 @@ import dev.cel.common.types.CelKind; import dev.cel.compiler.CelCompiler; import dev.cel.runtime.CelRuntime; -import dev.cel.runtime.CelRuntimeFactory; import lombok.extern.slf4j.Slf4j; /** @@ -52,8 +51,7 @@ public boolean evaluate(Message payload) { */ private void buildCelEnvironment(String celExpression) { CelCompiler celCompiler = CelUtils.initializeCelCompiler(this.descriptor); - CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder() - .build(); + CelRuntime celRuntime = CelUtils.initializeCelRuntime(); this.celProgram = CelUtils.initializeCelProgram(celExpression, celRuntime, celCompiler, celType -> celType.kind().equals(CelKind.BOOL)); } diff --git a/src/main/java/com/gotocompany/firehose/proto/ProtoToMetadataMapper.java b/src/main/java/com/gotocompany/firehose/proto/ProtoToMetadataMapper.java index 45f708e60..e3b538901 100644 --- a/src/main/java/com/gotocompany/firehose/proto/ProtoToMetadataMapper.java +++ b/src/main/java/com/gotocompany/firehose/proto/ProtoToMetadataMapper.java @@ -8,7 +8,6 @@ import com.gotocompany.firehose.utils.CelUtils; import dev.cel.compiler.CelCompiler; import dev.cel.runtime.CelRuntime; -import dev.cel.runtime.CelRuntimeFactory; import io.grpc.Metadata; import org.apache.commons.collections.MapUtils; @@ -102,7 +101,7 @@ private Object evaluateExpression(String input, Message message) { * @return a map of CEL expressions to their corresponding programs */ private Map initializeCelPrograms() { - CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build(); + CelRuntime celRuntime = CelUtils.initializeCelRuntime(); CelCompiler celCompiler = CelUtils.initializeCelCompiler(this.descriptor); return this.metadataTemplate.entrySet() .stream() diff --git a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java index 8d676edb6..231993c86 100644 --- a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java +++ b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java @@ -3,16 +3,18 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.Message; import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelOptions; import dev.cel.common.CelValidationException; import dev.cel.common.types.CelType; import dev.cel.common.types.StructTypeReference; import dev.cel.compiler.CelCompiler; import dev.cel.compiler.CelCompilerFactory; +import dev.cel.extensions.CelExtensions; import dev.cel.parser.CelStandardMacro; import dev.cel.runtime.CelEvaluationException; import dev.cel.runtime.CelRuntime; -import org.aeonbits.owner.util.Collections; - +import dev.cel.runtime.CelRuntimeFactory; +import java.util.Collections; import java.util.function.Predicate; /** @@ -28,7 +30,7 @@ public class CelUtils { */ public static Object evaluate(CelRuntime.Program program, Message payload) { try { - return program.eval(Collections.map(payload.getDescriptorForType().getFullName(), payload)); + return program.eval(Collections.singletonMap(payload.getDescriptorForType().getFullName(), payload)); } catch (CelEvaluationException e) { throw new IllegalArgumentException("Could not evaluate Cel expression", e); } @@ -43,10 +45,22 @@ public static CelCompiler initializeCelCompiler(Descriptors.Descriptor descripto return CelCompilerFactory.standardCelCompilerBuilder() .setStandardMacros(CelStandardMacro.values()) .addVar(descriptor.getFullName(), StructTypeReference.create(descriptor.getFullName())) + .addLibraries(CelExtensions.strings(), CelExtensions.bindings(), CelExtensions.math(CelOptions.DEFAULT)) .addMessageTypes(descriptor) .build(); } + /** + * Initializes the CEL runtime with extended libraries. + * + * @return the initialized CEL runtime + */ + public static CelRuntime initializeCelRuntime() { + return CelRuntimeFactory.standardCelRuntimeBuilder() + .addLibraries(CelExtensions.strings(), CelExtensions.math(CelOptions.DEFAULT)) + .build(); + } + /** * Initializes a CEL program for a given expression. * diff --git a/src/test/java/com/gotocompany/firehose/proto/ProtoToMetadataMapperTest.java b/src/test/java/com/gotocompany/firehose/proto/ProtoToMetadataMapperTest.java index f77da5264..e5138cea1 100644 --- a/src/test/java/com/gotocompany/firehose/proto/ProtoToMetadataMapperTest.java +++ b/src/test/java/com/gotocompany/firehose/proto/ProtoToMetadataMapperTest.java @@ -18,10 +18,13 @@ public class ProtoToMetadataMapperTest { public void setup() { Map template = new HashMap<>(); template.put("$GenericResponse.detail", "$GenericResponse.success"); + template.put("detail", "$GenericResponse.detail.lowerAscii()"); template.put("someField", "someValue"); template.put("$GenericResponse.success", "staticValue"); template.put("staticKey", "$(GenericResponse.errors[0].cause + '-' + GenericResponse.errors[0].code + '-' + string(GenericResponse.code))"); template.put("entity", "$GenericResponse.errors[0].entity"); + template.put("binding", "$cel.bind(code, GenericResponse.code, code + 100)"); + template.put("math", "$math.greatest(GenericResponse.code, 200)"); this.protoToMetadataMapper = new ProtoToMetadataMapper( GenericResponse.getDescriptor(), template @@ -32,7 +35,7 @@ public void setup() { public void shouldBuildDynamicMetadataWithCorrectPlaceholders() { GenericResponse payload = GenericResponse.newBuilder() .setSuccess(false) - .setDetail("detail_of_error") + .setDetail("Detail_Of_Error") .setCode(100) .addErrors(GenericError.newBuilder() .setCode("404") @@ -44,6 +47,8 @@ public void shouldBuildDynamicMetadataWithCorrectPlaceholders() { Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("detail_of_error", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertEquals("false", metadata.get(Metadata.Key.of("detail_of_error", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("detail", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertEquals("detail_of_error", metadata.get(Metadata.Key.of("detail", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("statickey", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertEquals("not_found-404-100", metadata.get(Metadata.Key.of("statickey", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("somefield", Metadata.ASCII_STRING_MARSHALLER))); @@ -52,6 +57,10 @@ public void shouldBuildDynamicMetadataWithCorrectPlaceholders() { Assertions.assertEquals("", metadata.get(Metadata.Key.of("entity", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("false", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertEquals("staticValue", metadata.get(Metadata.Key.of("false", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("binding", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertEquals("200", metadata.get(Metadata.Key.of("binding", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("math", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertEquals("200", metadata.get(Metadata.Key.of("math", Metadata.ASCII_STRING_MARSHALLER))); } @Test