From 1d7864d59cd780a4cec176889df9a1fe0ab03b4b Mon Sep 17 00:00:00 2001 From: Abe Varghese Date: Tue, 30 Jul 2024 19:18:59 +0530 Subject: [PATCH] Add namespace manager to retrieve REST function signatures. --- presto-function-namespace-managers/pom.xml | 33 ++ ...actSqlInvokedFunctionNamespaceManager.java | 1 + .../functionNamespace/ForRestServer.java | 26 ++ .../FunctionNamespaceManagerPlugin.java | 5 +- .../JsonBasedUdfFunctionMetadata.java | 26 +- .../{json => }/UdfFunctionSignatureMap.java | 2 +- .../json/FunctionDefinitionProvider.java | 2 + ...onFileBasedFunctionDefinitionProvider.java | 2 + ...JsonFileBasedFunctionNamespaceManager.java | 2 + .../rest/RestBasedCommunicationModule.java | 30 ++ .../rest/RestBasedFunctionApis.java | 128 +++++++ .../RestBasedFunctionNamespaceManager.java | 234 +++++++++++++ ...stBasedFunctionNamespaceManagerConfig.java | 38 +++ ...tBasedFunctionNamespaceManagerFactory.java | 73 ++++ ...stBasedFunctionNamespaceManagerModule.java | 55 +++ ...TestRestBasedFunctionNamespaceManager.java | 322 ++++++++++++++++++ .../main/resources/rest_function_server.yaml | 286 ++++++++++++++++ .../function/FunctionImplementationType.java | 3 +- .../spi/function/RestFunctionHandle.java | 53 +++ .../spi/function/RoutineCharacteristics.java | 4 + .../spi/function/SqlInvokedFunction.java | 32 ++ 21 files changed, 1349 insertions(+), 8 deletions(-) create mode 100644 presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/ForRestServer.java rename presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/{json => }/JsonBasedUdfFunctionMetadata.java (85%) rename presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/{json => }/UdfFunctionSignatureMap.java (96%) create mode 100644 presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedCommunicationModule.java create mode 100644 presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionApis.java create mode 100644 presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManager.java create mode 100644 presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerConfig.java create mode 100644 presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerFactory.java create mode 100644 presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerModule.java create mode 100644 presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestRestBasedFunctionNamespaceManager.java create mode 100644 presto-openapi/src/main/resources/rest_function_server.yaml create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/RestFunctionHandle.java diff --git a/presto-function-namespace-managers/pom.xml b/presto-function-namespace-managers/pom.xml index 120549c4f8c35..fdbee3e1f11c4 100644 --- a/presto-function-namespace-managers/pom.xml +++ b/presto-function-namespace-managers/pom.xml @@ -149,6 +149,21 @@ provided + + com.facebook.airlift + http-client + + + + com.fasterxml.jackson.core + jackson-databind + + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + + com.facebook.presto @@ -184,5 +199,23 @@ assertj-core test + + + com.facebook.airlift + jaxrs + test + + + + com.facebook.airlift + jaxrs-testing + test + + + + javax.ws.rs + javax.ws.rs-api + test + diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java index f8701da6284b9..dfbd9f737c7d2 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java @@ -325,6 +325,7 @@ protected ScalarFunctionImplementation sqlInvokedFunctionToImplementation(SqlInv return new SqlInvokedScalarFunctionImplementation(function.getBody()); case THRIFT: case GRPC: + case REST: checkArgument(function.getFunctionHandle().isPresent(), "Need functionHandle to get function implementation"); return new RemoteScalarFunctionImplementation(function.getFunctionHandle().get(), function.getRoutineCharacteristics().getLanguage(), implementationType); case JAVA: diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/ForRestServer.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/ForRestServer.java new file mode 100644 index 0000000000000..0b6c2ee2d294b --- /dev/null +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/ForRestServer.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functionNamespace; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@BindingAnnotation +public @interface ForRestServer +{ +} diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/FunctionNamespaceManagerPlugin.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/FunctionNamespaceManagerPlugin.java index 2be7feebf65ed..689c3e09765c4 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/FunctionNamespaceManagerPlugin.java +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/FunctionNamespaceManagerPlugin.java @@ -15,6 +15,7 @@ import com.facebook.presto.functionNamespace.json.JsonFileBasedFunctionNamespaceManagerFactory; import com.facebook.presto.functionNamespace.mysql.MySqlFunctionNamespaceManagerFactory; +import com.facebook.presto.functionNamespace.rest.RestBasedFunctionNamespaceManagerFactory; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; import com.google.common.collect.ImmutableList; @@ -25,6 +26,8 @@ public class FunctionNamespaceManagerPlugin @Override public Iterable getFunctionNamespaceManagerFactories() { - return ImmutableList.of(new MySqlFunctionNamespaceManagerFactory(), new JsonFileBasedFunctionNamespaceManagerFactory()); + return ImmutableList.of(new MySqlFunctionNamespaceManagerFactory(), + new JsonFileBasedFunctionNamespaceManagerFactory(), + new RestBasedFunctionNamespaceManagerFactory()); } } diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonBasedUdfFunctionMetadata.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java similarity index 85% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonBasedUdfFunctionMetadata.java rename to presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java index 47e6704e68458..43c3a67cd1b22 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonBasedUdfFunctionMetadata.java +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java @@ -11,13 +11,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.functionNamespace.json; +package com.facebook.presto.functionNamespace; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.spi.function.AggregationFunctionMetadata; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.RoutineCharacteristics; +import com.facebook.presto.spi.function.SqlFunctionId; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -30,9 +32,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; -/** - * The function metadata provided by the Json file to the {@link JsonFileBasedFunctionNamespaceManager}. - */ public class JsonBasedUdfFunctionMetadata { /** @@ -64,6 +63,8 @@ public class JsonBasedUdfFunctionMetadata * Optional Aggregate-specific metadata (required for aggregation functions) */ private final Optional aggregateMetadata; + private final Optional functionId; + private final Optional version; @JsonCreator public JsonBasedUdfFunctionMetadata( @@ -73,7 +74,9 @@ public JsonBasedUdfFunctionMetadata( @JsonProperty("paramTypes") List paramTypes, @JsonProperty("schema") String schema, @JsonProperty("routineCharacteristics") RoutineCharacteristics routineCharacteristics, - @JsonProperty("aggregateMetadata") Optional aggregateMetadata) + @JsonProperty("aggregateMetadata") Optional aggregateMetadata, + @JsonProperty("functionId") Optional functionId, + @JsonProperty("version") Optional version) { this.docString = requireNonNull(docString, "docString is null"); this.functionKind = requireNonNull(functionKind, "functionKind is null"); @@ -85,6 +88,8 @@ public JsonBasedUdfFunctionMetadata( checkArgument( (functionKind == AGGREGATE && aggregateMetadata.isPresent()) || (functionKind != AGGREGATE && !aggregateMetadata.isPresent()), "aggregateMetadata must be present for aggregation functions and absent otherwise"); + this.functionId = requireNonNull(functionId, "functionId is null"); + this.version = requireNonNull(version, "version is null"); } public String getDocString() @@ -102,6 +107,7 @@ public TypeSignature getOutputType() return outputType; } + @JsonIgnore public List getParamNames() { return IntStream.range(0, paramTypes.size()).boxed().map(idx -> "input" + idx).collect(toImmutableList()); @@ -126,4 +132,14 @@ public Optional getAggregateMetadata() { return aggregateMetadata; } + + public Optional getFunctionId() + { + return functionId; + } + + public Optional getVersion() + { + return version; + } } diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/UdfFunctionSignatureMap.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/UdfFunctionSignatureMap.java similarity index 96% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/UdfFunctionSignatureMap.java rename to presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/UdfFunctionSignatureMap.java index 6f9e96e832add..b5786f323dd60 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/UdfFunctionSignatureMap.java +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/UdfFunctionSignatureMap.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.functionNamespace.json; +package com.facebook.presto.functionNamespace; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/FunctionDefinitionProvider.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/FunctionDefinitionProvider.java index 460b40fb077fc..b1514c7b18197 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/FunctionDefinitionProvider.java +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/FunctionDefinitionProvider.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.functionNamespace.json; +import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap; + public interface FunctionDefinitionProvider { UdfFunctionSignatureMap getUdfDefinition(String filePath); diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonFileBasedFunctionDefinitionProvider.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonFileBasedFunctionDefinitionProvider.java index 728f4be72273f..749f7cc0d2824 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonFileBasedFunctionDefinitionProvider.java +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonFileBasedFunctionDefinitionProvider.java @@ -14,6 +14,8 @@ package com.facebook.presto.functionNamespace.json; import com.facebook.airlift.log.Logger; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; +import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap; import java.io.IOException; import java.nio.file.Files; diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonFileBasedFunctionNamespaceManager.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonFileBasedFunctionNamespaceManager.java index de85f56752a42..b830b1c4e1863 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonFileBasedFunctionNamespaceManager.java +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonFileBasedFunctionNamespaceManager.java @@ -20,8 +20,10 @@ import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.common.type.UserDefinedType; import com.facebook.presto.functionNamespace.AbstractSqlInvokedFunctionNamespaceManager; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; import com.facebook.presto.functionNamespace.ServingCatalog; import com.facebook.presto.functionNamespace.SqlInvokedFunctionNamespaceManagerConfig; +import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap; import com.facebook.presto.functionNamespace.execution.SqlFunctionExecutors; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.AggregationFunctionImplementation; diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedCommunicationModule.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedCommunicationModule.java new file mode 100644 index 0000000000000..7265edcf6bc8f --- /dev/null +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedCommunicationModule.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functionNamespace.rest; + +import com.facebook.presto.functionNamespace.ForRestServer; +import com.google.inject.Binder; +import com.google.inject.Module; + +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; + +public class RestBasedCommunicationModule + implements Module +{ + @Override + public void configure(Binder binder) + { + httpClientBinder(binder).bindHttpClient("restServer", ForRestServer.class); + } +} diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionApis.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionApis.java new file mode 100644 index 0000000000000..f196eec702e05 --- /dev/null +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionApis.java @@ -0,0 +1,128 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functionNamespace.rest; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.StatusResponseHandler; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.functionNamespace.ForRestServer; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; +import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap; +import com.facebook.presto.spi.PrestoException; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; + +import java.net.URI; +import java.util.List; +import java.util.Map; + +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; + +public class RestBasedFunctionApis +{ + public static final String ALL_FUNCTIONS_ENDPOINT = "/v1/functions"; + + private final HttpClient httpClient; + + private final JsonCodec>> functionSignatureMapJsonCodec; + + private final RestBasedFunctionNamespaceManagerConfig managerConfig; + + @Inject + public RestBasedFunctionApis( + JsonCodec>> nativeFunctionSignatureMapJsonCodec, + @ForRestServer HttpClient httpClient, + RestBasedFunctionNamespaceManagerConfig managerConfig) + { + this.functionSignatureMapJsonCodec = nativeFunctionSignatureMapJsonCodec; + this.httpClient = httpClient; + this.managerConfig = managerConfig; + } + + public String getFunctionsETag() + { + try { + URI uri = uriBuilderFrom(URI.create(managerConfig.getRestUrl())) + .appendPath(ALL_FUNCTIONS_ENDPOINT) + .build(); + Request request = Request.builder() + .prepareHead() + .setUri(uri) + .build(); + + StatusResponseHandler.StatusResponse response = httpClient.execute(request, StatusResponseHandler.createStatusResponseHandler()); + String version = response.getHeader("ETag"); + if (version == null) { + throw new IllegalStateException("Failed to retrieve API version: 'ETag' header is missing"); + } + return version; + } + catch (Exception e) { + throw new IllegalStateException("Failed to get functions ETag from REST server, " + e.getMessage()); + } + } + + public UdfFunctionSignatureMap getAllFunctions() + { + return getFunctionsAt(ALL_FUNCTIONS_ENDPOINT); + } + + public UdfFunctionSignatureMap getFunctions(String schema) + { + return getFunctionsAt(ALL_FUNCTIONS_ENDPOINT + "/" + schema); + } + + public UdfFunctionSignatureMap getFunctions(String schema, String functionName) + { + return getFunctionsAt(ALL_FUNCTIONS_ENDPOINT + "/" + schema + "/" + functionName); + } + + public String addFunction(String schema, String functionName, JsonBasedUdfFunctionMetadata metadata) + { + throw new PrestoException(NOT_SUPPORTED, "Add Function is yet to be added"); + } + + public String updateFunction(String schema, String functionName, String functionId, JsonBasedUdfFunctionMetadata metadata) + { + throw new PrestoException(NOT_SUPPORTED, "Update Function is yet to be added"); + } + + public String deleteFunction(String schema, String functionName, String functionId) + { + throw new PrestoException(NOT_SUPPORTED, "Delete Function is yet to be added"); + } + + private UdfFunctionSignatureMap getFunctionsAt(String endpoint) + throws IllegalStateException + { + try { + URI uri = uriBuilderFrom(URI.create(managerConfig.getRestUrl())) + .appendPath(endpoint) + .build(); + Request request = Request.builder() + .prepareGet() + .setUri(uri) + .build(); + + Map> nativeFunctionSignatureMap = httpClient.execute(request, createJsonResponseHandler(functionSignatureMapJsonCodec)); + return new UdfFunctionSignatureMap(ImmutableMap.copyOf(nativeFunctionSignatureMap)); + } + catch (Exception e) { + throw new IllegalStateException("Failed to get function definitions from REST server, " + e.getMessage()); + } + } +} diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManager.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManager.java new file mode 100644 index 0000000000000..bde552e64714d --- /dev/null +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManager.java @@ -0,0 +1,234 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functionNamespace.rest; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.common.type.UserDefinedType; +import com.facebook.presto.functionNamespace.AbstractSqlInvokedFunctionNamespaceManager; +import com.facebook.presto.functionNamespace.InvalidFunctionHandleException; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; +import com.facebook.presto.functionNamespace.ServingCatalog; +import com.facebook.presto.functionNamespace.SqlInvokedFunctionNamespaceManagerConfig; +import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap; +import com.facebook.presto.functionNamespace.execution.SqlFunctionExecutors; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.AggregationFunctionImplementation; +import com.facebook.presto.spi.function.AlterRoutineCharacteristics; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionVersion; +import com.facebook.presto.spi.function.Parameter; +import com.facebook.presto.spi.function.RestFunctionHandle; +import com.facebook.presto.spi.function.ScalarFunctionImplementation; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionHandle; +import com.facebook.presto.spi.function.SqlFunctionId; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.google.common.collect.ImmutableList; + +import javax.inject.Inject; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.REST; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Collections.emptyList; +import static java.util.Objects.requireNonNull; + +public class RestBasedFunctionNamespaceManager + extends AbstractSqlInvokedFunctionNamespaceManager +{ + private static final Logger log = Logger.get(RestBasedFunctionNamespaceManager.class); + private final RestBasedFunctionApis restApis; + private final List latestFunctions = new ArrayList<>(); + private Optional cachedETag = Optional.empty(); + + @Inject + public RestBasedFunctionNamespaceManager( + @ServingCatalog String catalogName, + SqlFunctionExecutors sqlFunctionExecutors, + SqlInvokedFunctionNamespaceManagerConfig config, + RestBasedFunctionApis restApis) + { + super(catalogName, sqlFunctionExecutors, config); + this.restApis = requireNonNull(restApis, "restApis is null"); + } + + @Override + public final AggregationFunctionImplementation getAggregateFunctionImplementation(FunctionHandle functionHandle, TypeManager typeManager) + { + throw new PrestoException(NOT_SUPPORTED, "Aggregate Function is not supported in RestBasedFunctionNamespaceManager"); + } + + private List getLatestFunctions() + { + // Check if the function list has been modified. + String newETag = restApis.getFunctionsETag(); + if (cachedETag.isPresent() && cachedETag.get().equals(newETag)) { + return latestFunctions; + } + + // Clear cached list of functions and get the latest list. + latestFunctions.clear(); + UdfFunctionSignatureMap udfFunctionSignatureMap = restApis.getAllFunctions(); + if (udfFunctionSignatureMap == null || udfFunctionSignatureMap.isEmpty()) { + return Collections.emptyList(); + } + + createSqlInvokedFunctions(udfFunctionSignatureMap, latestFunctions); + cachedETag = Optional.of(newETag); + return latestFunctions; + } + + private void createSqlInvokedFunctions(UdfFunctionSignatureMap udfFunctionSignatureMap, List functionList) + { + Map> udfSignatureMap = udfFunctionSignatureMap.getUDFSignatureMap(); + udfSignatureMap.forEach((name, metaInfoList) -> { + List functions = metaInfoList.stream().map(metaInfo -> createSqlInvokedFunction(name, metaInfo)).collect(toImmutableList()); + functionList.addAll(functions); + }); + } + + private SqlInvokedFunction createSqlInvokedFunction(String functionName, JsonBasedUdfFunctionMetadata jsonBasedUdfFunctionMetaData) + { + checkState(jsonBasedUdfFunctionMetaData.getRoutineCharacteristics().getLanguage().equals(REST), "RestBasedFunctionNamespaceManager only supports REST UDF"); + QualifiedObjectName qualifiedFunctionName = QualifiedObjectName.valueOf(new CatalogSchemaName(getCatalogName(), jsonBasedUdfFunctionMetaData.getSchema()), functionName); + List parameterNameList = jsonBasedUdfFunctionMetaData.getParamNames(); + List parameterTypeList = jsonBasedUdfFunctionMetaData.getParamTypes(); + + ImmutableList.Builder parameterBuilder = ImmutableList.builder(); + for (int i = 0; i < parameterNameList.size(); i++) { + parameterBuilder.add(new Parameter(parameterNameList.get(i), parameterTypeList.get(i))); + } + + FunctionVersion functionVersion = new FunctionVersion(jsonBasedUdfFunctionMetaData.getVersion()); + SqlFunctionId functionId = jsonBasedUdfFunctionMetaData.getFunctionId().isPresent() ? jsonBasedUdfFunctionMetaData.getFunctionId().get() : null; + return new SqlInvokedFunction( + qualifiedFunctionName, + parameterBuilder.build(), + emptyList(), + jsonBasedUdfFunctionMetaData.getOutputType(), + jsonBasedUdfFunctionMetaData.getDocString(), + jsonBasedUdfFunctionMetaData.getRoutineCharacteristics(), + "", + functionVersion, + jsonBasedUdfFunctionMetaData.getFunctionKind(), + functionId, + jsonBasedUdfFunctionMetaData.getAggregateMetadata(), + Optional.of(new RestFunctionHandle( + functionId, + functionVersion.toString(), + new Signature( + qualifiedFunctionName, + jsonBasedUdfFunctionMetaData.getFunctionKind(), + jsonBasedUdfFunctionMetaData.getOutputType(), + jsonBasedUdfFunctionMetaData.getParamTypes())))); + } + + @Override + protected Collection fetchFunctionsDirect(QualifiedObjectName functionName) + { + UdfFunctionSignatureMap udfFunctionSignatureMap = restApis.getFunctions(functionName.getSchemaName(), functionName.getObjectName()); + if (udfFunctionSignatureMap == null || udfFunctionSignatureMap.isEmpty()) { + return Collections.emptyList(); + } + + List functions = new ArrayList<>(); + createSqlInvokedFunctions(udfFunctionSignatureMap, functions); + return functions; + } + + @Override + protected UserDefinedType fetchUserDefinedTypeDirect(QualifiedObjectName typeName) + { + throw new PrestoException(NOT_SUPPORTED, "User Defined Type is not supported in RestBasedFunctionNamespaceManager"); + } + + protected Optional getSqlInvokedFunction(SqlFunctionHandle functionHandle) + { + Collection functions = fetchFunctionsDirect(functionHandle.getFunctionId().getFunctionName()); + + return functions.stream() + .filter(sqlFunction -> sqlFunction.getFunctionId().equals(functionHandle.getFunctionId()) && + sqlFunction.getVersion().toString().equals(functionHandle.getVersion())) + .findFirst(); + } + + @Override + protected FunctionMetadata fetchFunctionMetadataDirect(SqlFunctionHandle functionHandle) + { + checkCatalog(functionHandle); + + Optional function = getSqlInvokedFunction(functionHandle); + if (!function.isPresent()) { + throw new InvalidFunctionHandleException(functionHandle); + } + + return sqlInvokedFunctionToMetadata(function.get()); + } + + @Override + protected ScalarFunctionImplementation fetchFunctionImplementationDirect(SqlFunctionHandle functionHandle) + { + checkCatalog(functionHandle); + + Optional function = getSqlInvokedFunction(functionHandle); + if (!function.isPresent()) { + throw new InvalidFunctionHandleException(functionHandle); + } + + return sqlInvokedFunctionToImplementation(function.get()); + } + + @Override + public void createFunction(SqlInvokedFunction function, boolean replace) + { + throw new PrestoException(NOT_SUPPORTED, "Create Function is not supported in RestBasedFunctionNamespaceManager"); + } + + @Override + public void alterFunction(QualifiedObjectName functionName, Optional> parameterTypes, AlterRoutineCharacteristics alterRoutineCharacteristics) + { + throw new PrestoException(NOT_SUPPORTED, "Alter Function is not supported in RestBasedFunctionNamespaceManager"); + } + + @Override + public void dropFunction(QualifiedObjectName functionName, Optional> parameterTypes, boolean exists) + { + throw new PrestoException(NOT_SUPPORTED, "Drop Function is not supported in RestBasedFunctionNamespaceManager"); + } + + @Override + public Collection listFunctions(Optional likePattern, Optional escape) + { + return getLatestFunctions(); + } + + @Override + public void addUserDefinedType(UserDefinedType userDefinedType) + { + throw new PrestoException(NOT_SUPPORTED, "Add User Defined Type is not supported in RestBasedFunctionNamespaceManager"); + } +} diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerConfig.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerConfig.java new file mode 100644 index 0000000000000..d520d661aeacf --- /dev/null +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerConfig.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functionNamespace.rest; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; + +import javax.validation.constraints.NotNull; + +public class RestBasedFunctionNamespaceManagerConfig +{ + private String restUrl = ""; + + @NotNull + public String getRestUrl() + { + return restUrl; + } + + @Config("rest-based-function-manager.rest.url") + @ConfigDescription("URL to a REST server from which the namespace manager can retrieve function signatures") + public RestBasedFunctionNamespaceManagerConfig setRestUrl(String restUrl) + { + this.restUrl = restUrl; + return this; + } +} diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerFactory.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerFactory.java new file mode 100644 index 0000000000000..df6056c3e11a2 --- /dev/null +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerFactory.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functionNamespace.rest; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.presto.functionNamespace.FunctionNamespaceManagerPlugin; +import com.facebook.presto.functionNamespace.execution.NoopSqlFunctionExecutorsModule; +import com.facebook.presto.spi.function.FunctionHandleResolver; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.FunctionNamespaceManagerContext; +import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; +import com.facebook.presto.spi.function.RestFunctionHandle; +import com.google.inject.Injector; + +import java.util.Map; + +import static com.google.common.base.Throwables.throwIfUnchecked; + +/** + * Factory class to create instance of {@link RestBasedFunctionNamespaceManager}. + * This factor is registered in {@link FunctionNamespaceManagerPlugin#getFunctionNamespaceManagerFactories()}. + */ +public class RestBasedFunctionNamespaceManagerFactory + implements FunctionNamespaceManagerFactory +{ + public static final String NAME = "rest"; + + private static final RestFunctionHandle.Resolver HANDLE_RESOLVER = new RestFunctionHandle.Resolver(); + + @Override + public String getName() + { + return NAME; + } + + @Override + public FunctionHandleResolver getHandleResolver() + { + return HANDLE_RESOLVER; + } + + @Override + public FunctionNamespaceManager create(String catalogName, Map config, FunctionNamespaceManagerContext context) + { + try { + Bootstrap app = new Bootstrap( + new RestBasedCommunicationModule(), + new RestBasedFunctionNamespaceManagerModule(catalogName), + new NoopSqlFunctionExecutorsModule()); + + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); + return injector.getInstance(RestBasedFunctionNamespaceManager.class); + } + catch (Exception e) { + throwIfUnchecked(e); + throw new RuntimeException(e); + } + } +} diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerModule.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerModule.java new file mode 100644 index 0000000000000..864063202952f --- /dev/null +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManagerModule.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functionNamespace.rest; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; +import com.facebook.presto.functionNamespace.ServingCatalog; +import com.facebook.presto.functionNamespace.execution.SqlFunctionLanguageConfig; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.TypeLiteral; + +import java.util.List; +import java.util.Map; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; +import static com.google.inject.Scopes.SINGLETON; +import static java.util.Objects.requireNonNull; + +public class RestBasedFunctionNamespaceManagerModule + implements Module +{ + private final String catalogName; + + public RestBasedFunctionNamespaceManagerModule(String catalogName) + { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); + } + + @Override + public void configure(Binder binder) + { + binder.bind(new TypeLiteral() {}).annotatedWith(ServingCatalog.class).toInstance(catalogName); + + configBinder(binder).bindConfig(RestBasedFunctionNamespaceManagerConfig.class); + configBinder(binder).bindConfig(SqlFunctionLanguageConfig.class); + binder.bind(RestBasedFunctionApis.class).in(SINGLETON); + binder.bind(RestBasedFunctionNamespaceManager.class).in(SINGLETON); + binder.bind(new TypeLiteral>>>() {}) + .toInstance(new JsonCodecFactory().mapJsonCodec(String.class, listJsonCodec(JsonBasedUdfFunctionMetadata.class))); + } +} diff --git a/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestRestBasedFunctionNamespaceManager.java b/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestRestBasedFunctionNamespaceManager.java new file mode 100644 index 0000000000000..d99e050dab0df --- /dev/null +++ b/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestRestBasedFunctionNamespaceManager.java @@ -0,0 +1,322 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functionNamespace; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.jaxrs.JsonMapper; +import com.facebook.airlift.jaxrs.testing.JaxrsTestingHttpProcessor; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.functionNamespace.execution.NoopSqlFunctionExecutorsModule; +import com.facebook.presto.functionNamespace.rest.RestBasedFunctionNamespaceManager; +import com.facebook.presto.functionNamespace.rest.RestBasedFunctionNamespaceManagerModule; +import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.RoutineCharacteristics; +import com.facebook.presto.spi.function.ScalarFunctionImplementation; +import com.facebook.presto.spi.function.SqlFunctionHandle; +import com.facebook.presto.spi.function.SqlFunctionId; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Injector; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import javax.ws.rs.GET; +import javax.ws.rs.HEAD; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriBuilder; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +@Test(singleThreaded = true) +public class TestRestBasedFunctionNamespaceManager +{ + private RestBasedFunctionNamespaceManager functionNamespaceManager; + public static final String TEST_CATALOG = "unittest"; + public static final URI REST_SERVER_URI = URI.create("http://127.0.0.1:1122"); + TestingFunctionResource resource; + + @Path("/v1/functions") + public static class TestingFunctionResource + { + private Map> functions; + private String etag = "\"initial-etag\""; + + public TestingFunctionResource(Map> functions) + { + this.functions = functions; + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + public Map> getFunctions() + { + return functions; + } + + @GET + @Path("/{schema}/{functionName}") + @Produces(MediaType.APPLICATION_JSON) + public Map> getFunctionsBySchemaAndName(@PathParam("schema") String schema, @PathParam("functionName") String functionName) + { + Map> result = new HashMap<>(); + List functionList = functions.get(functionName); + + if (functionList != null) { + List filteredList = new ArrayList<>(); + for (JsonBasedUdfFunctionMetadata metadata : functionList) { + if (metadata.getSchema().equals(schema)) { + filteredList.add(metadata); + } + } + if (!filteredList.isEmpty()) { + result.put(functionName, filteredList); + } + } + return result; + } + + @HEAD + public Response getFunctionHeaders() + { + return Response.ok() + .header("ETag", etag) + .build(); + } + + public void updateFunctions(Map> newFunctions) + { + this.functions = newFunctions; + } + + public void updateETag(String newETag) + { + this.etag = newETag; + } + } + + public static Map> createUdfSignatureMap() + { + Map> udfSignatureMap = new HashMap<>(); + + // square function + List squareFunctions = new ArrayList<>(); + squareFunctions.add(new JsonBasedUdfFunctionMetadata( + "square an integer", + FunctionKind.SCALAR, + new TypeSignature("integer"), + Collections.singletonList(new TypeSignature("integer")), + "default", + new RoutineCharacteristics(RoutineCharacteristics.Language.REST, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), + Optional.empty(), + Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.default.square"), ImmutableList.of(parseTypeSignature("integer")))), + Optional.of("1"))); + squareFunctions.add(new JsonBasedUdfFunctionMetadata( + "square a double", + FunctionKind.SCALAR, + new TypeSignature("double"), + Collections.singletonList(new TypeSignature("double")), + "test_schema", + new RoutineCharacteristics(RoutineCharacteristics.Language.REST, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), + Optional.empty(), + Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.square"), ImmutableList.of(parseTypeSignature("double")))), + Optional.of("1"))); + udfSignatureMap.put("square", squareFunctions); + + // array_function_1 + List arrayFunction1 = new ArrayList<>(); + arrayFunction1.add(new JsonBasedUdfFunctionMetadata( + "combines two string arrays into one", + FunctionKind.SCALAR, + parseTypeSignature("ARRAY>"), + Arrays.asList(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>")), + "default", + new RoutineCharacteristics(RoutineCharacteristics.Language.REST, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), + Optional.empty(), + Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.default.array_function_1"), ImmutableList.of(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>")))), + Optional.of("1"))); + arrayFunction1.add(new JsonBasedUdfFunctionMetadata( + "combines two float arrays into one", + FunctionKind.SCALAR, + parseTypeSignature("ARRAY>"), + Arrays.asList(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>")), + "test_schema", + new RoutineCharacteristics(RoutineCharacteristics.Language.REST, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), + Optional.empty(), + Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.array_function_1"), ImmutableList.of(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>")))), + Optional.of("1"))); + arrayFunction1.add(new JsonBasedUdfFunctionMetadata( + "combines two double arrays into one", + FunctionKind.SCALAR, + parseTypeSignature("ARRAY"), + Arrays.asList(parseTypeSignature("ARRAY"), TypeSignature.parseTypeSignature("ARRAY")), + "test_schema", + new RoutineCharacteristics(RoutineCharacteristics.Language.REST, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), + Optional.empty(), + Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.array_function_1"), ImmutableList.of(parseTypeSignature("ARRAY"), parseTypeSignature("ARRAY")))), + Optional.of("1"))); + udfSignatureMap.put("array_function_1", arrayFunction1); + + // array_function_2 + List arrayFunction2 = new ArrayList<>(); + arrayFunction2.add(new JsonBasedUdfFunctionMetadata( + "transforms inputs into the output", + FunctionKind.SCALAR, + TypeSignature.parseTypeSignature("ARRAY>"), + Arrays.asList(TypeSignature.parseTypeSignature("ARRAY>"), TypeSignature.parseTypeSignature("ARRAY")), + "default", + new RoutineCharacteristics(RoutineCharacteristics.Language.REST, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), + Optional.empty(), + Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.default.array_function_2"), ImmutableList.of(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY")))), + Optional.of("1"))); + arrayFunction2.add(new JsonBasedUdfFunctionMetadata( + "transforms inputs into the output", + FunctionKind.SCALAR, + TypeSignature.parseTypeSignature("ARRAY>"), + Arrays.asList(TypeSignature.parseTypeSignature("ARRAY>"), TypeSignature.parseTypeSignature("ARRAY>"), TypeSignature.parseTypeSignature("ARRAY")), + "test_schema", + new RoutineCharacteristics(RoutineCharacteristics.Language.REST, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), + Optional.empty(), + Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.array_function_2"), ImmutableList.of(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY")))), + Optional.of("1"))); + udfSignatureMap.put("array_function_2", arrayFunction2); + + return udfSignatureMap; + } + + public static Map> createUpdatedUdfSignatureMap() + { + Map> udfSignatureMap = new HashMap<>(); + + // square function + List squareFunctions = new ArrayList<>(); + squareFunctions.add(new JsonBasedUdfFunctionMetadata( + "square an integer", + FunctionKind.SCALAR, + new TypeSignature("integer"), + Collections.singletonList(new TypeSignature("integer")), + "default", + new RoutineCharacteristics(RoutineCharacteristics.Language.REST, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), + Optional.empty(), + Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.default.square"), ImmutableList.of(parseTypeSignature("integer")))), + Optional.of("1"))); + squareFunctions.add(new JsonBasedUdfFunctionMetadata( + "square a double", + FunctionKind.SCALAR, + new TypeSignature("double"), + Collections.singletonList(new TypeSignature("double")), + "test_schema", + new RoutineCharacteristics(RoutineCharacteristics.Language.REST, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), + Optional.empty(), + Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.square"), ImmutableList.of(parseTypeSignature("double")))), + Optional.of("1"))); + udfSignatureMap.put("square", squareFunctions); + + return udfSignatureMap; + } + + @BeforeMethod + public void setUp() throws Exception + { + resource = new TestingFunctionResource(createUdfSignatureMap()); + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new Jdk8Module()); + mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + + JaxrsTestingHttpProcessor httpProcessor = new JaxrsTestingHttpProcessor( + UriBuilder.fromUri(REST_SERVER_URI).path("/").build(), + resource, + new JsonMapper(mapper)); + TestingHttpClient testingHttpClient = new TestingHttpClient(httpProcessor); + + functionNamespaceManager = createFunctionNamespaceManager(testingHttpClient); + } + + private RestBasedFunctionNamespaceManager createFunctionNamespaceManager(HttpClient httpClient) + { + Bootstrap app = new Bootstrap( + // Specially use a testing HTTP client instead of a real one + binder -> binder.bind(HttpClient.class).annotatedWith(ForRestServer.class).toInstance(httpClient), + new RestBasedFunctionNamespaceManagerModule(TEST_CATALOG), + new NoopSqlFunctionExecutorsModule()); + + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties( + ImmutableMap.of( + "rest-based-function-manager.rest.url", REST_SERVER_URI.toString(), + "supported-function-languages", "REST", + "function-implementation-type", "REST") + ).initialize(); + return injector.getInstance(RestBasedFunctionNamespaceManager.class); + } + + @Test + public void testListFunctions() + { + // Verify list of all functions + Collection functionList = functionNamespaceManager.listFunctions(Optional.empty(), Optional.empty()); + assertEquals(functionList.size(), 7); + + // Invalidate the cached list and verify the list of all functions. + resource.updateFunctions(createUpdatedUdfSignatureMap()); + resource.updateETag("\"updated-etag\""); + functionList = functionNamespaceManager.listFunctions(Optional.empty(), Optional.empty()); + assertEquals(functionList.size(), 2); + } + + @Test + public void testFetchFunctionsDirect() + { + QualifiedObjectName functionName = QualifiedObjectName.valueOf("unittest.test_schema.square"); + Collection functions = functionNamespaceManager.getFunctions(Optional.empty(), functionName); + assertEquals(functions.size(), 1); + SqlInvokedFunction function = functions.iterator().next(); + assertEquals(function.getSignature().getName(), functionName); + assertEquals(function.getSignature().getReturnType().getBase(), "double"); + } + + @Test + public void testFetchFunctionImplementationDirect() + { + QualifiedObjectName functionName = QualifiedObjectName.valueOf("unittest.default.square"); + List argumentTypes = ImmutableList.of(parseTypeSignature("integer")); + SqlFunctionId functionId = new SqlFunctionId(functionName, argumentTypes); + SqlFunctionHandle functionHandle = new SqlFunctionHandle(functionId, "1"); + ScalarFunctionImplementation implementation = functionNamespaceManager.getScalarFunctionImplementation(functionHandle); + assertNotNull(implementation); + } +} diff --git a/presto-openapi/src/main/resources/rest_function_server.yaml b/presto-openapi/src/main/resources/rest_function_server.yaml new file mode 100644 index 0000000000000..8f64f9469b987 --- /dev/null +++ b/presto-openapi/src/main/resources/rest_function_server.yaml @@ -0,0 +1,286 @@ +openapi: 3.0.0 +info: + title: Presto Remote Function Rest APIs + description: These APIs shall be implemented by REST server for retrieval and execution of remote functions. + version: "1" +servers: + - url: http://localhost:8080 + description: Presto endpoint when running locally +paths: + /v1/functions: + head: + summary: Retrieve the headers of the GET which list of functions supported by REST server. + description: | + This is useful for checking the version of the API, which will be returned as a header. + The `HEAD` request retrieves the headers that would be returned by the `GET` request. + responses: + '200': + description: Successfully retrieved headers. The body is empty. + headers: + Last-Modified: + description: The date and time when the resource was last modified. + schema: + type: string + format: date-time + ETag: + description: An identifier for a specific version of the resource. + schema: + type: string + '404': + description: The function list could not be found. + get: + summary: Retrieve list of all functions supported by REST server. + description: | + This endpoint retrieves all the function signatures that are available on the REST server. + The response includes details about each function, such as its kind, input and output types, + schema, and characteristics like determinism and null handling. + responses: + '200': + description: A map of function names to lists of function metadata. + content: + application/json: + schema: + $ref: '#/components/schemas/UdfSignatureMap' + /v1/functions/{schema}: + parameters: + - name: schema + in: path + required: true + schema: + type: string + description: The schema in which the function is defined. + get: + summary: Retrieve list of functions in the specified schema. + description: | + This endpoint returns the complete listing of all functions in the specified schema. + The response includes details about each function, such as its kind, input and output types, + schema, and characteristics like determinism and null handling. + responses: + '200': + description: A map of function names to lists of function metadata. + content: + application/json: + schema: + $ref: '#/components/schemas/UdfSignatureMap' + /v1/functions/{schema}/{functionName}: + parameters: + - name: schema + in: path + required: true + schema: + type: string + description: The schema in which the function is defined. + - name: functionName + in: path + required: true + schema: + type: string + description: The name of the function. + get: + summary: Retrieve list of functions with at schema with name. + description: | + This endpoint returns the complete listing of functions in the specified schema with the specified function name. + The response includes details about each function, such as its kind, input and output types, + schema, and characteristics like determinism and null handling. + responses: + '200': + description: A map of function names to lists of function metadata. + content: + application/json: + schema: + $ref: '#/components/schemas/UdfSignatureMap' + post: + summary: Add a new function in the specified schema with the specified name. + description: | + This endpoint creates a new function in the specified schema. The function object will contain the metadata of the function, + including its arguments, return type, and other metadata. + requestBody: + description: JSON object representing the function to be added. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/JsonBasedUdfFunctionMetadata' + responses: + '201': + description: The function was successfully created, and the function ID is returned. + content: + application/json: + schema: + type: string + description: The function ID of the newly created function. + '400': + description: Invalid request due to malformed input. + '409': + description: A function with the same name and signature already exists. + /v1/functions/{schema}/{functionName}/{functionId}: + parameters: + - name: schema + in: path + required: true + schema: + type: string + description: The schema in which the function is defined. + - name: functionName + in: path + required: true + schema: + type: string + description: The name of the function. + - name: functionId + in: path + required: true + schema: + type: string + description: The ID of the function. + put: + summary: Update the function in the specified schema with the specified name and function ID. + description: | + This endpoint updates the function in the specified schema. The function object will contain the updated metadata of the function. + requestBody: + description: JSON object representing the function to be updated. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/JsonBasedUdfFunctionMetadata' + responses: + '200': + description: The function was successfully updated, and the function ID is returned. + content: + application/json: + schema: + type: string + description: The function ID of the updated function. + '400': + description: Invalid request due to malformed input. + '404': + description: The function was not found. + '409': + description: Error occurred while updating the function. + delete: + summary: Delete the function in the specified schema with the specified name and function ID. + description: | + This endpoint deletes the function in the specified schema. The function is identified by its name and ID.. + responses: + '204': + description: The function was successfully deleted. The response body is empty. + '404': + description: The function was not found. +components: + schemas: + UdfSignatureMap: + type: object + description: A map of function names to lists of function metadata. + additionalProperties: + type: array + items: + $ref: '#/components/schemas/JsonBasedUdfFunctionMetadata' + JsonBasedUdfFunctionMetadata: + type: object + properties: + docString: + type: string + description: A description of what the function does. + example: "Returns the square of a number." + functionKind: + $ref: '#/components/schemas/FunctionKind' + outputType: + type: string + description: The output type of the function. + example: "integer" + paramTypes: + type: array + items: + type: string + description: The input parameter types of the function. + example: ["integer"] + schema: + type: string + description: The schema to which the function belongs. + example: "test_schema" + routineCharacteristics: + $ref: '#/components/schemas/RoutineCharacteristics' + aggregateMetadata: + $ref: '#/components/schemas/AggregateMetadata' + functionId: + $ref: '#/components/schemas/SqlFunctionId' + version: + type: string + description: The version of the function. This version shall be maintained by REST server for any change in the function. + example: "1" + FunctionKind: + type: string + description: The kind of function. + enum: + - SCALAR + - AGGREGATE + - WINDOW + example: "SCALAR" + RoutineCharacteristics: + type: object + properties: + language: + $ref: '#/components/schemas/Language' + determinism: + $ref: '#/components/schemas/Determinism' + nullCallClause: + $ref: '#/components/schemas/NullCallClause' + Language: + type: string + description: The implementation language of the function. + enum: + - SQL + - CPP + - REST + example: "REST" + Determinism: + type: string + description: Whether the function is deterministic. + enum: + - DETERMINISTIC + - NOT_DETERMINISTIC + example: "DETERMINISTIC" + NullCallClause: + type: string + description: How the function handles null inputs. + enum: + - RETURNS_NULL_ON_NULL_INPUT + - CALLED_ON_NULL_INPUT + example: "CALLED_ON_NULL_INPUT" + AggregateMetadata: + type: object + nullable: true + properties: + intermediateType: + type: string + description: The intermediate type used in aggregation. + example: "ROW(bigint, int)" + isOrderSensitive: + type: boolean + description: Whether the aggregation is sensitive to the order of inputs. + example: false + QualifiedObjectName: + type: object + properties: + catalogName: + type: string + description: The name of the catalog. + schemaName: + type: string + description: The name of the schema. + objectName: + type: string + description: The name of the function. + TypeSignature: + type: string + description: Serialized signature of the type. + SqlFunctionId: + type: object + properties: + functionName: + $ref: '#/components/schemas/QualifiedObjectName' + argumentTypes: + type: array + items: + $ref: '#/components/schemas/TypeSignature' \ No newline at end of file diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionImplementationType.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionImplementationType.java index 88bc6d8e1ebe9..12bf8c2376f1c 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionImplementationType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionImplementationType.java @@ -19,7 +19,8 @@ public enum FunctionImplementationType SQL(false, true), THRIFT(true, false), GRPC(true, false), - CPP(false, false); + CPP(false, false), + REST(false, true); private final boolean externalExecution; private final boolean evaluatedInCoordinator; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/RestFunctionHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/RestFunctionHandle.java new file mode 100644 index 0000000000000..375a426c837e3 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/RestFunctionHandle.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.api.Experimental; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import static java.util.Objects.requireNonNull; + +@Experimental +public class RestFunctionHandle + extends SqlFunctionHandle +{ + private final Signature signature; + + @JsonCreator + public RestFunctionHandle( + @JsonProperty("functionId") SqlFunctionId functionId, + @JsonProperty("version") String version, + @JsonProperty("signature") Signature signature) + { + super(functionId, version); + this.signature = requireNonNull(signature, "signature is null"); + } + + @JsonProperty + public Signature getSignature() + { + return signature; + } + + public static class Resolver + implements FunctionHandleResolver + { + @Override + public Class getFunctionHandleClass() + { + return RestFunctionHandle.class; + } + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/RoutineCharacteristics.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/RoutineCharacteristics.java index 75e4cda6e05b4..0f8c020989266 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/RoutineCharacteristics.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/RoutineCharacteristics.java @@ -19,6 +19,7 @@ import com.facebook.drift.annotations.ThriftField; import com.facebook.drift.annotations.ThriftStruct; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; @@ -40,6 +41,7 @@ public static class Language { public static final Language SQL = new Language("SQL"); public static final Language CPP = new Language("CPP"); + public static final Language REST = new Language("REST"); private final String language; @@ -165,11 +167,13 @@ public NullCallClause getNullCallClause() return nullCallClause; } + @JsonIgnore public boolean isDeterministic() { return determinism == DETERMINISTIC; } + @JsonIgnore public boolean isCalledOnNullInput() { return nullCallClause == CALLED_ON_NULL_INPUT; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java index 17e720a580277..d16f5f92f46b8 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java @@ -136,7 +136,39 @@ public SqlInvokedFunction( throw new IllegalArgumentException("aggregationMetadata must be present for aggregation functions and absent otherwise"); } } + public SqlInvokedFunction( + QualifiedObjectName functionName, + List parameters, + List typeVariableConstraints, + TypeSignature returnType, + String description, + RoutineCharacteristics routineCharacteristics, + String body, + FunctionVersion version, + FunctionKind kind, + SqlFunctionId functionId, + Optional aggregationMetadata, + Optional functionHandle) + { + this.parameters = requireNonNull(parameters, "parameters is null"); + this.description = requireNonNull(description, "description is null"); + this.routineCharacteristics = requireNonNull(routineCharacteristics, "routineCharacteristics is null"); + this.body = requireNonNull(body, "body is null"); + List argumentTypes = parameters.stream() + .map(Parameter::getType) + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); + + this.signature = new Signature(functionName, kind, typeVariableConstraints, emptyList(), returnType, argumentTypes, false); + this.functionId = requireNonNull(functionId, "functionId is null"); + this.functionVersion = requireNonNull(version, "version is null"); + this.functionHandle = requireNonNull(functionHandle, "functionHandle is null"); + this.aggregationMetadata = requireNonNull(aggregationMetadata, "aggregationMetadata is null"); + + if ((kind == AGGREGATE && !aggregationMetadata.isPresent()) || (kind != AGGREGATE && aggregationMetadata.isPresent())) { + throw new IllegalArgumentException("aggregationMetadata must be present for aggregation functions and absent otherwise"); + } + } public SqlInvokedFunction withVersion(String version) { if (hasVersion()) {