diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java index e1318ba4ab3c6..b53c1b129a5c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java @@ -57,10 +57,10 @@ public Response registerFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("data") FormDataContentDisposition fileDetail, - final @FormDataParam("functionConfig") String functionConfigJson) { + final @FormDataParam("functionDetails") String functionDetailsJson) { return functions.registerFunction( - tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson); + tenant, namespace, functionName, uploadedInputStream, fileDetail, functionDetailsJson); } @@ -72,10 +72,10 @@ public Response updateFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("data") FormDataContentDisposition fileDetail, - final @FormDataParam("functionConfig") String functionConfigJson) { + final @FormDataParam("functionDetails") String functionDetailsJson) { return functions.updateFunction( - tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson); + tenant, namespace, functionName, uploadedInputStream, fileDetail, functionDetailsJson); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java index 3808a488d8135..4a3dbe5d7b70f 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java @@ -21,7 +21,7 @@ import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException; import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException; import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; import org.apache.pulsar.functions.shaded.proto.InstanceCommunication.FunctionStatusList; import java.util.List; @@ -73,24 +73,24 @@ public interface Functions { * @throws PulsarAdminException * Unexpected error */ - FunctionConfig getFunction(String tenant, String namespace, String function) throws PulsarAdminException; + FunctionDetails getFunction(String tenant, String namespace, String function) throws PulsarAdminException; /** * Create a new function. * - * @param functionConfig + * @param functionDetails * the function configuration object * * @throws PulsarAdminException * Unexpected error */ - void createFunction(FunctionConfig functionConfig, String fileName) throws PulsarAdminException; + void createFunction(FunctionDetails functionDetails, String fileName) throws PulsarAdminException; /** * Update the configuration for a function. *

* - * @param functionConfig + * @param functionDetails * the function configuration object * * @throws NotAuthorizedException @@ -100,7 +100,7 @@ public interface Functions { * @throws PulsarAdminException * Unexpected error */ - void updateFunction(FunctionConfig functionConfig, String fileName) throws PulsarAdminException; + void updateFunction(FunctionDetails functionDetails, String fileName) throws PulsarAdminException; /** * Delete an existing function diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java index 175e4f31c73bf..eb793cce39679 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java @@ -23,7 +23,7 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.common.policies.data.*; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; import org.apache.pulsar.functions.shaded.proto.InstanceCommunication.FunctionStatusList; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataMultiPart; @@ -68,16 +68,16 @@ public List getFunctions(String tenant, String namespace) throws PulsarA } @Override - public FunctionConfig getFunction(String tenant, String namespace, String function) throws PulsarAdminException { + public FunctionDetails getFunction(String tenant, String namespace, String function) throws PulsarAdminException { try { Response response = request(functions.path(tenant).path(namespace).path(function)).get(); if (!response.getStatusInfo().equals(Response.Status.OK)) { throw new ClientErrorException(response); } String jsonResponse = response.readEntity(String.class); - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - mergeJson(jsonResponse, functionConfigBuilder); - return functionConfigBuilder.build(); + FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); + mergeJson(jsonResponse, functionDetailsBuilder); + return functionDetailsBuilder.build(); } catch (Exception e) { throw getApiException(e); } @@ -101,16 +101,16 @@ public FunctionStatusList getFunctionStatus( } @Override - public void createFunction(FunctionConfig functionConfig, String fileName) throws PulsarAdminException { + public void createFunction(FunctionDetails functionDetails, String fileName) throws PulsarAdminException { try { final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(new FileDataBodyPart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM_TYPE)); - mp.bodyPart(new FormDataBodyPart("functionConfig", - printJson(functionConfig), + mp.bodyPart(new FormDataBodyPart("functionDetails", + printJson(functionDetails), MediaType.APPLICATION_JSON_TYPE)); - request(functions.path(functionConfig.getTenant()).path(functionConfig.getNamespace()).path(functionConfig.getName())) + request(functions.path(functionDetails.getTenant()).path(functionDetails.getNamespace()).path(functionDetails.getName())) .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA), ErrorData.class); } catch (Exception e) { throw getApiException(e); @@ -128,16 +128,16 @@ public void deleteFunction(String cluster, String namespace, String function) th } @Override - public void updateFunction(FunctionConfig functionConfig, String fileName) throws PulsarAdminException { + public void updateFunction(FunctionDetails functionDetails, String fileName) throws PulsarAdminException { try { final FormDataMultiPart mp = new FormDataMultiPart(); if (fileName != null) { mp.bodyPart(new FileDataBodyPart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM_TYPE)); } - mp.bodyPart(new FormDataBodyPart("functionConfig", - printJson(functionConfig), + mp.bodyPart(new FormDataBodyPart("functionDetails", + printJson(functionDetails), MediaType.APPLICATION_JSON_TYPE)); - request(functions.path(functionConfig.getTenant()).path(functionConfig.getNamespace()).path(functionConfig.getName())) + request(functions.path(functionDetails.getTenant()).path(functionDetails.getNamespace()).path(functionDetails.getName())) .put(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA), ErrorData.class); } catch (Exception e) { throw getApiException(e); diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java index 5f745d15a8c7e..83220d027053a 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java @@ -53,7 +53,7 @@ import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.utils.DefaultSerDe; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf; import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil; import org.apache.pulsar.functions.utils.Reflections; @@ -188,7 +188,7 @@ public void testCreateFunction() throws Exception { assertEquals(inputTopicName, creater.getInputs()); assertEquals(outputTopicName, creater.getOutput()); - verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString()); + verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString()); } @@ -209,7 +209,7 @@ public void testCreateWithoutTenant() throws Exception { CreateFunction creater = cmd.getCreater(); assertEquals("tenant", creater.getFunctionConfig().getTenant()); - verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString()); + verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString()); } @Test @@ -229,7 +229,7 @@ public void testCreateWithoutNamespace() throws Exception { CreateFunction creater = cmd.getCreater(); assertEquals("tenant", creater.getFunctionConfig().getTenant()); assertEquals("namespace", creater.getFunctionConfig().getNamespace()); - verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString()); + verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString()); } @Test @@ -254,7 +254,7 @@ public void testCreateUsingFullyQualifiedFunctionName() throws Exception { assertEquals(tenant, creater.getFunctionConfig().getTenant()); assertEquals(namespace, creater.getFunctionConfig().getNamespace()); assertEquals(functionName, creater.getFunctionConfig().getName()); - verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString()); + verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString()); } @Test @@ -273,7 +273,7 @@ public void testCreateWithoutFunctionName() throws Exception { CreateFunction creater = cmd.getCreater(); assertEquals("CmdFunctionsTest$DummyFunction", creater.getFunctionConfig().getName()); - verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString()); + verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString()); } @Test @@ -290,7 +290,7 @@ public void testCreateWithoutOutputTopic() throws Exception { CreateFunction creater = cmd.getCreater(); assertEquals(inputTopicName + "-" + "CmdFunctionsTest$DummyFunction" + "-output", creater.getFunctionConfig().getOutput()); - verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString()); + verify(functions, times(1)).createFunction(any(FunctionDetails.class), anyString()); } @Test @@ -359,7 +359,7 @@ public void testUpdateFunction() throws Exception { assertEquals(inputTopicName, updater.getInputs()); assertEquals(outputTopicName, updater.getOutput()); - verify(functions, times(1)).updateFunction(any(FunctionConfig.class), anyString()); + verify(functions, times(1)).updateFunction(any(FunctionDetails.class), anyString()); } @Test diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 66b8630b5eeef..71ba928cb7371 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -32,6 +32,8 @@ import java.util.Map; import java.util.UUID; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.apache.bookkeeper.api.StorageClient; import org.apache.bookkeeper.api.kv.Table; import org.apache.bookkeeper.api.kv.result.KeyValue; @@ -43,6 +45,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.functions.api.Function; +import org.apache.pulsar.functions.utils.FunctionConfig; import org.apache.pulsar.functions.api.SerDe; import org.apache.pulsar.functions.api.utils.DefaultSerDe; import org.apache.pulsar.functions.instance.InstanceConfig; @@ -51,8 +54,7 @@ import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf; import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil; import org.apache.pulsar.functions.shaded.io.netty.buffer.Unpooled; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionConfig; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; import org.apache.pulsar.functions.utils.Reflections; import org.apache.pulsar.functions.utils.Utils; @@ -162,7 +164,7 @@ void processArguments() throws Exception { * Commands that require a function config */ @Getter - abstract class FunctionConfigCommand extends BaseCommand { + abstract class FunctionDetailsCommand extends BaseCommand { @Parameter(names = "--fqfn", description = "The Fully Qualified Function Name (FQFN) for the function") protected String fqfn; @Parameter(names = "--tenant", description = "The function's tenant") @@ -211,103 +213,100 @@ abstract class FunctionConfigCommand extends BaseCommand { void processArguments() throws Exception { super.processArguments(); - FunctionConfig.Builder functionConfigBuilder; - // Initialize config builder either from a supplied YAML config file or from scratch if (null != fnConfigFile) { - functionConfigBuilder = loadConfig(new File(fnConfigFile)); + functionConfig = loadConfig(new File(fnConfigFile)); } else { - functionConfigBuilder = FunctionConfig.newBuilder(); + functionConfig = new FunctionConfig(); } if (null != fqfn) { - parseFullyQualifiedFunctionName(fqfn, functionConfigBuilder); + parseFullyQualifiedFunctionName(fqfn, functionConfig); } else { if (null != tenant) { - functionConfigBuilder.setTenant(tenant); + functionConfig.setTenant(tenant); } if (null != namespace) { - functionConfigBuilder.setNamespace(namespace); + functionConfig.setNamespace(namespace); } if (null != functionName) { - functionConfigBuilder.setName(functionName); + functionConfig.setName(functionName); } } if (null != inputs) { - Arrays.asList(inputs.split(",")).forEach(functionConfigBuilder::addInputs); + Arrays.asList(inputs.split(",")).forEach(functionConfig.getInputs()::add); } if (null != customSerdeInputString) { Type type = new TypeToken>(){}.getType(); Map customSerdeInputMap = new Gson().fromJson(customSerdeInputString, type); - functionConfigBuilder.putAllCustomSerdeInputs(customSerdeInputMap); + functionConfig.setCustomSerdeInputs(customSerdeInputMap); } if (null != output) { - functionConfigBuilder.setOutput(output); + functionConfig.setOutput(output); } if (null != logTopic) { - functionConfigBuilder.setLogTopic(logTopic); + functionConfig.setLogTopic(logTopic); } if (null != className) { - functionConfigBuilder.setClassName(className); + functionConfig.setClassName(className); } if (null != outputSerdeClassName) { - functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); } if (null != processingGuarantees) { - functionConfigBuilder.setProcessingGuarantees(processingGuarantees); + functionConfig.setProcessingGuarantees(processingGuarantees); } if (null != subscriptionType) { - functionConfigBuilder.setSubscriptionType(subscriptionType); + functionConfig.setSubscriptionType(subscriptionType); } if (null != userConfigString) { Type type = new TypeToken>(){}.getType(); Map userConfigMap = new Gson().fromJson(userConfigString, type); - functionConfigBuilder.putAllUserConfig(userConfigMap); + functionConfig.setUserConfig(userConfigMap); } if (null != jarFile) { - doJavaSubmitChecks(functionConfigBuilder); - functionConfigBuilder.setRuntime(FunctionConfig.Runtime.JAVA); + doJavaSubmitChecks(functionConfig); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); userCodeFile = jarFile; } else if (null != pyFile) { - doPythonSubmitChecks(functionConfigBuilder); - functionConfigBuilder.setRuntime(FunctionConfig.Runtime.PYTHON); + doPythonSubmitChecks(functionConfig); + functionConfig.setRuntime(FunctionConfig.Runtime.PYTHON); userCodeFile = pyFile; } else { throw new RuntimeException("Either a Java jar or a Python file needs to be specified for the function"); } - if (functionConfigBuilder.getInputsCount() == 0 && functionConfigBuilder.getCustomSerdeInputsCount() == 0) { + if (functionConfig.getInputs().size() == 0 && functionConfig.getCustomSerdeInputs().size() == 0) { throw new RuntimeException("No input topic(s) specified for the function"); } if (parallelism == null) { - if (functionConfigBuilder.getParallelism() == 0) { - functionConfigBuilder.setParallelism(1); + if (functionConfig.getParallelism() == 0) { + functionConfig.setParallelism(1); } } else { int num = Integer.parseInt(parallelism); if (num <= 0) { throw new IllegalArgumentException("The parallelism factor (the number of instances) for the function must be positive"); } - functionConfigBuilder.setParallelism(num); + functionConfig.setParallelism(num); } - functionConfigBuilder.setAutoAck(true); - inferMissingArguments(functionConfigBuilder); - functionConfig = functionConfigBuilder.build(); + functionConfig.setAutoAck(true); + inferMissingArguments(functionConfig); } - private void doJavaSubmitChecks(FunctionConfig.Builder functionConfigBuilder) { + private void doJavaSubmitChecks(FunctionConfig functionConfig) { File file = new File(jarFile); // check if the function class exists in Jar and it implements Function class - if (!Reflections.classExistsInJar(file, functionConfigBuilder.getClassName())) { + if (!Reflections.classExistsInJar(file, functionConfig.getClassName())) { throw new IllegalArgumentException(String.format("Pulsar function class %s does not exist in jar %s", - functionConfigBuilder.getClassName(), jarFile)); - } else if (!Reflections.classInJarImplementsIface(file, functionConfigBuilder.getClassName(), Function.class) - && !Reflections.classInJarImplementsIface(file, functionConfigBuilder.getClassName(), java.util.function.Function.class)) { + functionConfig.getClassName(), jarFile)); + } else if (!Reflections.classInJarImplementsIface(file, functionConfig.getClassName(), Function.class) + && !Reflections.classInJarImplementsIface(file, functionConfig.getClassName(), java.util.function.Function.class)) { throw new IllegalArgumentException(String.format("The Pulsar function class %s in jar %s implements neither org.apache.pulsar.functions.api.Function nor java.util.function.Function", - functionConfigBuilder.getClassName(), jarFile)); + functionConfig.getClassName(), jarFile)); } ClassLoader userJarLoader; @@ -317,27 +316,27 @@ private void doJavaSubmitChecks(FunctionConfig.Builder functionConfigBuilder) { throw new RuntimeException("Failed to load user jar " + file, e); } - Object userClass = Reflections.createInstance(functionConfigBuilder.getClassName(), file); + Object userClass = Reflections.createInstance(functionConfig.getClassName(), file); Class[] typeArgs; if (userClass instanceof Function) { Function pulsarFunction = (Function) userClass; if (pulsarFunction == null) { throw new IllegalArgumentException(String.format("The Pulsar function class %s could not be instantiated from jar %s", - functionConfigBuilder.getClassName(), jarFile)); + functionConfig.getClassName(), jarFile)); } typeArgs = TypeResolver.resolveRawArguments(Function.class, pulsarFunction.getClass()); } else { java.util.function.Function function = (java.util.function.Function) userClass; if (function == null) { throw new IllegalArgumentException(String.format("The Java util function class %s could not be instantiated from jar %s", - functionConfigBuilder.getClassName(), jarFile)); + functionConfig.getClassName(), jarFile)); } typeArgs = TypeResolver.resolveRawArguments(java.util.function.Function.class, function.getClass()); } // Check if the Input serialization/deserialization class exists in jar or already loaded and that it // implements SerDe class - functionConfigBuilder.getCustomSerdeInputsMap().forEach((topicName, inputSerializer) -> { + functionConfig.getCustomSerdeInputs().forEach((topicName, inputSerializer) -> { if (!Reflections.classExists(inputSerializer) && !Reflections.classExistsInJar(new File(jarFile), inputSerializer)) { throw new IllegalArgumentException( @@ -382,23 +381,23 @@ private void doJavaSubmitChecks(FunctionConfig.Builder functionConfigBuilder) { } } }); - functionConfigBuilder.getInputsList().forEach((topicName) -> { + functionConfig.getInputs().forEach((topicName) -> { if (!DefaultSerDe.IsSupportedType(typeArgs[0])) { throw new RuntimeException("Default Serializer does not support type " + typeArgs[0]); } }); if (!Void.class.equals(typeArgs[1])) { - if (functionConfigBuilder.getOutputSerdeClassName() == null - || functionConfigBuilder.getOutputSerdeClassName().isEmpty() - || functionConfigBuilder.getOutputSerdeClassName().equals(DefaultSerDe.class.getName())) { + if (functionConfig.getOutputSerdeClassName() == null + || functionConfig.getOutputSerdeClassName().isEmpty() + || functionConfig.getOutputSerdeClassName().equals(DefaultSerDe.class.getName())) { if (!DefaultSerDe.IsSupportedType(typeArgs[1])) { throw new RuntimeException("Default Serializer does not support type " + typeArgs[1]); } } else { - SerDe serDe = (SerDe) Reflections.createInstance(functionConfigBuilder.getOutputSerdeClassName(), file); + SerDe serDe = (SerDe) Reflections.createInstance(functionConfig.getOutputSerdeClassName(), file); if (serDe == null) { throw new IllegalArgumentException(String.format("SerDe class %s does not exist in jar %s", - functionConfigBuilder.getOutputSerdeClassName(), jarFile)); + functionConfig.getOutputSerdeClassName(), jarFile)); } Class[] serDeTypes = TypeResolver.resolveRawArguments(SerDe.class, serDe.getClass()); @@ -420,78 +419,78 @@ private void doJavaSubmitChecks(FunctionConfig.Builder functionConfigBuilder) { } } - private void doPythonSubmitChecks(FunctionConfig.Builder functionConfigBuilder) { - if (functionConfigBuilder.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) { + private void doPythonSubmitChecks(FunctionConfig functionConfig) { + if (functionConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) { throw new RuntimeException("Effectively-once processing guarantees not yet supported in Python"); } } - private void inferMissingArguments(FunctionConfig.Builder builder) { - if (builder.getName() == null || builder.getName().isEmpty()) { - inferMissingFunctionName(builder); + private void inferMissingArguments(FunctionConfig functionConfig) { + if (functionConfig.getName() == null || functionConfig.getName().isEmpty()) { + inferMissingFunctionName(functionConfig); } - if (builder.getTenant() == null || builder.getTenant().isEmpty()) { - inferMissingTenant(builder); + if (functionConfig.getTenant() == null || functionConfig.getTenant().isEmpty()) { + inferMissingTenant(functionConfig); } - if (builder.getNamespace() == null || builder.getNamespace().isEmpty()) { - inferMissingNamespace(builder); + if (functionConfig.getNamespace() == null || functionConfig.getNamespace().isEmpty()) { + inferMissingNamespace(functionConfig); } - if (builder.getOutput() == null || builder.getOutput().isEmpty()) { - inferMissingOutput(builder); + if (functionConfig.getOutput() == null || functionConfig.getOutput().isEmpty()) { + inferMissingOutput(functionConfig); } } - private void inferMissingFunctionName(FunctionConfig.Builder builder) { - String [] domains = builder.getClassName().split("\\."); + private void inferMissingFunctionName(FunctionConfig functionConfig) { + String [] domains = functionConfig.getClassName().split("\\."); if (domains.length == 0) { - builder.setName(builder.getClassName()); + functionConfig.setName(functionConfig.getClassName()); } else { - builder.setName(domains[domains.length - 1]); + functionConfig.setName(domains[domains.length - 1]); } } - private void inferMissingTenant(FunctionConfig.Builder builder) { + private void inferMissingTenant(FunctionConfig functionConfig) { try { - String inputTopic = getUniqueInput(builder); - builder.setTenant(TopicName.get(inputTopic).getProperty()); + String inputTopic = getUniqueInput(functionConfig); + functionConfig.setTenant(TopicName.get(inputTopic).getProperty()); } catch (IllegalArgumentException ex) { throw new RuntimeException("You need to specify a tenant for the function", ex); } } - private void inferMissingNamespace(FunctionConfig.Builder builder) { + private void inferMissingNamespace(FunctionConfig functionConfig) { try { - String inputTopic = getUniqueInput(builder); - builder.setNamespace(TopicName.get(inputTopic).getNamespacePortion()); + String inputTopic = getUniqueInput(functionConfig); + functionConfig.setNamespace(TopicName.get(inputTopic).getNamespacePortion()); } catch (IllegalArgumentException ex) { throw new RuntimeException("You need to specify a namespace for the function"); } } - private void inferMissingOutput(FunctionConfig.Builder builder) { + private void inferMissingOutput(FunctionConfig functionConfig) { try { - String inputTopic = getUniqueInput(builder); - builder.setOutput(inputTopic + "-" + builder.getName() + "-output"); + String inputTopic = getUniqueInput(functionConfig); + functionConfig.setOutput(inputTopic + "-" + functionConfig.getName() + "-output"); } catch (IllegalArgumentException ex) { // It might be that we really don't need an output topic // So we cannot really throw an exception } } - private String getUniqueInput(FunctionConfig.Builder builder) { - if (builder.getInputsCount() + builder.getCustomSerdeInputsCount() != 1) { + private String getUniqueInput(FunctionConfig functionConfig) { + if (functionConfig.getInputs().size() + functionConfig.getCustomSerdeInputs().size() != 1) { throw new IllegalArgumentException(); } - if (builder.getInputsCount() == 1) { - return builder.getInputs(0); + if (functionConfig.getInputs().size() == 1) { + return functionConfig.getInputs().iterator().next(); } else { - return builder.getCustomSerdeInputsMap().keySet().iterator().next(); + return functionConfig.getCustomSerdeInputs().keySet().iterator().next(); } } } @Parameters(commandDescription = "Run the Pulsar Function locally (rather than deploying it to the Pulsar cluster)") - class LocalRunner extends FunctionConfigCommand { + class LocalRunner extends FunctionDetailsCommand { // TODO: this should become bookkeeper url and it should be fetched from pulsar client. @Parameter(names = "--stateStorageServiceUrl", description = "The URL for the state storage service (by default Apache BookKeeper)") @@ -518,7 +517,7 @@ void runCmd() throws Exception { List spawners = new LinkedList<>(); for (int i = 0; i < functionConfig.getParallelism(); ++i) { InstanceConfig instanceConfig = new InstanceConfig(); - instanceConfig.setFunctionConfig(convert(functionConfig)); + instanceConfig.setFunctionDetails(convertProto2(functionConfig)); // TODO: correctly implement function version and id instanceConfig.setFunctionVersion(UUID.randomUUID().toString()); instanceConfig.setFunctionId(UUID.randomUUID().toString()); @@ -551,13 +550,13 @@ public void run() { } @Parameters(commandDescription = "Create a Pulsar Function in cluster mode (i.e. deploy it on a Pulsar cluster)") - class CreateFunction extends FunctionConfigCommand { + class CreateFunction extends FunctionDetailsCommand { @Override void runCmd() throws Exception { if (!areAllRequiredFieldsPresent(functionConfig)) { throw new RuntimeException("Missing arguments"); } - admin.functions().createFunction(functionConfig, userCodeFile); + admin.functions().createFunction(convert(functionConfig), userCodeFile); print("Created successfully"); } } @@ -592,13 +591,13 @@ void runCmd() throws Exception { } @Parameters(commandDescription = "Update a Pulsar Function that's been deployed to a Pulsar cluster") - class UpdateFunction extends FunctionConfigCommand { + class UpdateFunction extends FunctionDetailsCommand { @Override void runCmd() throws Exception { if (!areAllRequiredFieldsPresent(functionConfig)) { throw new RuntimeException("Missing arguments"); } - admin.functions().updateFunction(functionConfig, userCodeFile); + admin.functions().updateFunction(convert(functionConfig), userCodeFile); print("Updated successfully"); } } @@ -750,35 +749,106 @@ TriggerFunction getTriggerer() { return triggerer; } - private static FunctionConfig.Builder loadConfig(File file) throws IOException { - String json = FunctionConfigUtils.convertYamlToJson(file); - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - Utils.mergeJson(json, functionConfigBuilder); - return functionConfigBuilder; + private static FunctionConfig loadConfig(File file) throws IOException { + + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + return mapper.readValue(file, FunctionConfig.class); } public static boolean areAllRequiredFieldsPresent(FunctionConfig functionConfig) { return functionConfig.getTenant() != null && functionConfig.getNamespace() != null && functionConfig.getName() != null && functionConfig.getClassName() != null - && (functionConfig.getInputsCount() > 0 || functionConfig.getCustomSerdeInputsCount() > 0) + && (functionConfig.getInputs().size() > 0 || functionConfig.getCustomSerdeInputs().size() > 0) && functionConfig.getParallelism() > 0; } + + private org.apache.pulsar.functions.proto.Function.FunctionDetails convertProto2(FunctionConfig functionConfig) + throws IOException { + org.apache.pulsar.functions.proto.Function.FunctionDetails.Builder functionDetailsBuilder = org.apache.pulsar.functions.proto.Function.FunctionDetails.newBuilder(); + Utils.mergeJson(FunctionsImpl.printJson(convert(functionConfig)), functionDetailsBuilder); + return functionDetailsBuilder.build(); + } - private org.apache.pulsar.functions.proto.Function.FunctionConfig convert(FunctionConfig functionConfig) + private FunctionDetails convert(FunctionConfig functionConfig) throws IOException { - org.apache.pulsar.functions.proto.Function.FunctionConfig.Builder functionConfigBuilder = org.apache.pulsar.functions.proto.Function.FunctionConfig.newBuilder(); - Utils.mergeJson(FunctionsImpl.printJson(functionConfig), functionConfigBuilder); - return functionConfigBuilder.build(); + FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); + if (functionConfig.getInputs() != null) { + functionDetailsBuilder.setTenant(functionConfig.getTenant()); + } + if (functionConfig.getNamespace() != null) { + functionDetailsBuilder.setNamespace(functionConfig.getNamespace()); + } + if (functionConfig.getName() != null) { + functionDetailsBuilder.setName(functionConfig.getName()); + } + if (functionConfig.getClassName() != null) { + functionDetailsBuilder.setClassName(functionConfig.getClassName()); + } + functionDetailsBuilder.putAllCustomSerdeInputs(functionConfig.getCustomSerdeInputs()); + if (functionConfig.getOutputSerdeClassName() != null) { + functionDetailsBuilder.setOutputSerdeClassName(functionConfig.getOutputSerdeClassName()); + } + if (functionConfig.getOutput() != null) { + functionDetailsBuilder.setOutput(functionConfig.getOutput()); + } + if (functionConfig.getLogTopic() != null) { + functionDetailsBuilder.setLogTopic(functionConfig.getLogTopic()); + } + if (functionConfig.getProcessingGuarantees() != null) { + functionDetailsBuilder.setProcessingGuarantees( + convertProcessingGuarantee(functionConfig.getProcessingGuarantees())); + } + functionDetailsBuilder.putAllUserConfig(functionConfig.getUserConfig()); + if (functionConfig.getSubscriptionType() != null) { + functionDetailsBuilder.setSubscriptionType( + convertSubscriptionType(functionConfig.getSubscriptionType())); + } + if (functionConfig.getRuntime() != null) { + functionDetailsBuilder.setRuntime(convertRuntime(functionConfig.getRuntime())); + } + functionDetailsBuilder.setAutoAck(functionConfig.isAutoAck()); + functionDetailsBuilder.addAllInputs(functionConfig.getInputs()); + functionDetailsBuilder.setParallelism(functionConfig.getParallelism()); + return functionDetailsBuilder.build(); + } + + private static FunctionDetails.SubscriptionType convertSubscriptionType( + FunctionConfig.SubscriptionType subscriptionType) { + for (FunctionDetails.SubscriptionType type : FunctionDetails.SubscriptionType.values()) { + if (type.name().equals(subscriptionType.name())) { + return type; + } + } + throw new RuntimeException("Unrecognized subscription type: " + subscriptionType.name()); + } + + private static FunctionDetails.ProcessingGuarantees convertProcessingGuarantee( + FunctionConfig.ProcessingGuarantees processingGuarantees) { + for (FunctionDetails.ProcessingGuarantees type : FunctionDetails.ProcessingGuarantees.values()) { + if (type.name().equals(processingGuarantees.name())) { + return type; + } + } + throw new RuntimeException("Unrecognized processing guarantee: " + processingGuarantees.name()); + } + + private static FunctionDetails.Runtime convertRuntime(FunctionConfig.Runtime runtime) { + for (FunctionDetails.Runtime type : FunctionDetails.Runtime.values()) { + if (type.name().equals(runtime.name())) { + return type; + } + } + throw new RuntimeException("Unrecognized runtime: " + runtime.name()); } - private void parseFullyQualifiedFunctionName(String fqfn, FunctionConfig.Builder functionConfigBuilder) { + private void parseFullyQualifiedFunctionName(String fqfn, FunctionConfig functionConfig) { String[] args = fqfn.split("/"); if (args.length != 3) { throw new RuntimeException("Fully qualified function names (FQFNs) must be of the form tenant/namespace/name"); } else { - functionConfigBuilder.setTenant(args[0]); - functionConfigBuilder.setNamespace(args[1]); - functionConfigBuilder.setName(args[2]); + functionConfig.setTenant(args[0]); + functionConfig.setNamespace(args[1]); + functionConfig.setName(args[2]); } } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java index 3db20102607c4..4d458f6e552c8 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java @@ -131,27 +131,27 @@ public Collection getInputTopics() { @Override public String getOutputTopic() { - return config.getFunctionConfig().getOutput(); + return config.getFunctionDetails().getOutput(); } @Override public String getOutputSerdeClassName() { - return config.getFunctionConfig().getOutputSerdeClassName(); + return config.getFunctionDetails().getOutputSerdeClassName(); } @Override public String getTenant() { - return config.getFunctionConfig().getTenant(); + return config.getFunctionDetails().getTenant(); } @Override public String getNamespace() { - return config.getFunctionConfig().getNamespace(); + return config.getFunctionDetails().getNamespace(); } @Override public String getFunctionName() { - return config.getFunctionConfig().getName(); + return config.getFunctionDetails().getName(); } @Override @@ -176,7 +176,7 @@ public Logger getLogger() { @Override public Optional getUserConfigValue(String key) { - return Optional.ofNullable(config.getFunctionConfig().getUserConfigOrDefault(key, null)); + return Optional.ofNullable(config.getFunctionDetails().getUserConfigOrDefault(key, null)); } @Override @@ -186,7 +186,7 @@ public String getUserConfigValueOrDefault(String key, String defaultValue) { @Override public Map getUserConfigMap() { - return config.getFunctionConfig().getUserConfigMap(); + return config.getFunctionDetails().getUserConfigMap(); } @Override diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java index 3492bb146ac57..5f26e43fc012f 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java @@ -23,7 +23,7 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; /** * This is the config passed to the Java Instance. Contains all the information @@ -38,6 +38,6 @@ public class InstanceConfig { private String instanceId; private String functionId; private String functionVersion; - private FunctionConfig functionConfig; + private FunctionDetails functionDetails; private int maxBufferedTuples; } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstance.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstance.java index 7018bc4fa85f7..f614ea9a2e46f 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstance.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstance.java @@ -50,7 +50,7 @@ public JavaInstance(InstanceConfig config, Object userClassObject, PulsarClient pulsarClient, Map inputConsumers) { // TODO: cache logger instances by functions? - Logger instanceLog = LoggerFactory.getLogger("function-" + config.getFunctionConfig().getName()); + Logger instanceLog = LoggerFactory.getLogger("function-" + config.getFunctionDetails().getName()); this.context = new ContextImpl(config, instanceLog, pulsarClient, clsLoader, inputConsumers); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index 008d5a93297bc..d4b6b7fca5391 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -58,7 +58,7 @@ import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager; import org.apache.pulsar.functions.api.SerDe; import org.apache.pulsar.functions.instance.state.StateContextImpl; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; import org.apache.pulsar.functions.utils.Reflections; /** @@ -115,9 +115,9 @@ public JavaInstanceRunnable(InstanceConfig instanceConfig, this.stateStorageServiceUrl = stateStorageServiceUrl; this.stats = new FunctionStats(); this.processor = MessageProcessor.create( - client, - instanceConfig.getFunctionConfig(), - queue); + client, + instanceConfig.getFunctionDetails(), + queue); } /** @@ -125,10 +125,10 @@ public JavaInstanceRunnable(InstanceConfig instanceConfig, */ JavaInstance setupJavaInstance() throws Exception { // initialize the thread context - ThreadContext.put("function", FunctionConfigUtils.getFullyQualifiedName(instanceConfig.getFunctionConfig())); + ThreadContext.put("function", FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails())); ThreadContext.put("instance", instanceConfig.getInstanceId()); - log.info("Starting Java Instance {}", instanceConfig.getFunctionConfig().getName()); + log.info("Starting Java Instance {}", instanceConfig.getFunctionDetails().getName()); // start the function thread loadJars(); @@ -136,7 +136,7 @@ JavaInstance setupJavaInstance() throws Exception { ClassLoader clsLoader = Thread.currentThread().getContextClassLoader(); Object object = Reflections.createInstance( - instanceConfig.getFunctionConfig().getClassName(), + instanceConfig.getFunctionDetails().getClassName(), clsLoader); if (!(object instanceof Function) && !(object instanceof java.util.function.Function)) { throw new RuntimeException("User class must either be Function or java.util.Function"); @@ -182,7 +182,7 @@ public void run() { log.debug("Received message: {}", msg.getActualMessage().getMessageId()); } catch (InterruptedException ie) { log.info("Function thread {} is interrupted", - FunctionConfigUtils.getFullyQualifiedName(instanceConfig.getFunctionConfig()), ie); + FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails()), ie); break; } @@ -223,12 +223,12 @@ public void run() { if (null != stateContext) { stateContext.flush() - .thenRun(() -> processResult(msg, result, processAt, doneProcessing)) - .exceptionally(cause -> { - // log the messages, since we DONT ack, pulsar consumer will re-deliver the messages. - log.error("Failed to flush the state updates of message {}", msg, cause); - return null; - }); + .thenRun(() -> processResult(msg, result, processAt, doneProcessing)) + .exceptionally(cause -> { + // log the messages, since we DONT ack, pulsar consumer will re-deliver the messages. + log.error("Failed to flush the state updates of message {}", msg, cause); + return null; + }); } else { processResult(msg, result, processAt, doneProcessing); } @@ -245,12 +245,12 @@ private void loadJars() throws Exception { log.info("Loading JAR files for function {} from jarFile {}", instanceConfig, jarFile); // create the function class loader fnCache.registerFunctionInstance( - instanceConfig.getFunctionId(), - instanceConfig.getInstanceId(), - Arrays.asList(jarFile), - Collections.emptyList()); + instanceConfig.getFunctionId(), + instanceConfig.getInstanceId(), + Arrays.asList(jarFile), + Collections.emptyList()); log.info("Initialize function class loader for function {} at function cache manager", - instanceConfig.getFunctionConfig().getName()); + instanceConfig.getFunctionDetails().getName()); this.fnClassLoader = fnCache.getClassLoader(instanceConfig.getFunctionId()); if (null == fnClassLoader) { @@ -267,46 +267,46 @@ private void setupStateTable() throws Exception { } String tableNs = String.format( - "%s_%s", - instanceConfig.getFunctionConfig().getTenant(), - instanceConfig.getFunctionConfig().getNamespace() + "%s_%s", + instanceConfig.getFunctionDetails().getTenant(), + instanceConfig.getFunctionDetails().getNamespace() ).replace('-', '_'); - String tableName = instanceConfig.getFunctionConfig().getName(); + String tableName = instanceConfig.getFunctionDetails().getName(); // TODO (sijie): use endpoint for now StorageClientSettings settings = StorageClientSettings.newBuilder() - .addEndpoints(NetUtils.parseEndpoint(stateStorageServiceUrl)) - .clientName("function-" + tableNs + "/" + tableName) - .build(); + .addEndpoints(NetUtils.parseEndpoint(stateStorageServiceUrl)) + .clientName("function-" + tableNs + "/" + tableName) + .build(); // TODO (sijie): provide a better way to provision the state table for functions try (StorageAdminClient storageAdminClient = StorageClientBuilder.newBuilder() - .withSettings(settings) - .buildAdmin()) { + .withSettings(settings) + .buildAdmin()) { try { result(storageAdminClient.getStream(tableNs, tableName)); } catch (NamespaceNotFoundException nnfe) { result(storageAdminClient.createNamespace(tableNs, NamespaceConfiguration.newBuilder() - .setDefaultStreamConf(DEFAULT_STREAM_CONF) - .build())); + .setDefaultStreamConf(DEFAULT_STREAM_CONF) + .build())); result(storageAdminClient.createStream(tableNs, tableName, DEFAULT_STREAM_CONF)); } catch (StreamNotFoundException snfe) { result(storageAdminClient.createStream(tableNs, tableName, DEFAULT_STREAM_CONF)); } } - log.info("Starting state table for function {}", instanceConfig.getFunctionConfig().getName()); + log.info("Starting state table for function {}", instanceConfig.getFunctionDetails().getName()); this.storageClient = StorageClientBuilder.newBuilder() - .withSettings(settings) - .withNamespace(tableNs) - .build(); + .withSettings(settings) + .withNamespace(tableNs) + .build(); this.stateTable = result(storageClient.openTable(tableName)); } private void processResult(InputMessage msg, JavaExecutionResult result, long startTime, long endTime) { - if (result.getUserException() != null) { + if (result.getUserException() != null) { log.info("Encountered user exception when processing message {}", msg, result.getUserException()); stats.incrementUserExceptions(result.getUserException()); processor.handleProcessException(msg, result.getUserException()); @@ -316,7 +316,7 @@ private void processResult(InputMessage msg, processor.handleProcessException(msg, result.getSystemException()); } else { stats.incrementSuccessfullyProcessed(endTime - startTime); - if (result.getResult() != null && instanceConfig.getFunctionConfig().getOutput() != null) { + if (result.getResult() != null && instanceConfig.getFunctionDetails().getOutput() != null) { byte[] output; try { output = outputSerDe.serialize(result.getResult()); @@ -340,9 +340,9 @@ private void processResult(InputMessage msg, private void sendOutputMessage(InputMessage srcMsg, byte[] output) { MessageBuilder msgBuilder = MessageBuilder.create() - .setContent(output) - .setProperty("__pfn_input_topic__", srcMsg.getTopicName()) - .setProperty("__pfn_input_msg_id__", new String(Base64.getEncoder().encode(srcMsg.getActualMessage().getMessageId().toByteArray()))); + .setContent(output) + .setProperty("__pfn_input_topic__", srcMsg.getTopicName()) + .setProperty("__pfn_input_msg_id__", new String(Base64.getEncoder().encode(srcMsg.getActualMessage().getMessageId().toByteArray()))); processor.sendOutputMessage(srcMsg, msgBuilder); } @@ -367,8 +367,8 @@ public void close() { // once the thread quits, clean up the instance fnCache.unregisterFunctionInstance( - instanceConfig.getFunctionId(), - instanceConfig.getInstanceId()); + instanceConfig.getFunctionId(), + instanceConfig.getInstanceId()); log.info("Unloading JAR files for function {}", instanceConfig); } @@ -415,7 +415,7 @@ public InstanceCommunication.FunctionStatus.Builder getFunctionStatus() { private static void addSystemMetrics(String metricName, double value, InstanceCommunication.MetricsData.Builder bldr) { InstanceCommunication.MetricsData.DataDigest digest = InstanceCommunication.MetricsData.DataDigest.newBuilder() - .setCount(value).setSum(value).setMax(value).setMin(0).build(); + .setCount(value).setSum(value).setMax(value).setMin(0).build(); bldr.putMetrics(metricName, digest); } @@ -449,8 +449,8 @@ private static SerDe initializeDefaultSerDe(Class[] typeArgs, boolean inputAr private void setupSerDe(Class[] typeArgs, ClassLoader clsLoader) { this.inputSerDe = new HashMap<>(); - instanceConfig.getFunctionConfig().getCustomSerdeInputsMap().forEach((k, v) -> this.inputSerDe.put(k, initializeSerDe(v, clsLoader, typeArgs, true))); - for (String topicName : instanceConfig.getFunctionConfig().getInputsList()) { + instanceConfig.getFunctionDetails().getCustomSerdeInputsMap().forEach((k, v) -> this.inputSerDe.put(k, initializeSerDe(v, clsLoader, typeArgs, true))); + for (String topicName : instanceConfig.getFunctionDetails().getInputsList()) { this.inputSerDe.put(topicName, initializeDefaultSerDe(typeArgs, true)); } @@ -473,12 +473,12 @@ private void setupSerDe(Class[] typeArgs, ClassLoader clsLoader) { } if (!Void.class.equals(typeArgs[1])) { // return type is not `Void.class` - if (instanceConfig.getFunctionConfig().getOutputSerdeClassName() == null - || instanceConfig.getFunctionConfig().getOutputSerdeClassName().isEmpty() - || instanceConfig.getFunctionConfig().getOutputSerdeClassName().equals(DefaultSerDe.class.getName())) { + if (instanceConfig.getFunctionDetails().getOutputSerdeClassName() == null + || instanceConfig.getFunctionDetails().getOutputSerdeClassName().isEmpty() + || instanceConfig.getFunctionDetails().getOutputSerdeClassName().equals(DefaultSerDe.class.getName())) { outputSerDe = initializeDefaultSerDe(typeArgs, false); } else { - this.outputSerDe = initializeSerDe(instanceConfig.getFunctionConfig().getOutputSerdeClassName(), clsLoader, typeArgs, false); + this.outputSerDe = initializeSerDe(instanceConfig.getFunctionDetails().getOutputSerdeClassName(), clsLoader, typeArgs, false); } Class[] outputSerdeTypeArgs = TypeResolver.resolveRawArguments(SerDe.class, outputSerDe.getClass()); if (outputSerDe.getClass().getName().equals(DefaultSerDe.class.getName())) { @@ -493,10 +493,10 @@ private void setupSerDe(Class[] typeArgs, ClassLoader clsLoader) { } private void setupLogHandler() { - if (instanceConfig.getFunctionConfig().getLogTopic() != null && - !instanceConfig.getFunctionConfig().getLogTopic().isEmpty()) { - logAppender = new LogAppender(client, instanceConfig.getFunctionConfig().getLogTopic(), - FunctionConfigUtils.getFullyQualifiedName(instanceConfig.getFunctionConfig())); + if (instanceConfig.getFunctionDetails().getLogTopic() != null && + !instanceConfig.getFunctionDetails().getLogTopic().isEmpty()) { + logAppender = new LogAppender(client, instanceConfig.getFunctionDetails().getLogTopic(), + FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails())); logAppender.start(); } } @@ -521,4 +521,4 @@ private void removeLogTopicHandler() { } config.getRootLogger().removeAppender(logAppender.getName()); } -} +} \ No newline at end of file diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtLeastOnceProcessor.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtLeastOnceProcessor.java index 740fd5c73289c..38c10151368b5 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtLeastOnceProcessor.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtLeastOnceProcessor.java @@ -29,7 +29,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.functions.instance.InputMessage; import org.apache.pulsar.functions.instance.producers.AbstractOneOuputTopicProducers; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; /** * A message processor that process messages at-most-once. @@ -41,10 +41,10 @@ public class AtLeastOnceProcessor extends MessageProcessorBase { private Producer producer; AtLeastOnceProcessor(PulsarClient client, - FunctionConfig functionConfig, + FunctionDetails functionDetails, SubscriptionType subType, LinkedBlockingDeque processQueue) { - super(client, functionConfig, subType, processQueue); + super(client, functionDetails, subType, processQueue); } @Override @@ -71,7 +71,7 @@ public void close() { try { producer.close(); } catch (PulsarClientException e) { - log.warn("Fail to close producer for processor {}", functionConfig.getOutput(), e); + log.warn("Fail to close producer for processor {}", functionDetails.getOutput(), e); } } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtMostOnceProcessor.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtMostOnceProcessor.java index 1713b73479796..d9f4eb1f36313 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtMostOnceProcessor.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/AtMostOnceProcessor.java @@ -28,7 +28,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.functions.instance.InputMessage; import org.apache.pulsar.functions.instance.producers.AbstractOneOuputTopicProducers; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; /** * A message processor that process messages at-most-once. @@ -39,16 +39,16 @@ class AtMostOnceProcessor extends MessageProcessorBase { private Producer producer; AtMostOnceProcessor(PulsarClient client, - FunctionConfig functionConfig, + FunctionDetails functionDetails, SubscriptionType subType, LinkedBlockingDeque processQueue) { - super(client, functionConfig, subType, processQueue); + super(client, functionDetails, subType, processQueue); } @Override protected void postReceiveMessage(InputMessage message) { super.postReceiveMessage(message); - if (functionConfig.getAutoAck()) { + if (functionDetails.getAutoAck()) { message.ack(); } } @@ -75,7 +75,7 @@ public void close() { try { producer.close(); } catch (PulsarClientException e) { - log.warn("Fail to close producer for processor {}", functionConfig.getOutput(), e); + log.warn("Fail to close producer for processor {}", functionDetails.getOutput(), e); } } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/EffectivelyOnceProcessor.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/EffectivelyOnceProcessor.java index f039cdfa642cc..5841026282e2b 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/EffectivelyOnceProcessor.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/EffectivelyOnceProcessor.java @@ -38,8 +38,8 @@ import org.apache.pulsar.functions.instance.InputMessage; import org.apache.pulsar.functions.instance.producers.MultiConsumersOneOuputTopicProducers; import org.apache.pulsar.functions.instance.producers.Producers; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; import org.apache.pulsar.functions.utils.Utils; /** @@ -54,9 +54,9 @@ class EffectivelyOnceProcessor extends MessageProcessorBase implements ConsumerE protected Producers outputProducer; EffectivelyOnceProcessor(PulsarClient client, - FunctionConfig functionConfig, + FunctionDetails functionDetails, LinkedBlockingDeque processQueue) { - super(client, functionConfig, SubscriptionType.Failover, processQueue); + super(client, functionDetails, SubscriptionType.Failover, processQueue); } /** @@ -180,7 +180,7 @@ public void sendOutputMessage(InputMessage inputMsg, .thenAccept(messageId -> inputMsg.ackCumulative()) .exceptionally(cause -> { log.error("Failed to send the process result {} of message {} to output topic {}", - outputMsg, inputMsg, functionConfig.getOutput(), cause); + outputMsg, inputMsg, functionDetails.getOutput(), cause); handleProcessException(inputMsg.getTopicName()); return null; }); @@ -251,7 +251,7 @@ private void resubscribe(String srcTopic) { srcTopic, client.subscribe( srcTopic, - FunctionConfigUtils.getFullyQualifiedName(functionConfig), + FunctionDetailsUtils.getFullyQualifiedName(functionDetails), conf )); } catch (PulsarClientException e) { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessor.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessor.java index b475e9f0d6215..1167d731675be 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessor.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessor.java @@ -27,8 +27,8 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.functions.api.SerDe; import org.apache.pulsar.functions.instance.InputMessage; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; -import org.apache.pulsar.functions.proto.Function.FunctionConfig.ProcessingGuarantees; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.Function.FunctionDetails.ProcessingGuarantees; /** * A processor that processes messages, used by {@link org.apache.pulsar.functions.instance.JavaInstanceRunnable}. @@ -37,12 +37,12 @@ public interface MessageProcessor extends AutoCloseable { static MessageProcessor create(PulsarClient client, - FunctionConfig functionConfig, + FunctionDetails functionDetails, LinkedBlockingDeque processQeueue) { - FunctionConfig.SubscriptionType fnSubType = functionConfig.getSubscriptionType(); - ProcessingGuarantees processingGuarantees = functionConfig.getProcessingGuarantees(); + FunctionDetails.SubscriptionType fnSubType = functionDetails.getSubscriptionType(); + ProcessingGuarantees processingGuarantees = functionDetails.getProcessingGuarantees(); SubscriptionType subType; - if (null == fnSubType || FunctionConfig.SubscriptionType.SHARED == fnSubType) { + if (null == fnSubType || FunctionDetails.SubscriptionType.SHARED == fnSubType) { subType = SubscriptionType.Shared; } else { subType = SubscriptionType.Failover; @@ -51,18 +51,18 @@ static MessageProcessor create(PulsarClient client, if (processingGuarantees == ProcessingGuarantees.EFFECTIVELY_ONCE) { return new EffectivelyOnceProcessor( client, - functionConfig, + functionDetails, processQeueue); } else if (processingGuarantees == ProcessingGuarantees.ATMOST_ONCE) { return new AtMostOnceProcessor( client, - functionConfig, + functionDetails, subType, processQeueue); } else { return new AtLeastOnceProcessor( client, - functionConfig, + functionDetails, subType, processQeueue); } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessorBase.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessorBase.java index ddb5f794370cb..46b2800ea6137 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessorBase.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/processors/MessageProcessorBase.java @@ -30,8 +30,8 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.functions.api.SerDe; import org.apache.pulsar.functions.instance.InputMessage; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; /** * The base implementation of {@link MessageProcessor}. @@ -40,7 +40,7 @@ abstract class MessageProcessorBase implements MessageProcessor { protected final PulsarClient client; - protected final FunctionConfig functionConfig; + protected final FunctionDetails functionDetails; protected final SubscriptionType subType; protected final LinkedBlockingDeque processQueue; @@ -51,11 +51,11 @@ abstract class MessageProcessorBase implements MessageProcessor { protected SerDe outputSerDe; protected MessageProcessorBase(PulsarClient client, - FunctionConfig functionConfig, + FunctionDetails functionDetails, SubscriptionType subType, LinkedBlockingDeque processQueue) { this.client = client; - this.functionConfig = functionConfig; + this.functionDetails = functionDetails; this.subType = subType; this.processQueue = processQueue; this.inputConsumers = Maps.newConcurrentMap(); @@ -69,15 +69,15 @@ protected MessageProcessorBase(PulsarClient client, public void setupInput(Map inputSerDe) throws Exception { log.info("Setting up input with input serdes: {}", inputSerDe); this.inputSerDe = inputSerDe; - for (Map.Entry entry : functionConfig.getCustomSerdeInputsMap().entrySet()) { + for (Map.Entry entry : functionDetails.getCustomSerdeInputsMap().entrySet()) { ConsumerConfiguration conf = createConsumerConfiguration(entry.getKey()); this.inputConsumers.put(entry.getKey(), client.subscribe(entry.getKey(), - FunctionConfigUtils.getFullyQualifiedName(functionConfig), conf)); + FunctionDetailsUtils.getFullyQualifiedName(functionDetails), conf)); } - for (String topicName : functionConfig.getInputsList()) { + for (String topicName : functionDetails.getInputsList()) { ConsumerConfiguration conf = createConsumerConfiguration(topicName); this.inputConsumers.put(topicName, client.subscribe(topicName, - FunctionConfigUtils.getFullyQualifiedName(functionConfig), conf)); + FunctionDetailsUtils.getFullyQualifiedName(functionDetails), conf)); } } @@ -126,9 +126,9 @@ protected void postReceiveMessage(InputMessage message) {} public void setupOutput(SerDe outputSerDe) throws Exception { this.outputSerDe = outputSerDe; - String outputTopic = functionConfig.getOutput(); + String outputTopic = functionDetails.getOutput(); if (outputTopic != null - && !functionConfig.getOutput().isEmpty() + && !functionDetails.getOutput().isEmpty() && outputSerDe != null) { log.info("Starting producer for output topic {}", outputTopic); initializeOutputProducer(outputTopic); diff --git a/pulsar-functions/instance/src/main/python/Function_pb2.py b/pulsar-functions/instance/src/main/python/Function_pb2.py index 95fcd7a910ab9..eeb1fabf15c9c 100644 --- a/pulsar-functions/instance/src/main/python/Function_pb2.py +++ b/pulsar-functions/instance/src/main/python/Function_pb2.py @@ -16,8 +16,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# - +# + # -*- encoding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! @@ -38,17 +38,17 @@ DESCRIPTOR = _descriptor.FileDescriptor( - name='pulsar-functions/proto/src/main/proto/Function.proto', + name='Function.proto', package='proto', syntax='proto3', - serialized_pb=_b('\n4pulsar-functions/proto/src/main/proto/Function.proto\x12\x05proto\"\x98\x06\n\x0e\x46unctionConfig\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12G\n\x11\x63ustomSerdeInputs\x18\x05 \x03(\x0b\x32,.proto.FunctionConfig.CustomSerdeInputsEntry\x12\x1c\n\x14outputSerdeClassName\x18\x06 \x01(\t\x12\x0e\n\x06output\x18\x07 \x01(\t\x12\x10\n\x08logTopic\x18\x08 \x01(\t\x12H\n\x14processingGuarantees\x18\t \x01(\x0e\x32*.proto.FunctionConfig.ProcessingGuarantees\x12\x39\n\nuserConfig\x18\n \x03(\x0b\x32%.proto.FunctionConfig.UserConfigEntry\x12@\n\x10subscriptionType\x18\x0b \x01(\x0e\x32&.proto.FunctionConfig.SubscriptionType\x12.\n\x07runtime\x18\x0c \x01(\x0e\x32\x1d.proto.FunctionConfig.Runtime\x12\x0f\n\x07\x61utoAck\x18\r \x01(\x08\x12\x0e\n\x06inputs\x18\x0e \x03(\t\x12\x13\n\x0bparallelism\x18\x0f \x01(\x05\x1a\x38\n\x16\x43ustomSerdeInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fUserConfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"O\n\x14ProcessingGuarantees\x12\x10\n\x0c\x41TLEAST_ONCE\x10\x00\x12\x0f\n\x0b\x41TMOST_ONCE\x10\x01\x12\x14\n\x10\x45\x46\x46\x45\x43TIVELY_ONCE\x10\x02\"-\n\x10SubscriptionType\x12\n\n\x06SHARED\x10\x00\x12\r\n\tEXCLUSIVE\x10\x01\"\x1f\n\x07Runtime\x12\x08\n\x04JAVA\x10\x00\x12\n\n\x06PYTHON\x10\x01\".\n\x17PackageLocationMetaData\x12\x13\n\x0bpackagePath\x18\x01 \x01(\t\"\x9f\x01\n\x10\x46unctionMetaData\x12-\n\x0e\x66unctionConfig\x18\x01 \x01(\x0b\x32\x15.proto.FunctionConfig\x12\x37\n\x0fpackageLocation\x18\x02 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x12\x0f\n\x07version\x18\x03 \x01(\x04\x12\x12\n\ncreateTime\x18\x04 \x01(\x04\"_\n\x08Snapshot\x12\x35\n\x14\x66unctionMetaDataList\x18\x01 \x03(\x0b\x32\x17.proto.FunctionMetaData\x12\x1c\n\x14lastAppliedMessageId\x18\x02 \x01(\x0c\"Q\n\x08Instance\x12\x31\n\x10\x66unctionMetaData\x18\x01 \x01(\x0b\x32\x17.proto.FunctionMetaData\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\"A\n\nAssignment\x12!\n\x08instance\x18\x01 \x01(\x0b\x32\x0f.proto.Instance\x12\x10\n\x08workerId\x18\x02 \x01(\tB-\n!org.apache.pulsar.functions.protoB\x08\x46unctionb\x06proto3') + serialized_pb=_b('\n\x0e\x46unction.proto\x12\x05proto\"\xac\x06\n\x0f\x46unctionDetails\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12H\n\x11\x63ustomSerdeInputs\x18\x05 \x03(\x0b\x32-.proto.FunctionDetails.CustomSerdeInputsEntry\x12\x1c\n\x14outputSerdeClassName\x18\x06 \x01(\t\x12\x0e\n\x06output\x18\x07 \x01(\t\x12\x10\n\x08logTopic\x18\x08 \x01(\t\x12I\n\x14processingGuarantees\x18\t \x01(\x0e\x32+.proto.FunctionDetails.ProcessingGuarantees\x12:\n\nuserConfig\x18\n \x03(\x0b\x32&.proto.FunctionDetails.UserConfigEntry\x12\x41\n\x10subscriptionType\x18\x0b \x01(\x0e\x32\'.proto.FunctionDetails.SubscriptionType\x12/\n\x07runtime\x18\x0c \x01(\x0e\x32\x1e.proto.FunctionDetails.Runtime\x12\x0f\n\x07\x61utoAck\x18\r \x01(\x08\x12\x0e\n\x06inputs\x18\x0e \x03(\t\x12\x13\n\x0bparallelism\x18\x0f \x01(\x05\x12\x0c\n\x04\x66qfn\x18\x10 \x01(\t\x1a\x38\n\x16\x43ustomSerdeInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fUserConfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"O\n\x14ProcessingGuarantees\x12\x10\n\x0c\x41TLEAST_ONCE\x10\x00\x12\x0f\n\x0b\x41TMOST_ONCE\x10\x01\x12\x14\n\x10\x45\x46\x46\x45\x43TIVELY_ONCE\x10\x02\"-\n\x10SubscriptionType\x12\n\n\x06SHARED\x10\x00\x12\r\n\tEXCLUSIVE\x10\x01\"\x1f\n\x07Runtime\x12\x08\n\x04JAVA\x10\x00\x12\n\n\x06PYTHON\x10\x01\".\n\x17PackageLocationMetaData\x12\x13\n\x0bpackagePath\x18\x01 \x01(\t\"\xa1\x01\n\x10\x46unctionMetaData\x12/\n\x0f\x66unctionDetails\x18\x01 \x01(\x0b\x32\x16.proto.FunctionDetails\x12\x37\n\x0fpackageLocation\x18\x02 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x12\x0f\n\x07version\x18\x03 \x01(\x04\x12\x12\n\ncreateTime\x18\x04 \x01(\x04\"Q\n\x08Instance\x12\x31\n\x10\x66unctionMetaData\x18\x01 \x01(\x0b\x32\x17.proto.FunctionMetaData\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\"A\n\nAssignment\x12!\n\x08instance\x18\x01 \x01(\x0b\x32\x0f.proto.Instance\x12\x10\n\x08workerId\x18\x02 \x01(\tB-\n!org.apache.pulsar.functions.protoB\x08\x46unctionb\x06proto3') ) -_FUNCTIONCONFIG_PROCESSINGGUARANTEES = _descriptor.EnumDescriptor( +_FUNCTIONDETAILS_PROCESSINGGUARANTEES = _descriptor.EnumDescriptor( name='ProcessingGuarantees', - full_name='proto.FunctionConfig.ProcessingGuarantees', + full_name='proto.FunctionDetails.ProcessingGuarantees', filename=None, file=DESCRIPTOR, values=[ @@ -67,14 +67,14 @@ ], containing_type=None, options=None, - serialized_start=697, - serialized_end=776, + serialized_start=679, + serialized_end=758, ) -_sym_db.RegisterEnumDescriptor(_FUNCTIONCONFIG_PROCESSINGGUARANTEES) +_sym_db.RegisterEnumDescriptor(_FUNCTIONDETAILS_PROCESSINGGUARANTEES) -_FUNCTIONCONFIG_SUBSCRIPTIONTYPE = _descriptor.EnumDescriptor( +_FUNCTIONDETAILS_SUBSCRIPTIONTYPE = _descriptor.EnumDescriptor( name='SubscriptionType', - full_name='proto.FunctionConfig.SubscriptionType', + full_name='proto.FunctionDetails.SubscriptionType', filename=None, file=DESCRIPTOR, values=[ @@ -89,14 +89,14 @@ ], containing_type=None, options=None, - serialized_start=778, - serialized_end=823, + serialized_start=760, + serialized_end=805, ) -_sym_db.RegisterEnumDescriptor(_FUNCTIONCONFIG_SUBSCRIPTIONTYPE) +_sym_db.RegisterEnumDescriptor(_FUNCTIONDETAILS_SUBSCRIPTIONTYPE) -_FUNCTIONCONFIG_RUNTIME = _descriptor.EnumDescriptor( +_FUNCTIONDETAILS_RUNTIME = _descriptor.EnumDescriptor( name='Runtime', - full_name='proto.FunctionConfig.Runtime', + full_name='proto.FunctionDetails.Runtime', filename=None, file=DESCRIPTOR, values=[ @@ -111,28 +111,28 @@ ], containing_type=None, options=None, - serialized_start=825, - serialized_end=856, + serialized_start=807, + serialized_end=838, ) -_sym_db.RegisterEnumDescriptor(_FUNCTIONCONFIG_RUNTIME) +_sym_db.RegisterEnumDescriptor(_FUNCTIONDETAILS_RUNTIME) -_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY = _descriptor.Descriptor( +_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY = _descriptor.Descriptor( name='CustomSerdeInputsEntry', - full_name='proto.FunctionConfig.CustomSerdeInputsEntry', + full_name='proto.FunctionDetails.CustomSerdeInputsEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='key', full_name='proto.FunctionConfig.CustomSerdeInputsEntry.key', index=0, + name='key', full_name='proto.FunctionDetails.CustomSerdeInputsEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='value', full_name='proto.FunctionConfig.CustomSerdeInputsEntry.value', index=1, + name='value', full_name='proto.FunctionDetails.CustomSerdeInputsEntry.value', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -150,26 +150,26 @@ extension_ranges=[], oneofs=[ ], - serialized_start=588, - serialized_end=644, + serialized_start=570, + serialized_end=626, ) -_FUNCTIONCONFIG_USERCONFIGENTRY = _descriptor.Descriptor( +_FUNCTIONDETAILS_USERCONFIGENTRY = _descriptor.Descriptor( name='UserConfigEntry', - full_name='proto.FunctionConfig.UserConfigEntry', + full_name='proto.FunctionDetails.UserConfigEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='key', full_name='proto.FunctionConfig.UserConfigEntry.key', index=0, + name='key', full_name='proto.FunctionDetails.UserConfigEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='value', full_name='proto.FunctionConfig.UserConfigEntry.value', index=1, + name='value', full_name='proto.FunctionDetails.UserConfigEntry.value', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -187,130 +187,137 @@ extension_ranges=[], oneofs=[ ], - serialized_start=646, - serialized_end=695, + serialized_start=628, + serialized_end=677, ) -_FUNCTIONCONFIG = _descriptor.Descriptor( - name='FunctionConfig', - full_name='proto.FunctionConfig', +_FUNCTIONDETAILS = _descriptor.Descriptor( + name='FunctionDetails', + full_name='proto.FunctionDetails', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='tenant', full_name='proto.FunctionConfig.tenant', index=0, + name='tenant', full_name='proto.FunctionDetails.tenant', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='namespace', full_name='proto.FunctionConfig.namespace', index=1, + name='namespace', full_name='proto.FunctionDetails.namespace', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='name', full_name='proto.FunctionConfig.name', index=2, + name='name', full_name='proto.FunctionDetails.name', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='className', full_name='proto.FunctionConfig.className', index=3, + name='className', full_name='proto.FunctionDetails.className', index=3, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='customSerdeInputs', full_name='proto.FunctionConfig.customSerdeInputs', index=4, + name='customSerdeInputs', full_name='proto.FunctionDetails.customSerdeInputs', index=4, number=5, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='outputSerdeClassName', full_name='proto.FunctionConfig.outputSerdeClassName', index=5, + name='outputSerdeClassName', full_name='proto.FunctionDetails.outputSerdeClassName', index=5, number=6, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='output', full_name='proto.FunctionConfig.output', index=6, + name='output', full_name='proto.FunctionDetails.output', index=6, number=7, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='logTopic', full_name='proto.FunctionConfig.logTopic', index=7, + name='logTopic', full_name='proto.FunctionDetails.logTopic', index=7, number=8, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='processingGuarantees', full_name='proto.FunctionConfig.processingGuarantees', index=8, + name='processingGuarantees', full_name='proto.FunctionDetails.processingGuarantees', index=8, number=9, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='userConfig', full_name='proto.FunctionConfig.userConfig', index=9, + name='userConfig', full_name='proto.FunctionDetails.userConfig', index=9, number=10, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='subscriptionType', full_name='proto.FunctionConfig.subscriptionType', index=10, + name='subscriptionType', full_name='proto.FunctionDetails.subscriptionType', index=10, number=11, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='runtime', full_name='proto.FunctionConfig.runtime', index=11, + name='runtime', full_name='proto.FunctionDetails.runtime', index=11, number=12, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='autoAck', full_name='proto.FunctionConfig.autoAck', index=12, + name='autoAck', full_name='proto.FunctionDetails.autoAck', index=12, number=13, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='inputs', full_name='proto.FunctionConfig.inputs', index=13, + name='inputs', full_name='proto.FunctionDetails.inputs', index=13, number=14, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='parallelism', full_name='proto.FunctionConfig.parallelism', index=14, + name='parallelism', full_name='proto.FunctionDetails.parallelism', index=14, number=15, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='fqfn', full_name='proto.FunctionDetails.fqfn', index=15, + number=16, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), ], extensions=[ ], - nested_types=[_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY, _FUNCTIONCONFIG_USERCONFIGENTRY, ], + nested_types=[_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY, _FUNCTIONDETAILS_USERCONFIGENTRY, ], enum_types=[ - _FUNCTIONCONFIG_PROCESSINGGUARANTEES, - _FUNCTIONCONFIG_SUBSCRIPTIONTYPE, - _FUNCTIONCONFIG_RUNTIME, + _FUNCTIONDETAILS_PROCESSINGGUARANTEES, + _FUNCTIONDETAILS_SUBSCRIPTIONTYPE, + _FUNCTIONDETAILS_RUNTIME, ], options=None, is_extendable=False, @@ -318,8 +325,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=64, - serialized_end=856, + serialized_start=26, + serialized_end=838, ) @@ -349,8 +356,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=858, - serialized_end=904, + serialized_start=840, + serialized_end=886, ) @@ -362,7 +369,7 @@ containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='functionConfig', full_name='proto.FunctionMetaData.functionConfig', index=0, + name='functionDetails', full_name='proto.FunctionMetaData.functionDetails', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -401,46 +408,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=907, - serialized_end=1066, -) - - -_SNAPSHOT = _descriptor.Descriptor( - name='Snapshot', - full_name='proto.Snapshot', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='functionMetaDataList', full_name='proto.Snapshot.functionMetaDataList', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='lastAppliedMessageId', full_name='proto.Snapshot.lastAppliedMessageId', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1068, - serialized_end=1163, + serialized_start=889, + serialized_end=1050, ) @@ -477,8 +446,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1165, - serialized_end=1246, + serialized_start=1052, + serialized_end=1133, ) @@ -515,87 +484,78 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1248, - serialized_end=1313, + serialized_start=1135, + serialized_end=1200, ) -_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY.containing_type = _FUNCTIONCONFIG -_FUNCTIONCONFIG_USERCONFIGENTRY.containing_type = _FUNCTIONCONFIG -_FUNCTIONCONFIG.fields_by_name['customSerdeInputs'].message_type = _FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY -_FUNCTIONCONFIG.fields_by_name['processingGuarantees'].enum_type = _FUNCTIONCONFIG_PROCESSINGGUARANTEES -_FUNCTIONCONFIG.fields_by_name['userConfig'].message_type = _FUNCTIONCONFIG_USERCONFIGENTRY -_FUNCTIONCONFIG.fields_by_name['subscriptionType'].enum_type = _FUNCTIONCONFIG_SUBSCRIPTIONTYPE -_FUNCTIONCONFIG.fields_by_name['runtime'].enum_type = _FUNCTIONCONFIG_RUNTIME -_FUNCTIONCONFIG_PROCESSINGGUARANTEES.containing_type = _FUNCTIONCONFIG -_FUNCTIONCONFIG_SUBSCRIPTIONTYPE.containing_type = _FUNCTIONCONFIG -_FUNCTIONCONFIG_RUNTIME.containing_type = _FUNCTIONCONFIG -_FUNCTIONMETADATA.fields_by_name['functionConfig'].message_type = _FUNCTIONCONFIG +_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY.containing_type = _FUNCTIONDETAILS +_FUNCTIONDETAILS_USERCONFIGENTRY.containing_type = _FUNCTIONDETAILS +_FUNCTIONDETAILS.fields_by_name['customSerdeInputs'].message_type = _FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY +_FUNCTIONDETAILS.fields_by_name['processingGuarantees'].enum_type = _FUNCTIONDETAILS_PROCESSINGGUARANTEES +_FUNCTIONDETAILS.fields_by_name['userConfig'].message_type = _FUNCTIONDETAILS_USERCONFIGENTRY +_FUNCTIONDETAILS.fields_by_name['subscriptionType'].enum_type = _FUNCTIONDETAILS_SUBSCRIPTIONTYPE +_FUNCTIONDETAILS.fields_by_name['runtime'].enum_type = _FUNCTIONDETAILS_RUNTIME +_FUNCTIONDETAILS_PROCESSINGGUARANTEES.containing_type = _FUNCTIONDETAILS +_FUNCTIONDETAILS_SUBSCRIPTIONTYPE.containing_type = _FUNCTIONDETAILS +_FUNCTIONDETAILS_RUNTIME.containing_type = _FUNCTIONDETAILS +_FUNCTIONMETADATA.fields_by_name['functionDetails'].message_type = _FUNCTIONDETAILS _FUNCTIONMETADATA.fields_by_name['packageLocation'].message_type = _PACKAGELOCATIONMETADATA -_SNAPSHOT.fields_by_name['functionMetaDataList'].message_type = _FUNCTIONMETADATA _INSTANCE.fields_by_name['functionMetaData'].message_type = _FUNCTIONMETADATA _ASSIGNMENT.fields_by_name['instance'].message_type = _INSTANCE -DESCRIPTOR.message_types_by_name['FunctionConfig'] = _FUNCTIONCONFIG +DESCRIPTOR.message_types_by_name['FunctionDetails'] = _FUNCTIONDETAILS DESCRIPTOR.message_types_by_name['PackageLocationMetaData'] = _PACKAGELOCATIONMETADATA DESCRIPTOR.message_types_by_name['FunctionMetaData'] = _FUNCTIONMETADATA -DESCRIPTOR.message_types_by_name['Snapshot'] = _SNAPSHOT DESCRIPTOR.message_types_by_name['Instance'] = _INSTANCE DESCRIPTOR.message_types_by_name['Assignment'] = _ASSIGNMENT _sym_db.RegisterFileDescriptor(DESCRIPTOR) -FunctionConfig = _reflection.GeneratedProtocolMessageType('FunctionConfig', (_message.Message,), dict( +FunctionDetails = _reflection.GeneratedProtocolMessageType('FunctionDetails', (_message.Message,), dict( CustomSerdeInputsEntry = _reflection.GeneratedProtocolMessageType('CustomSerdeInputsEntry', (_message.Message,), dict( - DESCRIPTOR = _FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY, - __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2' - # @@protoc_insertion_point(class_scope:proto.FunctionConfig.CustomSerdeInputsEntry) + DESCRIPTOR = _FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY, + __module__ = 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionDetails.CustomSerdeInputsEntry) )) , UserConfigEntry = _reflection.GeneratedProtocolMessageType('UserConfigEntry', (_message.Message,), dict( - DESCRIPTOR = _FUNCTIONCONFIG_USERCONFIGENTRY, - __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2' - # @@protoc_insertion_point(class_scope:proto.FunctionConfig.UserConfigEntry) + DESCRIPTOR = _FUNCTIONDETAILS_USERCONFIGENTRY, + __module__ = 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionDetails.UserConfigEntry) )) , - DESCRIPTOR = _FUNCTIONCONFIG, - __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2' - # @@protoc_insertion_point(class_scope:proto.FunctionConfig) + DESCRIPTOR = _FUNCTIONDETAILS, + __module__ = 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionDetails) )) -_sym_db.RegisterMessage(FunctionConfig) -_sym_db.RegisterMessage(FunctionConfig.CustomSerdeInputsEntry) -_sym_db.RegisterMessage(FunctionConfig.UserConfigEntry) +_sym_db.RegisterMessage(FunctionDetails) +_sym_db.RegisterMessage(FunctionDetails.CustomSerdeInputsEntry) +_sym_db.RegisterMessage(FunctionDetails.UserConfigEntry) PackageLocationMetaData = _reflection.GeneratedProtocolMessageType('PackageLocationMetaData', (_message.Message,), dict( DESCRIPTOR = _PACKAGELOCATIONMETADATA, - __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2' + __module__ = 'Function_pb2' # @@protoc_insertion_point(class_scope:proto.PackageLocationMetaData) )) _sym_db.RegisterMessage(PackageLocationMetaData) FunctionMetaData = _reflection.GeneratedProtocolMessageType('FunctionMetaData', (_message.Message,), dict( DESCRIPTOR = _FUNCTIONMETADATA, - __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2' + __module__ = 'Function_pb2' # @@protoc_insertion_point(class_scope:proto.FunctionMetaData) )) _sym_db.RegisterMessage(FunctionMetaData) -Snapshot = _reflection.GeneratedProtocolMessageType('Snapshot', (_message.Message,), dict( - DESCRIPTOR = _SNAPSHOT, - __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2' - # @@protoc_insertion_point(class_scope:proto.Snapshot) - )) -_sym_db.RegisterMessage(Snapshot) - Instance = _reflection.GeneratedProtocolMessageType('Instance', (_message.Message,), dict( DESCRIPTOR = _INSTANCE, - __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2' + __module__ = 'Function_pb2' # @@protoc_insertion_point(class_scope:proto.Instance) )) _sym_db.RegisterMessage(Instance) Assignment = _reflection.GeneratedProtocolMessageType('Assignment', (_message.Message,), dict( DESCRIPTOR = _ASSIGNMENT, - __module__ = 'pulsar_functions.proto.src.main.proto.Function_pb2' + __module__ = 'Function_pb2' # @@protoc_insertion_point(class_scope:proto.Assignment) )) _sym_db.RegisterMessage(Assignment) @@ -603,8 +563,8 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n!org.apache.pulsar.functions.protoB\010Function')) -_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY.has_options = True -_FUNCTIONCONFIG_CUSTOMSERDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) -_FUNCTIONCONFIG_USERCONFIGENTRY.has_options = True -_FUNCTIONCONFIG_USERCONFIGENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY.has_options = True +_FUNCTIONDETAILS_CUSTOMSERDEINPUTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_FUNCTIONDETAILS_USERCONFIGENTRY.has_options = True +_FUNCTIONDETAILS_USERCONFIGENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) # @@protoc_insertion_point(module_scope) diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index cc5852613046d..c0165b600ead8 100644 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -40,7 +40,7 @@ Log = log.Log # Equivalent of the InstanceConfig in Java -InstanceConfig = namedtuple('InstanceConfig', 'instance_id function_id function_version function_config max_buffered_tuples') +InstanceConfig = namedtuple('InstanceConfig', 'instance_id function_id function_version function_details max_buffered_tuples') # This is the message that the consumers put on the queue for the function thread to process InternalMessage = namedtuple('InternalMessage', 'message topic serde consumer') InternalQuitMessage = namedtuple('InternalQuitMessage', 'quit') @@ -95,8 +95,8 @@ def compute_latency(self): return self.latency / self.nsuccessfullyprocessed class PythonInstance(object): - def __init__(self, instance_id, function_id, function_version, function_config, max_buffered_tuples, user_code, log_topic, pulsar_client): - self.instance_config = InstanceConfig(instance_id, function_id, function_version, function_config, max_buffered_tuples) + def __init__(self, instance_id, function_id, function_version, function_details, max_buffered_tuples, user_code, log_topic, pulsar_client): + self.instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples) self.user_code = user_code self.queue = Queue.Queue(max_buffered_tuples) self.log_topic_handler = None @@ -110,9 +110,9 @@ def __init__(self, instance_id, function_id, function_version, function_config, self.function_purefunction = None self.producer = None self.exeuction_thread = None - self.atmost_once = self.instance_config.function_config.processingGuarantees == Function_pb2.FunctionConfig.ProcessingGuarantees.Value('ATMOST_ONCE') - self.atleast_once = self.instance_config.function_config.processingGuarantees == Function_pb2.FunctionConfig.ProcessingGuarantees.Value('ATLEAST_ONCE') - self.auto_ack = self.instance_config.function_config.autoAck + self.atmost_once = self.instance_config.function_details.processingGuarantees == Function_pb2.FunctionDetails.ProcessingGuarantees.Value('ATMOST_ONCE') + self.atleast_once = self.instance_config.function_details.processingGuarantees == Function_pb2.FunctionDetails.ProcessingGuarantees.Value('ATLEAST_ONCE') + self.auto_ack = self.instance_config.function_details.autoAck self.contextimpl = None self.total_stats = Stats() self.current_stats = Stats() @@ -122,10 +122,10 @@ def run(self): mode = pulsar._pulsar.ConsumerType.Exclusive if self.atmost_once: mode = pulsar._pulsar.ConsumerType.Shared - subscription_name = str(self.instance_config.function_config.tenant) + "/" + \ - str(self.instance_config.function_config.namespace) + "/" + \ - str(self.instance_config.function_config.name) - for topic, serde in self.instance_config.function_config.customSerdeInputs.items(): + subscription_name = str(self.instance_config.function_details.tenant) + "/" + \ + str(self.instance_config.function_details.namespace) + "/" + \ + str(self.instance_config.function_details.name) + for topic, serde in self.instance_config.function_details.customSerdeInputs.items(): serde_kclass = util.import_class(os.path.dirname(self.user_code), serde) self.input_serdes[topic] = serde_kclass() Log.info("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) @@ -135,7 +135,7 @@ def run(self): message_listener=partial(self.message_listener, topic, self.input_serdes[topic]) ) - for topic in self.instance_config.function_config.inputs: + for topic in self.instance_config.function_details.inputs: global DEFAULT_SERIALIZER serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER) self.input_serdes[topic] = serde_kclass() @@ -146,10 +146,10 @@ def run(self): message_listener=partial(self.message_listener, topic, self.input_serdes[topic]) ) - function_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_config.className) + function_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.className) if function_kclass is None: - Log.critical("Could not import User Function Module %s" % self.instance_config.function_config.className) - raise NameError("Could not import User Function Module %s" % self.instance_config.function_config.className) + Log.critical("Could not import User Function Module %s" % self.instance_config.function_details.className) + raise NameError("Could not import User Function Module %s" % self.instance_config.function_details.className) try: self.function_class = function_kclass() except: @@ -233,9 +233,9 @@ def process_result(self, output, msg): msg.consumer.acknowledge(msg.message) def setup_output_serde(self): - if self.instance_config.function_config.outputSerdeClassName != None and \ - len(self.instance_config.function_config.outputSerdeClassName) > 0: - serde_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_config.outputSerdeClassName) + if self.instance_config.function_details.outputSerdeClassName != None and \ + len(self.instance_config.function_details.outputSerdeClassName) > 0: + serde_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.outputSerdeClassName) self.output_serde = serde_kclass() else: global DEFAULT_SERIALIZER @@ -243,11 +243,11 @@ def setup_output_serde(self): self.output_serde = serde_kclass() def setup_producer(self): - if self.instance_config.function_config.output != None and \ - len(self.instance_config.function_config.output) > 0: - Log.info("Setting up producer for topic %s" % self.instance_config.function_config.output) + if self.instance_config.function_details.output != None and \ + len(self.instance_config.function_details.output) > 0: + Log.info("Setting up producer for topic %s" % self.instance_config.function_details.output) self.producer = self.pulsar_client.create_producer( - str(self.instance_config.function_config.output), + str(self.instance_config.function_details.output), block_if_queue_full=True, batching_enabled=True, batching_max_publish_delay_ms=1, diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py index 2638fa5669a9b..0e9d85775c650 100644 --- a/pulsar-functions/instance/src/main/python/python_instance_main.py +++ b/pulsar-functions/instance/src/main/python/python_instance_main.py @@ -85,11 +85,11 @@ def main(): Log.info("Starting Python instance with %s" % str(args)) - function_config = Function_pb2.FunctionConfig() - function_config.tenant = args.tenant - function_config.namespace = args.namespace - function_config.name = args.name - function_config.className = args.function_classname + function_details = Function_pb2.FunctionDetails() + function_details.tenant = args.tenant + function_details.namespace = args.namespace + function_details.name = args.name + function_details.className = args.function_classname if args.custom_serde_input_topics is None and args.input_topics is None: Log.critical("Atleast one input topic must be present") sys.exit(1) @@ -100,27 +100,27 @@ def main(): Log.critical("CustomSerde InputTopcis and Serde classnames should match") sys.exit(1) for i in xrange(len(input_topics)): - function_config.customSerdeInputs[input_topics[i]] = input_serde[i] + function_details.customSerdeInputs[input_topics[i]] = input_serde[i] if args.input_topics is not None: for topic in args.input_topics.split(","): - function_config.inputs.append(topic) + function_details.inputs.append(topic) if args.output_topic != None and len(args.output_topic) != 0: - function_config.output = args.output_topic + function_details.output = args.output_topic if args.output_serde_classname != None and len(args.output_serde_classname) != 0: - function_config.outputSerdeClassName = args.output_serde_classname - function_config.processingGuarantees = Function_pb2.FunctionConfig.ProcessingGuarantees.Value(args.processing_guarantees) + function_details.outputSerdeClassName = args.output_serde_classname + function_details.processingGuarantees = Function_pb2.FunctionDetails.ProcessingGuarantees.Value(args.processing_guarantees) if args.auto_ack == "true": - function_config.autoAck = True + function_details.autoAck = True else: - function_config.autoAck = False + function_details.autoAck = False if args.user_config != None and len(args.user_config) != 0: user_config = json.loads(args.user_config) for (key, value) in user_config.items(): - function_config.userConfig[str(key)] = str(value) + function_details.userConfig[str(key)] = str(value) pulsar_client = pulsar.Client(args.pulsar_serviceurl) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), - str(args.function_version), function_config, + str(args.function_version), function_details, int(args.max_buffered_tuples), str(args.py), args.log_topic, pulsar_client) pyinstance.run() diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableProcessTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableProcessTest.java index 8e099a6bce4b0..1f76bff493a83 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableProcessTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableProcessTest.java @@ -78,11 +78,11 @@ import org.apache.pulsar.functions.api.utils.DefaultSerDe; import org.apache.pulsar.functions.instance.processors.AtLeastOnceProcessor; import org.apache.pulsar.functions.instance.processors.MessageProcessor; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; -import org.apache.pulsar.functions.proto.Function.FunctionConfig.ProcessingGuarantees; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.Function.FunctionDetails.ProcessingGuarantees; import org.apache.pulsar.functions.utils.Reflections; import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; import org.apache.pulsar.functions.utils.Utils; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; @@ -208,16 +208,16 @@ public synchronized void addSendFuture(CompletableFuture future) { private static final String TEST_STORAGE_SERVICE_URL = "127.0.0.1:4181"; private static final LinkedBlockingQueue voidFunctionQueue = new LinkedBlockingQueue<>(); - private FunctionConfig fnConfig; + private FunctionDetails functionDetails; private InstanceConfig config; private FunctionCacheManager fnCache; private PulsarClient mockClient; private FunctionStats mockFunctionStats; private final Map, ProducerInstance> mockProducers - = Collections.synchronizedMap(new HashMap<>()); + = Collections.synchronizedMap(new HashMap<>()); private final Map, ConsumerInstance> mockConsumers - = Collections.synchronizedMap(new HashMap<>()); + = Collections.synchronizedMap(new HashMap<>()); private StorageClient mockStorageClient; private Table mockTable; @@ -226,23 +226,23 @@ public void setup() throws Exception { mockProducers.clear(); mockConsumers.clear(); - fnConfig = FunctionConfig.newBuilder() - .setAutoAck(true) - .setClassName(TestFunction.class.getName()) - .addInputs("test-src-topic") - .setName("test-function") - .setOutput("test-output-topic") - .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE) - .setTenant("test-tenant") - .setNamespace("test-namespace") - .build(); + functionDetails = FunctionDetails.newBuilder() + .setAutoAck(true) + .setClassName(TestFunction.class.getName()) + .addInputs("test-src-topic") + .setName("test-function") + .setOutput("test-output-topic") + .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE) + .setTenant("test-tenant") + .setNamespace("test-namespace") + .build(); config = new InstanceConfig(); config.setFunctionId("test-function-id"); config.setFunctionVersion("v1"); config.setInstanceId("test-instance-id"); config.setMaxBufferedTuples(1000); - config.setFunctionConfig(fnConfig); + config.setFunctionDetails(functionDetails); mockClient = mock(PulsarClientImpl.class); @@ -253,46 +253,46 @@ public void setup() throws Exception { ClassLoader clsLoader = JavaInstanceRunnableTest.class.getClassLoader(); when(fnCache.getClassLoader(anyString())) - .thenReturn(clsLoader); + .thenReturn(clsLoader); // mock producer & consumer when(mockClient.createProducer(anyString(), any(ProducerConfiguration.class))) - .thenAnswer(invocationOnMock -> { - String topic = invocationOnMock.getArgumentAt(0, String.class); - ProducerConfiguration conf = invocationOnMock.getArgumentAt(1, ProducerConfiguration.class); - String producerName = conf.getProducerName(); - - Pair pair = Pair.of(topic, producerName); - ProducerInstance producerInstance = mockProducers.get(pair); - if (null == producerInstance) { - Producer producer = mock(Producer.class); - LinkedBlockingQueue msgQueue = new LinkedBlockingQueue<>(); - final ProducerInstance instance = new ProducerInstance(producer, msgQueue); - producerInstance = instance; - when(producer.getProducerName()) - .thenReturn(producerName); - when(producer.getTopic()) - .thenReturn(topic); - when(producer.sendAsync(any(Message.class))) - .thenAnswer(invocationOnMock1 -> { - Message msg = invocationOnMock1.getArgumentAt(0, Message.class); - log.info("producer send message {}", msg); - - CompletableFuture future = new CompletableFuture<>(); - instance.addSendFuture(future); - msgQueue.put(msg); - return future; - }); - when(producer.closeAsync()).thenReturn(FutureUtils.Void()); - - mockProducers.put(pair, producerInstance); - } - return producerInstance.getProducer(); - }); + .thenAnswer(invocationOnMock -> { + String topic = invocationOnMock.getArgumentAt(0, String.class); + ProducerConfiguration conf = invocationOnMock.getArgumentAt(1, ProducerConfiguration.class); + String producerName = conf.getProducerName(); + + Pair pair = Pair.of(topic, producerName); + ProducerInstance producerInstance = mockProducers.get(pair); + if (null == producerInstance) { + Producer producer = mock(Producer.class); + LinkedBlockingQueue msgQueue = new LinkedBlockingQueue<>(); + final ProducerInstance instance = new ProducerInstance(producer, msgQueue); + producerInstance = instance; + when(producer.getProducerName()) + .thenReturn(producerName); + when(producer.getTopic()) + .thenReturn(topic); + when(producer.sendAsync(any(Message.class))) + .thenAnswer(invocationOnMock1 -> { + Message msg = invocationOnMock1.getArgumentAt(0, Message.class); + log.info("producer send message {}", msg); + + CompletableFuture future = new CompletableFuture<>(); + instance.addSendFuture(future); + msgQueue.put(msg); + return future; + }); + when(producer.closeAsync()).thenReturn(FutureUtils.Void()); + + mockProducers.put(pair, producerInstance); + } + return producerInstance.getProducer(); + }); when(mockClient.subscribe( - anyString(), - anyString(), - any(ConsumerConfiguration.class) + anyString(), + anyString(), + any(ConsumerConfiguration.class) )).thenAnswer(invocationOnMock -> { String topic = invocationOnMock.getArgumentAt(0, String.class); String subscription = invocationOnMock.getArgumentAt(1, String.class); @@ -308,26 +308,26 @@ public void setup() throws Exception { when(consumer.getTopic()).thenReturn(topic); when(consumer.getSubscription()).thenReturn(subscription); when(consumer.acknowledgeAsync(any(Message.class))) - .thenAnswer(invocationOnMock1 -> { - Message msg = invocationOnMock1.getArgumentAt(0, Message.class); - log.info("Ack message {} : message id = {}", msg, msg.getMessageId()); + .thenAnswer(invocationOnMock1 -> { + Message msg = invocationOnMock1.getArgumentAt(0, Message.class); + log.info("Ack message {} : message id = {}", msg, msg.getMessageId()); - instance.removeMessage(msg.getMessageId()); - return FutureUtils.Void(); - }); + instance.removeMessage(msg.getMessageId()); + return FutureUtils.Void(); + }); when(consumer.acknowledgeCumulativeAsync(any(Message.class))) - .thenAnswer(invocationOnMock1 -> { - Message msg = invocationOnMock1.getArgumentAt(0, Message.class); - log.info("Ack message cumulatively message id = {}", msg, msg.getMessageId()); + .thenAnswer(invocationOnMock1 -> { + Message msg = invocationOnMock1.getArgumentAt(0, Message.class); + log.info("Ack message cumulatively message id = {}", msg, msg.getMessageId()); - instance.removeMessagesBefore(msg.getMessageId()); - return FutureUtils.Void(); - }); + instance.removeMessagesBefore(msg.getMessageId()); + return FutureUtils.Void(); + }); when(consumer.closeAsync()) - .thenAnswer(invocationOnMock1 -> { - mockConsumers.remove(pair, instance); - return FutureUtils.Void(); - }); + .thenAnswer(invocationOnMock1 -> { + mockConsumers.remove(pair, instance); + return FutureUtils.Void(); + }); doAnswer(invocationOnMock1 -> { mockConsumers.remove(pair, instance); return null; @@ -353,11 +353,11 @@ public void setup() throws Exception { PowerMockito.mockStatic(StorageClientBuilder.class); PowerMockito.when( - StorageClientBuilder.newBuilder() + StorageClientBuilder.newBuilder() ).thenReturn(mockBuilder); when(adminClient.getStream(anyString(), anyString())).thenReturn(FutureUtils.value( - StreamProperties.newBuilder().build())); + StreamProperties.newBuilder().build())); mockTable = mock(Table.class); when(mockStorageClient.openTable(anyString())).thenReturn(FutureUtils.value(mockTable)); @@ -367,38 +367,38 @@ public void setup() throws Exception { mockFunctionStats = spy(new FunctionStats()); PowerMockito.whenNew(FunctionStats.class) - .withNoArguments() - .thenReturn(mockFunctionStats); + .withNoArguments() + .thenReturn(mockFunctionStats); // Mock message builder PowerMockito.mockStatic(MessageBuilder.class); PowerMockito.when(MessageBuilder.create()) - .thenAnswer(invocationOnMock -> { - - Message msg = mock(Message.class); - MessageBuilder builder = mock(MessageBuilder.class); - when(builder.setContent(any(byte[].class))) - .thenAnswer(invocationOnMock1 -> { - byte[] content = invocationOnMock1.getArgumentAt(0, byte[].class); - when(msg.getData()).thenReturn(content); - return builder; - }); - when(builder.setSequenceId(anyLong())) - .thenAnswer(invocationOnMock1 -> { - long seqId = invocationOnMock1.getArgumentAt(0, long.class); - when(msg.getSequenceId()).thenReturn(seqId); - return builder; - }); - when(builder.setProperty(anyString(), anyString())) - .thenAnswer(invocationOnMock1 -> { - String key = invocationOnMock1.getArgumentAt(0, String.class); - String value = invocationOnMock1.getArgumentAt(1, String.class); - when(msg.getProperty(eq(key))).thenReturn(value); - return builder; + .thenAnswer(invocationOnMock -> { + + Message msg = mock(Message.class); + MessageBuilder builder = mock(MessageBuilder.class); + when(builder.setContent(any(byte[].class))) + .thenAnswer(invocationOnMock1 -> { + byte[] content = invocationOnMock1.getArgumentAt(0, byte[].class); + when(msg.getData()).thenReturn(content); + return builder; }); - when(builder.build()).thenReturn(msg); - return builder; - }); + when(builder.setSequenceId(anyLong())) + .thenAnswer(invocationOnMock1 -> { + long seqId = invocationOnMock1.getArgumentAt(0, long.class); + when(msg.getSequenceId()).thenReturn(seqId); + return builder; + }); + when(builder.setProperty(anyString(), anyString())) + .thenAnswer(invocationOnMock1 -> { + String key = invocationOnMock1.getArgumentAt(0, String.class); + String value = invocationOnMock1.getArgumentAt(1, String.class); + when(msg.getProperty(eq(key))).thenReturn(value); + return builder; + }); + when(builder.build()).thenReturn(msg); + return builder; + }); } /** @@ -407,11 +407,11 @@ public void setup() throws Exception { @Test public void testSetupJavaInstance() throws Exception { JavaInstanceRunnable runnable = new JavaInstanceRunnable( - config, - fnCache, - "test-jar-file", - mockClient, - TEST_STORAGE_SERVICE_URL); + config, + fnCache, + "test-jar-file", + mockClient, + TEST_STORAGE_SERVICE_URL); runnable.setupJavaInstance(); @@ -419,17 +419,17 @@ public void testSetupJavaInstance() throws Exception { // 1. verify jar is loaded verify(fnCache, times(1)) - .registerFunctionInstance( - eq(config.getFunctionId()), - eq(config.getInstanceId()), - eq(Arrays.asList("test-jar-file")), - eq(Collections.emptyList()) - ); + .registerFunctionInstance( + eq(config.getFunctionId()), + eq(config.getInstanceId()), + eq(Arrays.asList("test-jar-file")), + eq(Collections.emptyList()) + ); verify(fnCache, times(1)) - .getClassLoader(eq(config.getFunctionId())); + .getClassLoader(eq(config.getFunctionId())); // 2. verify serde is setup - for (String inputTopic : fnConfig.getInputsList()) { + for (String inputTopic : functionDetails.getInputsList()) { assertTrue(runnable.getInputSerDe().containsKey(inputTopic)); assertTrue(runnable.getInputSerDe().get(inputTopic) instanceof DefaultSerDe); DefaultSerDe serDe = (DefaultSerDe) runnable.getInputSerDe().get(inputTopic); @@ -440,8 +440,8 @@ public void testSetupJavaInstance() throws Exception { MessageProcessor processor = runnable.getProcessor(); assertTrue(processor instanceof AtLeastOnceProcessor); assertSame(mockProducers.get(Pair.of( - fnConfig.getOutput(), - null + functionDetails.getOutput(), + null )).getProducer(), ((AtLeastOnceProcessor) processor).getProducer()); assertEquals(mockConsumers.size(), processor.getInputConsumers().size()); @@ -449,8 +449,8 @@ public void testSetupJavaInstance() throws Exception { String topic = consumerEntry.getKey(); Consumer mockConsumer = mockConsumers.get(Pair.of( - topic, - FunctionConfigUtils.getFullyQualifiedName(fnConfig))).getConsumer(); + topic, + FunctionDetailsUtils.getFullyQualifiedName(functionDetails))).getConsumer(); assertSame(mockConsumer, consumerEntry.getValue()); } @@ -475,32 +475,32 @@ public void testSetupJavaInstance() throws Exception { // function is unregistered verify(fnCache, times(1)).unregisterFunctionInstance( - eq(config.getFunctionId()), eq(config.getInstanceId())); + eq(config.getFunctionId()), eq(config.getInstanceId())); } @Test public void testAtMostOnceProcessing() throws Exception { - FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig) - .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE) - .build(); - config.setFunctionConfig(newFnConfig); + FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails) + .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE) + .build(); + config.setFunctionDetails(newFunctionDetails); @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor(); try (JavaInstanceRunnable runnable = new JavaInstanceRunnable( - config, - fnCache, - "test-jar-file", - mockClient, - null)) { + config, + fnCache, + "test-jar-file", + mockClient, + null)) { executorService.submit(runnable); Pair consumerId = Pair.of( - newFnConfig.getInputs(0), - FunctionConfigUtils.getFullyQualifiedName(newFnConfig)); + newFunctionDetails.getInputs(0), + FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails)); ConsumerInstance consumerInstance = mockConsumers.get(consumerId); while (null == consumerInstance) { TimeUnit.MILLISECONDS.sleep(20); @@ -514,10 +514,10 @@ public void testAtMostOnceProcessing() throws Exception { Message msg = mock(Message.class); when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8)); when(msg.getMessageId()) - .thenReturn(new MessageIdImpl(1L, i, 0)); + .thenReturn(new MessageIdImpl(1L, i, 0)); consumerInstance.addMessage(msg); consumerInstance.getConf().getMessageListener() - .received(consumerInstance.getConsumer(), msg); + .received(consumerInstance.getConsumer(), msg); } // wait until all the messages are published @@ -531,7 +531,7 @@ public void testAtMostOnceProcessing() throws Exception { // verify acknowledge before send completes verify(consumerInstance.getConsumer(), times(10)) - .acknowledgeAsync(any(Message.class)); + .acknowledgeAsync(any(Message.class)); assertEquals(0, consumerInstance.getNumMessages()); // complete all the publishes @@ -543,33 +543,33 @@ public void testAtMostOnceProcessing() throws Exception { // acknowledges count should remain same verify(consumerInstance.getConsumer(), times(10)) - .acknowledgeAsync(any(Message.class)); + .acknowledgeAsync(any(Message.class)); } } @Test public void testAtMostOnceProcessingFailures() throws Exception { - FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig) - .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE) - .setClassName(TestFailureFunction.class.getName()) - .build(); - config.setFunctionConfig(newFnConfig); + FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails) + .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE) + .setClassName(TestFailureFunction.class.getName()) + .build(); + config.setFunctionDetails(newFunctionDetails); @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor(); try (JavaInstanceRunnable runnable = new JavaInstanceRunnable( - config, - fnCache, - "test-jar-file", - mockClient, - null)) { + config, + fnCache, + "test-jar-file", + mockClient, + null)) { executorService.submit(runnable); Pair consumerId = Pair.of( - newFnConfig.getInputs(0), - FunctionConfigUtils.getFullyQualifiedName(newFnConfig)); + newFunctionDetails.getInputs(0), + FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails)); ConsumerInstance consumerInstance = mockConsumers.get(consumerId); while (null == consumerInstance) { TimeUnit.MILLISECONDS.sleep(20); @@ -583,10 +583,10 @@ public void testAtMostOnceProcessingFailures() throws Exception { Message msg = mock(Message.class); when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8)); when(msg.getMessageId()) - .thenReturn(new MessageIdImpl(1L, i, 0)); + .thenReturn(new MessageIdImpl(1L, i, 0)); consumerInstance.addMessage(msg); consumerInstance.getConf().getMessageListener() - .received(consumerInstance.getConsumer(), msg); + .received(consumerInstance.getConsumer(), msg); } // wait until all the messages are published @@ -604,7 +604,7 @@ public void testAtMostOnceProcessingFailures() throws Exception { // verify acknowledge before send completes verify(consumerInstance.getConsumer(), times(10)) - .acknowledgeAsync(any(Message.class)); + .acknowledgeAsync(any(Message.class)); assertEquals(0, consumerInstance.getNumMessages()); // complete all the publishes @@ -616,33 +616,33 @@ public void testAtMostOnceProcessingFailures() throws Exception { // acknowledges count should remain same verify(consumerInstance.getConsumer(), times(10)) - .acknowledgeAsync(any(Message.class)); + .acknowledgeAsync(any(Message.class)); assertEquals(0, consumerInstance.getNumMessages()); } } @Test public void testAtLeastOnceProcessing() throws Exception { - FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig) - .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE) - .build(); - config.setFunctionConfig(newFnConfig); + FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails) + .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE) + .build(); + config.setFunctionDetails(newFunctionDetails); @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor(); try (JavaInstanceRunnable runnable = new JavaInstanceRunnable( - config, - fnCache, - "test-jar-file", - mockClient, - null)) { + config, + fnCache, + "test-jar-file", + mockClient, + null)) { executorService.submit(runnable); Pair consumerId = Pair.of( - newFnConfig.getInputs(0), - FunctionConfigUtils.getFullyQualifiedName(newFnConfig)); + newFunctionDetails.getInputs(0), + FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails)); ConsumerInstance consumerInstance = mockConsumers.get(consumerId); while (null == consumerInstance) { TimeUnit.MILLISECONDS.sleep(20); @@ -656,10 +656,10 @@ public void testAtLeastOnceProcessing() throws Exception { Message msg = mock(Message.class); when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8)); when(msg.getMessageId()) - .thenReturn(new MessageIdImpl(1L, i, 0)); + .thenReturn(new MessageIdImpl(1L, i, 0)); consumerInstance.addMessage(msg); consumerInstance.getConf().getMessageListener() - .received(consumerInstance.getConsumer(), msg); + .received(consumerInstance.getConsumer(), msg); } // wait until all the messages are published @@ -673,7 +673,7 @@ public void testAtLeastOnceProcessing() throws Exception { // verify acknowledge before send completes verify(consumerInstance.getConsumer(), times(0)) - .acknowledgeAsync(any(Message.class)); + .acknowledgeAsync(any(Message.class)); assertEquals(10, consumerInstance.getNumMessages()); // complete all the publishes @@ -685,34 +685,34 @@ public void testAtLeastOnceProcessing() throws Exception { // acknowledges count should remain same verify(consumerInstance.getConsumer(), times(10)) - .acknowledgeAsync(any(Message.class)); + .acknowledgeAsync(any(Message.class)); assertEquals(0, consumerInstance.getNumMessages()); } } @Test public void testAtLeastOnceProcessingFailures() throws Exception { - FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig) - .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE) - .setClassName(TestFailureFunction.class.getName()) - .build(); - config.setFunctionConfig(newFnConfig); + FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails) + .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE) + .setClassName(TestFailureFunction.class.getName()) + .build(); + config.setFunctionDetails(newFunctionDetails); @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor(); try (JavaInstanceRunnable runnable = new JavaInstanceRunnable( - config, - fnCache, - "test-jar-file", - mockClient, - null)) { + config, + fnCache, + "test-jar-file", + mockClient, + null)) { executorService.submit(runnable); Pair consumerId = Pair.of( - newFnConfig.getInputs(0), - FunctionConfigUtils.getFullyQualifiedName(newFnConfig)); + newFunctionDetails.getInputs(0), + FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails)); ConsumerInstance consumerInstance = mockConsumers.get(consumerId); while (null == consumerInstance) { TimeUnit.MILLISECONDS.sleep(20); @@ -726,10 +726,10 @@ public void testAtLeastOnceProcessingFailures() throws Exception { Message msg = mock(Message.class); when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8)); when(msg.getMessageId()) - .thenReturn(new MessageIdImpl(1L, i, 0)); + .thenReturn(new MessageIdImpl(1L, i, 0)); consumerInstance.addMessage(msg); consumerInstance.getConf().getMessageListener() - .received(consumerInstance.getConsumer(), msg); + .received(consumerInstance.getConsumer(), msg); } // wait until all the messages are published @@ -747,7 +747,7 @@ public void testAtLeastOnceProcessingFailures() throws Exception { // verify acknowledge before send completes verify(consumerInstance.getConsumer(), times(0)) - .acknowledgeAsync(any(Message.class)); + .acknowledgeAsync(any(Message.class)); assertEquals(10, consumerInstance.getNumMessages()); // complete all the publishes @@ -759,38 +759,38 @@ public void testAtLeastOnceProcessingFailures() throws Exception { // only 5 succeed messages are acknowledged verify(consumerInstance.getConsumer(), times(5)) - .acknowledgeAsync(any(Message.class)); + .acknowledgeAsync(any(Message.class)); assertEquals(5, consumerInstance.getNumMessages()); for (int i = 0; i < 10; i++) { assertEquals( - i % 2 == 0, - consumerInstance.containMessage(new MessageIdImpl(1L, i, 0))); + i % 2 == 0, + consumerInstance.containMessage(new MessageIdImpl(1L, i, 0))); } } } @Test public void testEffectivelyOnceProcessing() throws Exception { - FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig) - .setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE) - .build(); - config.setFunctionConfig(newFnConfig); + FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails) + .setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE) + .build(); + config.setFunctionDetails(newFunctionDetails); @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor(); try (JavaInstanceRunnable runnable = new JavaInstanceRunnable( - config, - fnCache, - "test-jar-file", - mockClient, - null)) { + config, + fnCache, + "test-jar-file", + mockClient, + null)) { executorService.submit(runnable); Pair consumerId = Pair.of( - newFnConfig.getInputs(0), - FunctionConfigUtils.getFullyQualifiedName(newFnConfig)); + newFunctionDetails.getInputs(0), + FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails)); ConsumerInstance consumerInstance = mockConsumers.get(consumerId); while (null == consumerInstance) { TimeUnit.MILLISECONDS.sleep(20); @@ -802,10 +802,10 @@ public void testEffectivelyOnceProcessing() throws Exception { Message msg = mock(Message.class); when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8)); when(msg.getMessageId()) - .thenReturn(new MessageIdImpl(1L, i, 0)); + .thenReturn(new MessageIdImpl(1L, i, 0)); consumerInstance.addMessage(msg); consumerInstance.getConf().getMessageListener() - .received(consumerInstance.getConsumer(), msg); + .received(consumerInstance.getConsumer(), msg); } ProducerInstance producerInstance; @@ -821,14 +821,14 @@ public void testEffectivelyOnceProcessing() throws Exception { assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8)); // sequence id is not set for AT_MOST_ONCE processing assertEquals( - Utils.getSequenceId( - new MessageIdImpl(1L, i, 0)), - msg.getSequenceId()); + Utils.getSequenceId( + new MessageIdImpl(1L, i, 0)), + msg.getSequenceId()); } // verify acknowledge before send completes verify(consumerInstance.getConsumer(), times(0)) - .acknowledgeCumulativeAsync(any(Message.class)); + .acknowledgeCumulativeAsync(any(Message.class)); assertEquals(10, consumerInstance.getNumMessages()); // complete all the publishes @@ -840,34 +840,34 @@ public void testEffectivelyOnceProcessing() throws Exception { // acknowledges count should remain same verify(consumerInstance.getConsumer(), times(10)) - .acknowledgeCumulativeAsync(any(Message.class)); + .acknowledgeCumulativeAsync(any(Message.class)); assertEquals(0, consumerInstance.getNumMessages()); } } @Test public void testEffectivelyOnceProcessingFailures() throws Exception { - FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig) - .setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE) - .setClassName(TestFailureFunction.class.getName()) - .build(); - config.setFunctionConfig(newFnConfig); + FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails) + .setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE) + .setClassName(TestFailureFunction.class.getName()) + .build(); + config.setFunctionDetails(newFunctionDetails); @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor(); try (JavaInstanceRunnable runnable = new JavaInstanceRunnable( - config, - fnCache, - "test-jar-file", - mockClient, - null)) { + config, + fnCache, + "test-jar-file", + mockClient, + null)) { executorService.submit(runnable); Pair consumerId = Pair.of( - newFnConfig.getInputs(0), - FunctionConfigUtils.getFullyQualifiedName(newFnConfig)); + newFunctionDetails.getInputs(0), + FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails)); ConsumerInstance consumerInstance = mockConsumers.get(consumerId); while (null == consumerInstance) { TimeUnit.MILLISECONDS.sleep(20); @@ -880,13 +880,13 @@ public void testEffectivelyOnceProcessingFailures() throws Exception { Message msg = mock(Message.class); when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8)); when(msg.getMessageId()) - .thenReturn(new MessageIdImpl(1L, i, 0)); + .thenReturn(new MessageIdImpl(1L, i, 0)); msgs[i-1] = msg; consumerInstance.addMessage(msg); consumerInstance.getConf().getMessageListener() - .received(consumerInstance.getConsumer(), msg); + .received(consumerInstance.getConsumer(), msg); } ProducerInstance producerInstance; @@ -899,15 +899,15 @@ public void testEffectivelyOnceProcessingFailures() throws Exception { Message msg = producerInstance.msgQueue.take(); assertEquals("message-1!", new String(msg.getData(), UTF_8)); assertEquals( - Utils.getSequenceId( - new MessageIdImpl(1L, 1, 0)), - msg.getSequenceId()); + Utils.getSequenceId( + new MessageIdImpl(1L, 1, 0)), + msg.getSequenceId()); assertNull(producerInstance.msgQueue.poll()); // the first result message is sent but the send future is not completed yet // so no acknowledge would happen verify(consumerInstance.getConsumer(), times(0)) - .acknowledgeCumulativeAsync(any(Message.class)); + .acknowledgeCumulativeAsync(any(Message.class)); // since the second message failed to process, for correctness, the instance // will close the existing consumer and resubscribe @@ -920,21 +920,21 @@ public void testEffectivelyOnceProcessingFailures() throws Exception { Message secondMsg = mock(Message.class); when(secondMsg.getData()).thenReturn("message-2".getBytes(UTF_8)); when(secondMsg.getMessageId()) - .thenReturn(new MessageIdImpl(1L, 2, 0)); + .thenReturn(new MessageIdImpl(1L, 2, 0)); secondInstance.addMessage(secondMsg); secondInstance.getConf().getMessageListener() - .received(secondInstance.getConsumer(), secondMsg); + .received(secondInstance.getConsumer(), secondMsg); Message secondReceivedMsg = producerInstance.msgQueue.take(); assertEquals("message-2!", new String(secondReceivedMsg.getData(), UTF_8)); assertEquals( - Utils.getSequenceId( - new MessageIdImpl(1L, 2, 0)), - secondReceivedMsg.getSequenceId()); + Utils.getSequenceId( + new MessageIdImpl(1L, 2, 0)), + secondReceivedMsg.getSequenceId()); // the first result message is sent verify(secondInstance.getConsumer(), times(0)) - .acknowledgeCumulativeAsync(any(Message.class)); + .acknowledgeCumulativeAsync(any(Message.class)); // complete all the publishes synchronized (producerInstance) { @@ -946,41 +946,41 @@ public void testEffectivelyOnceProcessingFailures() throws Exception { // all 2 messages are sent verify(consumerInstance.getConsumer(), times(1)) - .acknowledgeCumulativeAsync(same(msgs[0])); + .acknowledgeCumulativeAsync(same(msgs[0])); verify(consumerInstance.getConsumer(), times(0)) - .acknowledgeCumulativeAsync(same(msgs[1])); + .acknowledgeCumulativeAsync(same(msgs[1])); verify(consumerInstance.getConsumer(), times(0)) - .acknowledgeCumulativeAsync(same(secondMsg)); + .acknowledgeCumulativeAsync(same(secondMsg)); verify(secondInstance.getConsumer(), times(0)) - .acknowledgeCumulativeAsync(same(msgs[0])); + .acknowledgeCumulativeAsync(same(msgs[0])); verify(secondInstance.getConsumer(), times(0)) - .acknowledgeCumulativeAsync(same(msgs[1])); + .acknowledgeCumulativeAsync(same(msgs[1])); } } @Test public void testVoidFunction() throws Exception { - FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig) - .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE) - .setClassName(TestVoidFunction.class.getName()) - .build(); - config.setFunctionConfig(newFnConfig); + FunctionDetails newFunctionDetails = FunctionDetails.newBuilder(functionDetails) + .setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE) + .setClassName(TestVoidFunction.class.getName()) + .build(); + config.setFunctionDetails(newFunctionDetails); @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor(); try (JavaInstanceRunnable runnable = new JavaInstanceRunnable( - config, - fnCache, - "test-jar-file", - mockClient, - null)) { + config, + fnCache, + "test-jar-file", + mockClient, + null)) { executorService.submit(runnable); Pair consumerId = Pair.of( - newFnConfig.getInputs(0), - FunctionConfigUtils.getFullyQualifiedName(newFnConfig)); + newFunctionDetails.getInputs(0), + FunctionDetailsUtils.getFullyQualifiedName(newFunctionDetails)); ConsumerInstance consumerInstance = mockConsumers.get(consumerId); while (null == consumerInstance) { TimeUnit.MILLISECONDS.sleep(20); @@ -992,10 +992,10 @@ public void testVoidFunction() throws Exception { Message msg = mock(Message.class); when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8)); when(msg.getMessageId()) - .thenReturn(new MessageIdImpl(1L, i, 0)); + .thenReturn(new MessageIdImpl(1L, i, 0)); consumerInstance.addMessage(msg); consumerInstance.getConf().getMessageListener() - .received(consumerInstance.getConsumer(), msg); + .received(consumerInstance.getConsumer(), msg); } // wait until all the messages are published @@ -1009,4 +1009,4 @@ public void testVoidFunction() throws Exception { assertTrue(mockProducers.isEmpty()); } } -} +} \ No newline at end of file diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java index 1e9781c35c183..a1672ba44b82c 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java @@ -25,7 +25,7 @@ import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.SerDe; import org.apache.pulsar.functions.api.utils.DefaultSerDe; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.testng.annotations.Test; import java.lang.reflect.InvocationTargetException; @@ -48,17 +48,17 @@ public byte[] serialize(Integer input) { } private static InstanceConfig createInstanceConfig(boolean addCustom, String outputSerde) { - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); + FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); if (!addCustom) { - functionConfigBuilder.addInputs("TEST"); + functionDetailsBuilder.addInputs("TEST"); } else { - functionConfigBuilder.putCustomSerdeInputs("TEST", IntegerSerDe.class.getName()); + functionDetailsBuilder.putCustomSerdeInputs("TEST", IntegerSerDe.class.getName()); } if (outputSerde != null) { - functionConfigBuilder.setOutputSerdeClassName(outputSerde); + functionDetailsBuilder.setOutputSerdeClassName(outputSerde); } InstanceConfig instanceConfig = new InstanceConfig(); - instanceConfig.setFunctionConfig(functionConfigBuilder.build()); + instanceConfig.setFunctionDetails(functionDetailsBuilder.build()); instanceConfig.setMaxBufferedTuples(1024); return instanceConfig; } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java index 53c93a485530d..c6f75775353c9 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java @@ -24,7 +24,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.utils.DefaultSerDe; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.testng.annotations.Test; import java.util.HashMap; @@ -32,11 +32,11 @@ public class JavaInstanceTest { private static InstanceConfig createInstanceConfig() { - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - functionConfigBuilder.addInputs("TEST"); - functionConfigBuilder.setOutputSerdeClassName(DefaultSerDe.class.getName()); + FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); + functionDetailsBuilder.addInputs("TEST"); + functionDetailsBuilder.setOutputSerdeClassName(DefaultSerDe.class.getName()); InstanceConfig instanceConfig = new InstanceConfig(); - instanceConfig.setFunctionConfig(functionConfigBuilder.build()); + instanceConfig.setFunctionDetails(functionDetailsBuilder.build()); return instanceConfig; } diff --git a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/MetricsSink.java b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/MetricsSink.java index 488edf528366a..d22bf6a316883 100644 --- a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/MetricsSink.java +++ b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/MetricsSink.java @@ -42,11 +42,10 @@ public interface MetricsSink extends AutoCloseable { /** * Process a metrics record in the sink - * - * @param record the record to put - * @param functionConfig functionConfig that generated this record + * @param record the record to put + * @param functionDetails functionDetails that generated this record */ - void processRecord(MetricsData record, Function.FunctionConfig functionConfig); + void processRecord(MetricsData record, Function.FunctionDetails functionDetails); /** * Flush any buffered metrics diff --git a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java index 06e0cb2b31ef3..64bc97d1169a9 100644 --- a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java +++ b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java @@ -29,7 +29,7 @@ import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.metrics.MetricsSink; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; import org.apache.pulsar.functions.utils.Utils; /** @@ -82,9 +82,9 @@ private PrintStream openNewFile(String filename) { } @Override - public void processRecord(InstanceCommunication.MetricsData record, Function.FunctionConfig functionConfig) { + public void processRecord(InstanceCommunication.MetricsData record, Function.FunctionDetails FunctionDetails) { if (isFileStart) { - String filenamePrefix = filenameKey + "." + FunctionConfigUtils.getFullyQualifiedName(functionConfig); + String filenamePrefix = filenameKey + "." + FunctionDetailsUtils.getFullyQualifiedName(FunctionDetails); writer = openNewFile(String.format("%s.%d", filenamePrefix, currentFileIndex)); writer.print("["); diff --git a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/PrometheusSink.java b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/PrometheusSink.java index 65a24ca4ab590..26aa9400831d1 100644 --- a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/PrometheusSink.java +++ b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/PrometheusSink.java @@ -29,7 +29,7 @@ import lombok.Getter; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication.MetricsData; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; /** * A web sink that exposes and endpoint that Prometheus can scrape @@ -67,9 +67,9 @@ byte[] generateResponse() throws IOException { final StringBuilder sb = new StringBuilder(); metrics.forEach((String source, Map sourceMetrics) -> { - String tenant = FunctionConfigUtils.extractTenantFromFQN(source); - String namespace = FunctionConfigUtils.extractNamespaceFromFQN(source); - String name = FunctionConfigUtils.extractFunctionNameFromFQN(source); + String tenant = FunctionDetailsUtils.extractTenantFromFQN(source); + String namespace = FunctionDetailsUtils.extractNamespaceFromFQN(source); + String name = FunctionDetailsUtils.extractFunctionNameFromFQN(source); sourceMetrics.forEach((String metricName, Double value) -> { @@ -91,8 +91,8 @@ byte[] generateResponse() throws IOException { } @Override - public void processRecord(MetricsData record, Function.FunctionConfig functionConfig) { - final String source = FunctionConfigUtils.getFullyQualifiedName(functionConfig); + public void processRecord(MetricsData record, Function.FunctionDetails functionDetails) { + final String source = FunctionDetailsUtils.getFullyQualifiedName(functionDetails); Map sourceCache = metricsCache.getIfPresent(source); if (sourceCache == null) { diff --git a/pulsar-functions/metrics/src/test/java/org/apache/pulsar/functions/metrics/sink/PrometheusSinkTests.java b/pulsar-functions/metrics/src/test/java/org/apache/pulsar/functions/metrics/sink/PrometheusSinkTests.java index 63bb5798c2e2c..887427f3ab760 100644 --- a/pulsar-functions/metrics/src/test/java/org/apache/pulsar/functions/metrics/sink/PrometheusSinkTests.java +++ b/pulsar-functions/metrics/src/test/java/org/apache/pulsar/functions/metrics/sink/PrometheusSinkTests.java @@ -28,7 +28,7 @@ import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; import org.testng.annotations.Test; import org.testng.annotations.BeforeMethod; import static org.testng.Assert.assertEquals; @@ -86,29 +86,29 @@ public void before() { public void testMetricsGrouping() { PrometheusTestSink sink = new PrometheusTestSink(); sink.init(defaultConf); - Function.FunctionConfig functionConfig = createFunctionConfig("tenant", "namespace", "functionname"); - sink.processRecord(records, functionConfig); + Function.FunctionDetails functionDetails = createFunctionDetails("tenant", "namespace", "functionname"); + sink.processRecord(records, functionDetails); final Map> metrics = sink.getMetrics(); - assertTrue(metrics.containsKey(FunctionConfigUtils.getFullyQualifiedName(functionConfig))); + assertTrue(metrics.containsKey(FunctionDetailsUtils.getFullyQualifiedName(functionDetails))); } @Test public void testResponse() throws IOException { PrometheusTestSink sink = new PrometheusTestSink(); sink.init(defaultConf); - Function.FunctionConfig functionConfig = createFunctionConfig("tenant", "namespace", "functionname"); - sink.processRecord(records, functionConfig); + Function.FunctionDetails functionDetails = createFunctionDetails("tenant", "namespace", "functionname"); + sink.processRecord(records, functionDetails); final List expectedLines = Arrays.asList( - createMetric(functionConfig, "metric_1_count", 2), - createMetric(functionConfig, "metric_1_sum", 5), - createMetric(functionConfig, "metric_1_max", 3), - createMetric(functionConfig, "metric_1_min", 2), - createMetric(functionConfig, "metric_2_count", 3), - createMetric(functionConfig, "metric_2_sum", 6), - createMetric(functionConfig, "metric_2_max", 3), - createMetric(functionConfig, "metric_2_min", 1) + createMetric(functionDetails, "metric_1_count", 2), + createMetric(functionDetails, "metric_1_sum", 5), + createMetric(functionDetails, "metric_1_max", 3), + createMetric(functionDetails, "metric_1_min", 2), + createMetric(functionDetails, "metric_2_count", 3), + createMetric(functionDetails, "metric_2_sum", 6), + createMetric(functionDetails, "metric_2_max", 3), + createMetric(functionDetails, "metric_2_min", 1) ); final Set generatedLines = @@ -121,16 +121,16 @@ public void testResponse() throws IOException { }); } - private String createMetric(Function.FunctionConfig functionConfig, + private String createMetric(Function.FunctionDetails functionDetails, String metric, double value) { return String.format("pulsar_function_%s" + "{tenant=\"%s\",namespace=\"%s\",functionname=\"%s\"}" + " %s %d", - metric, functionConfig.getTenant(), functionConfig.getNamespace(), functionConfig.getName(), value, NOW); + metric, functionDetails.getTenant(), functionDetails.getNamespace(), functionDetails.getName(), value, NOW); } - private Function.FunctionConfig createFunctionConfig(String tenant, String namespace, String name) { - Function.FunctionConfig.Builder bldr = Function.FunctionConfig.newBuilder(); + private Function.FunctionDetails createFunctionDetails(String tenant, String namespace, String name) { + Function.FunctionDetails.Builder bldr = Function.FunctionDetails.newBuilder(); bldr.setTenant(tenant); bldr.setNamespace(namespace); bldr.setName(name); diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto index c12059fe45c7d..c9b3da2831790 100644 --- a/pulsar-functions/proto/src/main/proto/Function.proto +++ b/pulsar-functions/proto/src/main/proto/Function.proto @@ -23,7 +23,7 @@ package proto; option java_package = "org.apache.pulsar.functions.proto"; option java_outer_classname = "Function"; -message FunctionConfig { +message FunctionDetails { enum ProcessingGuarantees { ATLEAST_ONCE = 0; // [default value] ATMOST_ONCE = 1; @@ -60,7 +60,7 @@ message PackageLocationMetaData { } message FunctionMetaData { - FunctionConfig functionConfig = 1; + FunctionDetails functionDetails = 1; PackageLocationMetaData packageLocation = 2; uint64 version = 3; uint64 createTime = 4; diff --git a/pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionConfigTest.java b/pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionDetailsTest.java similarity index 79% rename from pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionConfigTest.java rename to pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionDetailsTest.java index b4cb287b68f2e..c5f7dfae8970a 100644 --- a/pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionConfigTest.java +++ b/pulsar-functions/proto/src/test/java/org/apache/pulsar/functions/proto/FunctionDetailsTest.java @@ -20,21 +20,21 @@ import static org.testng.Assert.assertEquals; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; -import org.apache.pulsar.functions.proto.Function.FunctionConfig.ProcessingGuarantees; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.Function.FunctionDetails.ProcessingGuarantees; import org.testng.annotations.Test; /** - * Unit test for {@link FunctionConfig}. + * Unit test for {@link FunctionDetails}. */ -public class FunctionConfigTest { +public class FunctionDetailsTest { /** * Make sure the default processing guarantee is always `ATLEAST_ONCE`. */ @Test public void testDefaultProcessingGuarantee() { - FunctionConfig fc = FunctionConfig.newBuilder().build(); + FunctionDetails fc = FunctionDetails.newBuilder().build(); assertEquals(ProcessingGuarantees.ATLEAST_ONCE, fc.getProcessingGuarantees()); } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java index 594ea10d92772..c549eab4eb72a 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java @@ -30,7 +30,7 @@ import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.functions.instance.InstanceConfig; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.proto.InstanceControlGrpc; @@ -74,7 +74,7 @@ public class JavaInstanceMain { protected String logTopic; @Parameter(names = "--processing_guarantees", description = "Processing Guarantees\n", required = true) - protected FunctionConfig.ProcessingGuarantees processingGuarantees; + protected FunctionDetails.ProcessingGuarantees processingGuarantees; @Parameter(names = "--instance_id", description = "Instance Id\n", required = true) protected String instanceId; @@ -114,15 +114,15 @@ public void start() throws Exception { instanceConfig.setFunctionVersion(functionVersion); instanceConfig.setInstanceId(instanceId); instanceConfig.setMaxBufferedTuples(maxBufferedTuples); - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - functionConfigBuilder.setTenant(tenant); - functionConfigBuilder.setNamespace(namespace); - functionConfigBuilder.setName(functionName); - functionConfigBuilder.setClassName(className); + FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); + functionDetailsBuilder.setTenant(tenant); + functionDetailsBuilder.setNamespace(namespace); + functionDetailsBuilder.setName(functionName); + functionDetailsBuilder.setClassName(className); if (defaultSerdeInputTopics != null) { String[] inputTopics = defaultSerdeInputTopics.split(","); for (String inputTopic : inputTopics) { - functionConfigBuilder.addInputs(inputTopic); + functionDetailsBuilder.addInputs(inputTopic); } } if (customSerdeInputTopics != null && customSerdeClassnames != null) { @@ -132,31 +132,31 @@ public void start() throws Exception { throw new RuntimeException("Error specifying inputs"); } for (int i = 0; i < inputTopics.length; ++i) { - functionConfigBuilder.putCustomSerdeInputs(inputTopics[i], inputSerdeClassNames[i]); + functionDetailsBuilder.putCustomSerdeInputs(inputTopics[i], inputSerdeClassNames[i]); } } if (outputSerdeClassName != null) { - functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName); + functionDetailsBuilder.setOutputSerdeClassName(outputSerdeClassName); } if (outputTopicName != null) { - functionConfigBuilder.setOutput(outputTopicName); + functionDetailsBuilder.setOutput(outputTopicName); } if (logTopic != null) { - functionConfigBuilder.setLogTopic(logTopic); + functionDetailsBuilder.setLogTopic(logTopic); } - functionConfigBuilder.setProcessingGuarantees(processingGuarantees); + functionDetailsBuilder.setProcessingGuarantees(processingGuarantees); if (autoAck.equals("true")) { - functionConfigBuilder.setAutoAck(true); + functionDetailsBuilder.setAutoAck(true); } else { - functionConfigBuilder.setAutoAck(false); + functionDetailsBuilder.setAutoAck(false); } if (userConfig != null && !userConfig.isEmpty()) { Type type = new TypeToken>(){}.getType(); Map userConfigMap = new Gson().fromJson(userConfig, type); - functionConfigBuilder.putAllUserConfig(userConfigMap); + functionDetailsBuilder.putAllUserConfig(userConfigMap); } - FunctionConfig functionConfig = functionConfigBuilder.build(); - instanceConfig.setFunctionConfig(functionConfig); + FunctionDetails functionDetails = functionDetailsBuilder.build(); + instanceConfig.setFunctionDetails(functionDetails); ThreadRuntimeFactory containerFactory = new ThreadRuntimeFactory( "LocalRunnerThreadGroup", diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java index 1bdedd7c453e3..0e8b300cd755f 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java @@ -72,17 +72,17 @@ private List composeArgs(InstanceConfig instanceConfig, String codeFile, String pulsarServiceUrl) { List args = new LinkedList<>(); - if (instanceConfig.getFunctionConfig().getRuntime() == Function.FunctionConfig.Runtime.JAVA) { + if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.JAVA) { args.add("java"); args.add("-cp"); args.add(instanceFile); args.add("-Dlog4j.configurationFile=java_instance_log4j2.yml"); args.add("-Dpulsar.log.dir=" + logDirectory); - args.add("-Dpulsar.log.file=" + instanceConfig.getFunctionConfig().getName()); + args.add("-Dpulsar.log.file=" + instanceConfig.getFunctionDetails().getName()); args.add("org.apache.pulsar.functions.runtime.JavaInstanceMain"); args.add("--jar"); args.add(codeFile); - } else if (instanceConfig.getFunctionConfig().getRuntime() == Function.FunctionConfig.Runtime.PYTHON) { + } else if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.PYTHON) { args.add("python"); args.add(instanceFile); args.add("--py"); @@ -90,7 +90,7 @@ private List composeArgs(InstanceConfig instanceConfig, args.add("--logging_directory"); args.add(logDirectory); args.add("--logging_file"); - args.add(instanceConfig.getFunctionConfig().getName()); + args.add(instanceConfig.getFunctionDetails().getName()); } args.add("--instance_id"); args.add(instanceConfig.getInstanceId()); @@ -99,22 +99,22 @@ private List composeArgs(InstanceConfig instanceConfig, args.add("--function_version"); args.add(instanceConfig.getFunctionVersion()); args.add("--tenant"); - args.add(instanceConfig.getFunctionConfig().getTenant()); + args.add(instanceConfig.getFunctionDetails().getTenant()); args.add("--namespace"); - args.add(instanceConfig.getFunctionConfig().getNamespace()); + args.add(instanceConfig.getFunctionDetails().getNamespace()); args.add("--name"); - args.add(instanceConfig.getFunctionConfig().getName()); + args.add(instanceConfig.getFunctionDetails().getName()); args.add("--function_classname"); - args.add(instanceConfig.getFunctionConfig().getClassName()); - if (instanceConfig.getFunctionConfig().getLogTopic() != null && - !instanceConfig.getFunctionConfig().getLogTopic().isEmpty()) { + args.add(instanceConfig.getFunctionDetails().getClassName()); + if (instanceConfig.getFunctionDetails().getLogTopic() != null && + !instanceConfig.getFunctionDetails().getLogTopic().isEmpty()) { args.add("--log_topic"); - args.add(instanceConfig.getFunctionConfig().getLogTopic()); + args.add(instanceConfig.getFunctionDetails().getLogTopic()); } - if (instanceConfig.getFunctionConfig().getCustomSerdeInputsCount() > 0) { + if (instanceConfig.getFunctionDetails().getCustomSerdeInputsCount() > 0) { String inputTopicString = ""; String inputSerdeClassNameString = ""; - for (Map.Entry entry : instanceConfig.getFunctionConfig().getCustomSerdeInputsMap().entrySet()) { + for (Map.Entry entry : instanceConfig.getFunctionDetails().getCustomSerdeInputsMap().entrySet()) { if (inputTopicString.isEmpty()) { inputTopicString = entry.getKey(); } else { @@ -131,9 +131,9 @@ private List composeArgs(InstanceConfig instanceConfig, args.add("--custom_serde_classnames"); args.add(inputSerdeClassNameString); } - if (instanceConfig.getFunctionConfig().getInputsCount() > 0) { + if (instanceConfig.getFunctionDetails().getInputsCount() > 0) { String inputTopicString = ""; - for (String topicName : instanceConfig.getFunctionConfig().getInputsList()) { + for (String topicName : instanceConfig.getFunctionDetails().getInputsList()) { if (inputTopicString.isEmpty()) { inputTopicString = topicName; } else { @@ -144,28 +144,28 @@ private List composeArgs(InstanceConfig instanceConfig, args.add(inputTopicString); } args.add("--auto_ack"); - if (instanceConfig.getFunctionConfig().getAutoAck()) { + if (instanceConfig.getFunctionDetails().getAutoAck()) { args.add("true"); } else { args.add("false"); } - if (instanceConfig.getFunctionConfig().getOutput() != null - && !instanceConfig.getFunctionConfig().getOutput().isEmpty()) { + if (instanceConfig.getFunctionDetails().getOutput() != null + && !instanceConfig.getFunctionDetails().getOutput().isEmpty()) { args.add("--output_topic"); - args.add(instanceConfig.getFunctionConfig().getOutput()); + args.add(instanceConfig.getFunctionDetails().getOutput()); } - if (instanceConfig.getFunctionConfig().getOutputSerdeClassName() != null - && !instanceConfig.getFunctionConfig().getOutputSerdeClassName().isEmpty()) { + if (instanceConfig.getFunctionDetails().getOutputSerdeClassName() != null + && !instanceConfig.getFunctionDetails().getOutputSerdeClassName().isEmpty()) { args.add("--output_serde_classname"); - args.add(instanceConfig.getFunctionConfig().getOutputSerdeClassName()); + args.add(instanceConfig.getFunctionDetails().getOutputSerdeClassName()); } args.add("--processing_guarantees"); - args.add(String.valueOf(instanceConfig.getFunctionConfig().getProcessingGuarantees())); + args.add(String.valueOf(instanceConfig.getFunctionDetails().getProcessingGuarantees())); args.add("--pulsar_serviceurl"); args.add(pulsarServiceUrl); args.add("--max_buffered_tuples"); args.add(String.valueOf(instanceConfig.getMaxBufferedTuples())); - Map userConfig = instanceConfig.getFunctionConfig().getUserConfigMap(); + Map userConfig = instanceConfig.getFunctionDetails().getUserConfigMap(); if (userConfig != null && !userConfig.isEmpty()) { args.add("--user_config"); args.add(new Gson().toJson(userConfig)); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java index 31673d2ef7e70..2223cb7cada80 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java @@ -85,7 +85,7 @@ public ProcessRuntimeFactory(String pulsarServiceUrl, @Override public ProcessRuntime createContainer(InstanceConfig instanceConfig, String codeFile) { String instanceFile; - switch (instanceConfig.getFunctionConfig().getRuntime()) { + switch (instanceConfig.getFunctionDetails().getRuntime()) { case JAVA: instanceFile = javaInstanceJarFile; break; @@ -93,7 +93,7 @@ public ProcessRuntime createContainer(InstanceConfig instanceConfig, String code instanceFile = pythonInstanceFile; break; default: - throw new RuntimeException("Unsupported Runtime " + instanceConfig.getFunctionConfig().getRuntime()); + throw new RuntimeException("Unsupported Runtime " + instanceConfig.getFunctionDetails().getRuntime()); } return new ProcessRuntime( instanceConfig, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java index 6bcb988e1f14e..272ca6a9e9c94 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java @@ -57,7 +57,7 @@ public RuntimeSpawner(InstanceConfig instanceConfig, } public void start() throws Exception { - log.info("RuntimeSpawner starting function {} - {}", this.instanceConfig.getFunctionConfig().getName(), + log.info("RuntimeSpawner starting function {} - {}", this.instanceConfig.getFunctionDetails().getName(), this.instanceConfig.getInstanceId()); runtime = runtimeFactory.createContainer(this.instanceConfig, codeFile); runtime.start(); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ThreadRuntime.java index 1d53a70e382f8..8cd858ac754f5 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ThreadRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ThreadRuntime.java @@ -30,7 +30,7 @@ import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus; import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager; import org.apache.pulsar.functions.instance.JavaInstanceRunnable; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; /** * A function container implemented using java thread. @@ -54,7 +54,7 @@ class ThreadRuntime implements Runtime { PulsarClient pulsarClient, String stateStorageServiceUrl) { this.instanceConfig = instanceConfig; - if (instanceConfig.getFunctionConfig().getRuntime() != Function.FunctionConfig.Runtime.JAVA) { + if (instanceConfig.getFunctionDetails().getRuntime() != Function.FunctionDetails.Runtime.JAVA) { throw new RuntimeException("Thread Container only supports Java Runtime"); } this.javaInstanceRunnable = new JavaInstanceRunnable( @@ -74,7 +74,7 @@ public void start() { log.info("ThreadContainer starting function with instance config {}", instanceConfig); startupException = null; this.fnThread = new Thread(threadGroup, javaInstanceRunnable, - FunctionConfigUtils.getFullyQualifiedName(instanceConfig.getFunctionConfig())); + FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails())); this.fnThread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @Override public void uncaughtException(Thread t, Throwable e) { diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java index d19d57d7b4860..48602e39c4772 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java @@ -21,7 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.functions.instance.InstanceConfig; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.runtime.ProcessRuntime; import org.apache.pulsar.functions.runtime.ProcessRuntimeFactory; import org.apache.pulsar.functions.runtime.ThreadRuntime; @@ -68,25 +68,25 @@ public void tearDown() { this.factory.close(); } - FunctionConfig createFunctionConfig(FunctionConfig.Runtime runtime) { - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - functionConfigBuilder.setRuntime(runtime); - functionConfigBuilder.setTenant(TEST_TENANT); - functionConfigBuilder.setNamespace(TEST_NAMESPACE); - functionConfigBuilder.setName(TEST_NAME); - functionConfigBuilder.setClassName("org.apache.pulsar.functions.utils.functioncache.AddFunction"); - functionConfigBuilder.addInputs(TEST_NAME + "-input1"); - functionConfigBuilder.addInputs(TEST_NAME + "-input2"); - functionConfigBuilder.setOutput(TEST_NAME + "-output"); - functionConfigBuilder.setOutputSerdeClassName("org.apache.pulsar.functions.runtime.serde.Utf8Serializer"); - functionConfigBuilder.setLogTopic(TEST_NAME + "-log"); - return functionConfigBuilder.build(); + FunctionDetails createFunctionDetails(FunctionDetails.Runtime runtime) { + FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); + functionDetailsBuilder.setRuntime(runtime); + functionDetailsBuilder.setTenant(TEST_TENANT); + functionDetailsBuilder.setNamespace(TEST_NAMESPACE); + functionDetailsBuilder.setName(TEST_NAME); + functionDetailsBuilder.setClassName("org.apache.pulsar.functions.utils.functioncache.AddFunction"); + functionDetailsBuilder.addInputs(TEST_NAME + "-input1"); + functionDetailsBuilder.addInputs(TEST_NAME + "-input2"); + functionDetailsBuilder.setOutput(TEST_NAME + "-output"); + functionDetailsBuilder.setOutputSerdeClassName("org.apache.pulsar.functions.runtime.serde.Utf8Serializer"); + functionDetailsBuilder.setLogTopic(TEST_NAME + "-log"); + return functionDetailsBuilder.build(); } - InstanceConfig createJavaInstanceConfig(FunctionConfig.Runtime runtime) { + InstanceConfig createJavaInstanceConfig(FunctionDetails.Runtime runtime) { InstanceConfig config = new InstanceConfig(); - config.setFunctionConfig(createFunctionConfig(runtime)); + config.setFunctionDetails(createFunctionDetails(runtime)); config.setFunctionId(java.util.UUID.randomUUID().toString()); config.setFunctionVersion("1.0"); config.setInstanceId(java.util.UUID.randomUUID().toString()); @@ -97,26 +97,26 @@ InstanceConfig createJavaInstanceConfig(FunctionConfig.Runtime runtime) { @Test public void testJavaConstructor() { - InstanceConfig config = createJavaInstanceConfig(FunctionConfig.Runtime.JAVA); + InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA); ProcessRuntime container = factory.createContainer(config, userJarFile); List args = container.getProcessArgs(); assertEquals(args.size(), 41); args.remove(args.size() - 1); String expectedArgs = "java -cp " + javaInstanceJarFile + " -Dlog4j.configurationFile=java_instance_log4j2.yml " - + "-Dpulsar.log.dir=" + logDirectory + "/functions" + " -Dpulsar.log.file=" + config.getFunctionConfig().getName() + + "-Dpulsar.log.dir=" + logDirectory + "/functions" + " -Dpulsar.log.file=" + config.getFunctionDetails().getName() + " org.apache.pulsar.functions.runtime.JavaInstanceMain" + " --jar " + userJarFile + " --instance_id " + config.getInstanceId() + " --function_id " + config.getFunctionId() - + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionConfig().getTenant() - + " --namespace " + config.getFunctionConfig().getNamespace() - + " --name " + config.getFunctionConfig().getName() - + " --function_classname " + config.getFunctionConfig().getClassName() - + " --log_topic " + config.getFunctionConfig().getLogTopic() + + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionDetails().getTenant() + + " --namespace " + config.getFunctionDetails().getNamespace() + + " --name " + config.getFunctionDetails().getName() + + " --function_classname " + config.getFunctionDetails().getClassName() + + " --log_topic " + config.getFunctionDetails().getLogTopic() + " --input_topics " + TEST_NAME + "-input1," + TEST_NAME + "-input2" + " --auto_ack false" - + " --output_topic " + config.getFunctionConfig().getOutput() - + " --output_serde_classname " + config.getFunctionConfig().getOutputSerdeClassName() + + " --output_topic " + config.getFunctionDetails().getOutput() + + " --output_serde_classname " + config.getFunctionDetails().getOutputSerdeClassName() + " --processing_guarantees ATLEAST_ONCE" + " --pulsar_serviceurl " + pulsarServiceUrl + " --max_buffered_tuples 1024 --port"; @@ -125,7 +125,7 @@ public void testJavaConstructor() { @Test public void testPythonConstructor() { - InstanceConfig config = createJavaInstanceConfig(FunctionConfig.Runtime.PYTHON); + InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.PYTHON); ProcessRuntime container = factory.createContainer(config, userJarFile); List args = container.getProcessArgs(); @@ -133,17 +133,17 @@ public void testPythonConstructor() { args.remove(args.size() - 1); String expectedArgs = "python " + pythonInstanceFile + " --py " + userJarFile + " --logging_directory " - + logDirectory + "/functions" + " --logging_file " + config.getFunctionConfig().getName() + " --instance_id " + + logDirectory + "/functions" + " --logging_file " + config.getFunctionDetails().getName() + " --instance_id " + config.getInstanceId() + " --function_id " + config.getFunctionId() - + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionConfig().getTenant() - + " --namespace " + config.getFunctionConfig().getNamespace() - + " --name " + config.getFunctionConfig().getName() - + " --function_classname " + config.getFunctionConfig().getClassName() - + " --log_topic " + config.getFunctionConfig().getLogTopic() + + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionDetails().getTenant() + + " --namespace " + config.getFunctionDetails().getNamespace() + + " --name " + config.getFunctionDetails().getName() + + " --function_classname " + config.getFunctionDetails().getClassName() + + " --log_topic " + config.getFunctionDetails().getLogTopic() + " --input_topics " + TEST_NAME + "-input1," + TEST_NAME + "-input2" + " --auto_ack false" - + " --output_topic " + config.getFunctionConfig().getOutput() - + " --output_serde_classname " + config.getFunctionConfig().getOutputSerdeClassName() + + " --output_topic " + config.getFunctionDetails().getOutput() + + " --output_serde_classname " + config.getFunctionDetails().getOutputSerdeClassName() + " --processing_guarantees ATLEAST_ONCE" + " --pulsar_serviceurl " + pulsarServiceUrl + " --max_buffered_tuples 1024 --port"; diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java new file mode 100644 index 0000000000000..fe293eafa0f30 --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; + +@Getter +@Setter +@Data +@EqualsAndHashCode +@ToString +public class FunctionConfig { + + public enum ProcessingGuarantees { + ATLEAST_ONCE, + ATMOST_ONCE, + EFFECTIVELY_ONCE + } + + public enum SubscriptionType { + SHARED, + EXCLUSIVE + } + + public enum Runtime { + JAVA, + PYTHON + } + + private String tenant; + private String namespace; + private String name; + private String className; + + private Collection inputs = new LinkedList<>(); + private Map customSerdeInputs = new HashMap<>(); + + private String output; + private String outputSerdeClassName; + + private String logTopic; + private ProcessingGuarantees processingGuarantees; + private Map userConfig = new HashMap<>(); + private SubscriptionType subscriptionType; + private Runtime runtime; + private boolean autoAck; + private int parallelism; + private String fqfn; +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java similarity index 52% rename from pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java rename to pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java index c715b033dc94c..77060302f03be 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java @@ -19,33 +19,12 @@ package org.apache.pulsar.functions.utils; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import java.io.File; -import java.io.IOException; +public class FunctionDetailsUtils { -public class FunctionConfigUtils { - - public static String convertYamlToJson(File file) throws IOException { - - ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory()); - Object obj = yamlReader.readValue(file, Object.class); - - ObjectMapper jsonWriter = new ObjectMapper(); - return jsonWriter.writeValueAsString(obj); - } - - public static FunctionConfig.Builder loadConfig(File file) throws IOException { - String json = convertYamlToJson(file); - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - Utils.mergeJson(json, functionConfigBuilder); - return functionConfigBuilder; - } - - public static String getFullyQualifiedName(FunctionConfig functionConfig) { - return getFullyQualifiedName(functionConfig.getTenant(), functionConfig.getNamespace(), functionConfig.getName()); + public static String getFullyQualifiedName(FunctionDetails FunctionDetails) { + return getFullyQualifiedName(FunctionDetails.getTenant(), FunctionDetails.getNamespace(), FunctionDetails.getName()); } public static String getFullyQualifiedName(String tenant, String namespace, String functionName) { @@ -64,34 +43,34 @@ public static String extractFunctionNameFromFQN(String fullyQualifiedName) { return fullyQualifiedName.split("/")[2]; } - public static boolean areAllRequiredFieldsPresent(FunctionConfig functionConfig) { - if (functionConfig.getTenant() == null || functionConfig.getNamespace() == null - || functionConfig.getName() == null || functionConfig.getClassName() == null - || (functionConfig.getInputsCount() <= 0 && functionConfig.getCustomSerdeInputsCount() <= 0) - || functionConfig.getParallelism() <= 0) { + public static boolean areAllRequiredFieldsPresent(FunctionDetails FunctionDetails) { + if (FunctionDetails.getTenant() == null || FunctionDetails.getNamespace() == null + || FunctionDetails.getName() == null || FunctionDetails.getClassName() == null + || (FunctionDetails.getInputsCount() <= 0 && FunctionDetails.getCustomSerdeInputsCount() <= 0) + || FunctionDetails.getParallelism() <= 0) { return false; } else { return true; } } - public static String getDownloadFileName(FunctionConfig functionConfig) { - String[] hierarchy = functionConfig.getClassName().split("\\."); + public static String getDownloadFileName(FunctionDetails FunctionDetails) { + String[] hierarchy = FunctionDetails.getClassName().split("\\."); String fileName; if (hierarchy.length <= 0) { - fileName = functionConfig.getClassName(); + fileName = FunctionDetails.getClassName(); } else if (hierarchy.length == 1) { fileName = hierarchy[0]; } else { fileName = hierarchy[hierarchy.length - 2]; } - switch (functionConfig.getRuntime()) { + switch (FunctionDetails.getRuntime()) { case JAVA: return fileName + ".jar"; case PYTHON: return fileName + ".py"; default: - throw new RuntimeException("Unknown runtime " + functionConfig.getRuntime()); + throw new RuntimeException("Unknown runtime " + FunctionDetails.getRuntime()); } } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 1eb3d288ae02c..702f314a7fdaa 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -33,7 +33,7 @@ import org.apache.pulsar.functions.runtime.RuntimeFactory; import org.apache.pulsar.functions.instance.InstanceConfig; import org.apache.pulsar.functions.runtime.RuntimeSpawner; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; import java.io.File; import java.io.FileOutputStream; @@ -105,7 +105,7 @@ private void startFunction(FunctionRuntimeInfo functionRuntimeInfo) throws Excep Function.Instance instance = functionRuntimeInfo.getFunctionInstance(); FunctionMetaData functionMetaData = instance.getFunctionMetaData(); log.info("Starting function {} - {} ...", - functionMetaData.getFunctionConfig().getName(), instance.getInstanceId()); + functionMetaData.getFunctionDetails().getName(), instance.getInstanceId()); File pkgDir = new File( workerConfig.getDownloadDirectory(), getDownloadPackagePath(functionMetaData)); @@ -115,7 +115,7 @@ private void startFunction(FunctionRuntimeInfo functionRuntimeInfo) throws Excep File pkgFile = new File( pkgDir, - new File(FunctionConfigUtils.getDownloadFileName(functionMetaData.getFunctionConfig())).getName()); + new File(FunctionDetailsUtils.getDownloadFileName(functionMetaData.getFunctionDetails())).getName()); if (!pkgFile.exists()) { // download only when the package file doesn't exist @@ -155,7 +155,7 @@ private void startFunction(FunctionRuntimeInfo functionRuntimeInfo) throws Excep } InstanceConfig instanceConfig = new InstanceConfig(); - instanceConfig.setFunctionConfig(functionMetaData.getFunctionConfig()); + instanceConfig.setFunctionDetails(functionMetaData.getFunctionDetails()); // TODO: set correct function id and version when features implemented instanceConfig.setFunctionId(UUID.randomUUID().toString()); instanceConfig.setFunctionVersion(UUID.randomUUID().toString()); @@ -172,7 +172,7 @@ private void stopFunction(FunctionRuntimeInfo functionRuntimeInfo) { Function.Instance instance = functionRuntimeInfo.getFunctionInstance(); FunctionMetaData functionMetaData = instance.getFunctionMetaData(); log.info("Stopping function {} - {}...", - functionMetaData.getFunctionConfig().getName(), instance.getInstanceId()); + functionMetaData.getFunctionDetails().getName(), instance.getInstanceId()); if (functionRuntimeInfo.getRuntimeSpawner() != null) { functionRuntimeInfo.getRuntimeSpawner().close(); functionRuntimeInfo.setRuntimeSpawner(null); @@ -188,7 +188,7 @@ private void stopFunction(FunctionRuntimeInfo functionRuntimeInfo) { FileUtils.deleteDirectory(pkgDir); } catch (IOException e) { log.warn("Failed to delete package for function: {}", - FunctionConfigUtils.getFullyQualifiedName(functionMetaData.getFunctionConfig()), e); + FunctionDetailsUtils.getFullyQualifiedName(functionMetaData.getFunctionDetails()), e); } } } @@ -196,9 +196,9 @@ private void stopFunction(FunctionRuntimeInfo functionRuntimeInfo) { private String getDownloadPackagePath(FunctionMetaData functionMetaData) { return StringUtils.join( new String[]{ - functionMetaData.getFunctionConfig().getTenant(), - functionMetaData.getFunctionConfig().getNamespace(), - functionMetaData.getFunctionConfig().getName(), + functionMetaData.getFunctionDetails().getTenant(), + functionMetaData.getFunctionDetails().getNamespace(), + functionMetaData.getFunctionDetails().getName(), }, File.separatorChar); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java index 8a8f95e5c4cc1..b5f9751540918 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java @@ -153,7 +153,7 @@ public synchronized Collection listFunctions(String tenant, String names return ret; } for (FunctionMetaData functionMetaData : this.functionMetaDataMap.get(tenant).get(namespace).values()) { - ret.add(functionMetaData.getFunctionConfig().getName()); + ret.add(functionMetaData.getFunctionDetails().getName()); } return ret; } @@ -178,19 +178,19 @@ public synchronized CompletableFuture updateFunction(FunctionMeta long version = 0; - String tenant = functionMetaData.getFunctionConfig().getTenant(); + String tenant = functionMetaData.getFunctionDetails().getTenant(); if (!this.functionMetaDataMap.containsKey(tenant)) { this.functionMetaDataMap.put(tenant, new ConcurrentHashMap<>()); } Map> namespaces = this.functionMetaDataMap.get(tenant); - String namespace = functionMetaData.getFunctionConfig().getNamespace(); + String namespace = functionMetaData.getFunctionDetails().getNamespace(); if (!namespaces.containsKey(namespace)) { namespaces.put(namespace, new ConcurrentHashMap<>()); } Map functionMetaDatas = namespaces.get(namespace); - String functionName = functionMetaData.getFunctionConfig().getName(); + String functionName = functionMetaData.getFunctionDetails().getName(); if (functionMetaDatas.containsKey(functionName)) { version = functionMetaDatas.get(functionName).getVersion() + 1; } @@ -251,12 +251,12 @@ public void processRequest(MessageId messageId, Request.ServiceRequest serviceRe */ private boolean containsFunctionMetaData(FunctionMetaData functionMetaData) { - return containsFunctionMetaData(functionMetaData.getFunctionConfig()); + return containsFunctionMetaData(functionMetaData.getFunctionDetails()); } - private boolean containsFunctionMetaData(Function.FunctionConfig functionConfig) { + private boolean containsFunctionMetaData(Function.FunctionDetails functionDetails) { return containsFunctionMetaData( - functionConfig.getTenant(), functionConfig.getNamespace(), functionConfig.getName()); + functionDetails.getTenant(), functionDetails.getNamespace(), functionDetails.getName()); } private boolean containsFunctionMetaData(String tenant, String namespace, String functionName) { @@ -274,9 +274,9 @@ private boolean containsFunctionMetaData(String tenant, String namespace, String synchronized void proccessDeregister(Request.ServiceRequest deregisterRequest) { FunctionMetaData deregisterRequestFs = deregisterRequest.getFunctionMetaData(); - String functionName = deregisterRequestFs.getFunctionConfig().getName(); - String tenant = deregisterRequestFs.getFunctionConfig().getTenant(); - String namespace = deregisterRequestFs.getFunctionConfig().getNamespace(); + String functionName = deregisterRequestFs.getFunctionDetails().getName(); + String tenant = deregisterRequestFs.getFunctionDetails().getTenant(); + String namespace = deregisterRequestFs.getFunctionDetails().getNamespace(); boolean needsScheduling = false; @@ -363,25 +363,25 @@ private void completeRequest(Request.ServiceRequest serviceRequest, boolean isSu private boolean isRequestOutdated(Request.ServiceRequest serviceRequest) { FunctionMetaData requestFunctionMetaData = serviceRequest.getFunctionMetaData(); - Function.FunctionConfig functionConfig = requestFunctionMetaData.getFunctionConfig(); - FunctionMetaData currentFunctionMetaData = this.functionMetaDataMap.get(functionConfig.getTenant()) - .get(functionConfig.getNamespace()).get(functionConfig.getName()); + Function.FunctionDetails functionDetails = requestFunctionMetaData.getFunctionDetails(); + FunctionMetaData currentFunctionMetaData = this.functionMetaDataMap.get(functionDetails.getTenant()) + .get(functionDetails.getNamespace()).get(functionDetails.getName()); return currentFunctionMetaData.getVersion() >= requestFunctionMetaData.getVersion(); } @VisibleForTesting void setFunctionMetaData(FunctionMetaData functionMetaData) { - Function.FunctionConfig functionConfig = functionMetaData.getFunctionConfig(); - if (!this.functionMetaDataMap.containsKey(functionConfig.getTenant())) { - this.functionMetaDataMap.put(functionConfig.getTenant(), new ConcurrentHashMap<>()); + Function.FunctionDetails functionDetails = functionMetaData.getFunctionDetails(); + if (!this.functionMetaDataMap.containsKey(functionDetails.getTenant())) { + this.functionMetaDataMap.put(functionDetails.getTenant(), new ConcurrentHashMap<>()); } - if (!this.functionMetaDataMap.get(functionConfig.getTenant()).containsKey(functionConfig.getNamespace())) { - this.functionMetaDataMap.get(functionConfig.getTenant()) - .put(functionConfig.getNamespace(), new ConcurrentHashMap<>()); + if (!this.functionMetaDataMap.get(functionDetails.getTenant()).containsKey(functionDetails.getNamespace())) { + this.functionMetaDataMap.get(functionDetails.getTenant()) + .put(functionDetails.getNamespace(), new ConcurrentHashMap<>()); } - this.functionMetaDataMap.get(functionConfig.getTenant()) - .get(functionConfig.getNamespace()).put(functionConfig.getName(), functionMetaData); + this.functionMetaDataMap.get(functionDetails.getTenant()) + .get(functionDetails.getNamespace()).put(functionDetails.getName(), functionMetaData); } @VisibleForTesting diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java index a20c782e062d1..f7e793a3a5a90 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java @@ -181,13 +181,13 @@ public static Collection findFunctionAssignments(String tenant, .filter( assignment -> (tenant.equals(assignment.getInstance() - .getFunctionMetaData().getFunctionConfig() + .getFunctionMetaData().getFunctionDetails() .getTenant()) && namespace.equals((assignment.getInstance() - .getFunctionMetaData().getFunctionConfig() + .getFunctionMetaData().getFunctionDetails() .getNamespace())) && functionName.equals(assignment.getInstance() - .getFunctionMetaData().getFunctionConfig() + .getFunctionMetaData().getFunctionDetails() .getName()))) .collect(Collectors.toList())); } @@ -316,9 +316,9 @@ public InstanceCommunication.FunctionStatusList getAllFunctionStatus( for (Assignment assignment : assignments) { InstanceCommunication.FunctionStatus functionStatus = this.getFunctionInstanceStatus( - assignment.getInstance().getFunctionMetaData().getFunctionConfig().getTenant(), - assignment.getInstance().getFunctionMetaData().getFunctionConfig().getNamespace(), - assignment.getInstance().getFunctionMetaData().getFunctionConfig().getName(), + assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(), + assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(), + assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(), assignment.getInstance().getInstanceId()); functionStatusListBuilder.addFunctionStatusList(functionStatus); @@ -468,9 +468,9 @@ private Assignment findAssignment(String tenant, String namespace, String functi private Assignment findAssignment(Assignment assignment) { return findAssignment( - assignment.getInstance().getFunctionMetaData().getFunctionConfig().getTenant(), - assignment.getInstance().getFunctionMetaData().getFunctionConfig().getNamespace(), - assignment.getInstance().getFunctionMetaData().getFunctionConfig().getName(), + assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(), + assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(), + assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(), assignment.getInstance().getInstanceId() ); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java index c9fd53987a352..0a829cbf084cc 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java @@ -56,11 +56,11 @@ public static void generate(WorkerService workerService, String cluster, SimpleT InstanceCommunication.MetricsData.DataDigest dataDigest = metricsEntry.getValue(); String tenant = functionRuntimeInfo.getFunctionInstance() - .getFunctionMetaData().getFunctionConfig().getTenant(); + .getFunctionMetaData().getFunctionDetails().getTenant(); String namespace = functionRuntimeInfo.getFunctionInstance() - .getFunctionMetaData().getFunctionConfig().getNamespace(); + .getFunctionMetaData().getFunctionDetails().getNamespace(); String name = functionRuntimeInfo.getFunctionInstance() - .getFunctionMetaData().getFunctionConfig().getName(); + .getFunctionMetaData().getFunctionDetails().getName(); int instanceId = functionRuntimeInfo.getFunctionInstance().getInstanceId(); String qualifiedNamespace = String.format("%s/%s", tenant, namespace); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java index 4942b34782faf..4e5fb2139d805 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java @@ -47,7 +47,7 @@ import org.apache.pulsar.common.policies.data.ConsumerStats; import org.apache.pulsar.common.policies.data.PersistentTopicStats; import org.apache.pulsar.functions.proto.Function; -import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; /** * A simple implementation of leader election using a pulsar topic. @@ -180,7 +180,7 @@ public void checkFailures(FunctionMetaDataManager functionMetaDataManager, List functionMetaDataList = functionMetaDataManager.getAllFunctionMetaData(); Map functionMetaDataMap = new HashMap<>(); for (Function.FunctionMetaData entry : functionMetaDataList) { - functionMetaDataMap.put(FunctionConfigUtils.getFullyQualifiedName(entry.getFunctionConfig()), entry); + functionMetaDataMap.put(FunctionDetailsUtils.getFullyQualifiedName(entry.getFunctionDetails()), entry); } Map> currentAssignments = functionRuntimeManager.getCurrentAssignments(); Map assignmentMap = new HashMap<>(); @@ -193,8 +193,8 @@ public void checkFailures(FunctionMetaDataManager functionMetaDataManager, Iterator> it = unsignedFunctionDurations.entrySet().iterator(); while (it.hasNext()) { Map.Entry entry = it.next(); - String fullyQualifiedFunctionName = FunctionConfigUtils.getFullyQualifiedName( - entry.getKey().getFunctionMetaData().getFunctionConfig()); + String fullyQualifiedFunctionName = FunctionDetailsUtils.getFullyQualifiedName( + entry.getKey().getFunctionMetaData().getFunctionDetails()); String fullyQualifiedInstanceId = Utils.getFullyQualifiedInstanceId(entry.getKey()); //remove functions that don't exist anymore if (!functionMetaDataMap.containsKey(fullyQualifiedFunctionName)) { @@ -215,9 +215,9 @@ public void checkFailures(FunctionMetaDataManager functionMetaDataManager, // check for function instances that haven't been assigned for (Function.FunctionMetaData functionMetaData : functionMetaDataList) { Collection assignments - = FunctionRuntimeManager.findFunctionAssignments(functionMetaData.getFunctionConfig().getTenant(), - functionMetaData.getFunctionConfig().getNamespace(), - functionMetaData.getFunctionConfig().getName(), + = FunctionRuntimeManager.findFunctionAssignments(functionMetaData.getFunctionDetails().getTenant(), + functionMetaData.getFunctionDetails().getNamespace(), + functionMetaData.getFunctionDetails().getName(), currentAssignments); Set assignedInstances = assignments.stream() diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java index fd61161af9601..c6b72a5436158 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/SchedulerManager.java @@ -192,7 +192,7 @@ public static Map computeAllInstances(List computeInstances(FunctionMetaData functionMetaData) { List functionInstances = new LinkedList<>(); - int instances = functionMetaData.getFunctionConfig().getParallelism(); + int instances = functionMetaData.getFunctionDetails().getParallelism(); for (int i = 0; i < instances; i++) { functionInstances.add(Function.Instance.newBuilder() .setFunctionMetaData(functionMetaData) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java index d5d716a5376c3..fba562fea5167 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java @@ -186,9 +186,9 @@ public static PulsarAdmin getPulsarAdminClient(String pulsarWebServiceUrl) { public static String getFullyQualifiedInstanceId(Function.Instance instance) { return getFullyQualifiedInstanceId( - instance.getFunctionMetaData().getFunctionConfig().getTenant(), - instance.getFunctionMetaData().getFunctionConfig().getNamespace(), - instance.getFunctionMetaData().getFunctionConfig().getName(), + instance.getFunctionMetaData().getFunctionDetails().getTenant(), + instance.getFunctionMetaData().getFunctionDetails().getNamespace(), + instance.getFunctionMetaData().getFunctionDetails().getName(), instance.getInstanceId()); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index f6d6523ef30eb..1df28715943ba 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -47,7 +47,7 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.common.policies.data.ErrorData; import org.apache.pulsar.functions.proto.Function; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.Function.FunctionMetaData; import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; import org.apache.pulsar.functions.proto.InstanceCommunication; @@ -87,12 +87,12 @@ public Response registerFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("data") FormDataContentDisposition fileDetail, - final @FormDataParam("functionConfig") String functionConfigJson) { - FunctionConfig functionConfig; + final @FormDataParam("functionDetails") String functionDetailsJson) { + FunctionDetails functionDetails; // validate parameters try { - functionConfig = validateUpdateRequestParams(tenant, namespace, functionName, - uploadedInputStream, fileDetail, functionConfigJson); + functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, + uploadedInputStream, fileDetail, functionDetailsJson); } catch (IllegalArgumentException e) { log.error("Invalid register function request @ /{}/{}/{}", tenant, namespace, functionName, e); @@ -112,7 +112,7 @@ public Response registerFunction(final @PathParam("tenant") String tenant, // function state FunctionMetaData.Builder functionMetaDataBuilder = FunctionMetaData.newBuilder() - .setFunctionConfig(functionConfig) + .setFunctionDetails(functionDetails) .setCreateTime(System.currentTimeMillis()) .setVersion(0); @@ -136,13 +136,13 @@ public Response updateFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("data") FormDataContentDisposition fileDetail, - final @FormDataParam("functionConfig") String functionConfigJson) { + final @FormDataParam("functionDetails") String functionDetailsJson) { - FunctionConfig functionConfig; + FunctionDetails functionDetails; // validate parameters try { - functionConfig = validateUpdateRequestParams(tenant, namespace, functionName, - uploadedInputStream, fileDetail, functionConfigJson); + functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, + uploadedInputStream, fileDetail, functionDetailsJson); } catch (IllegalArgumentException e) { log.error("Invalid update function request @ /{}/{}/{}", tenant, namespace, functionName, e); @@ -161,7 +161,7 @@ public Response updateFunction(final @PathParam("tenant") String tenant, // function state FunctionMetaData.Builder functionMetaDataBuilder = FunctionMetaData.newBuilder() - .setFunctionConfig(functionConfig) + .setFunctionDetails(functionDetails) .setCreateTime(System.currentTimeMillis()) .setVersion(0); @@ -262,8 +262,8 @@ public Response getFunctionInfo(final @PathParam("tenant") String tenant, } FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, functionName); - String functionConfigJson = org.apache.pulsar.functions.utils.Utils.printJson(functionMetaData.getFunctionConfig()); - return Response.status(Status.OK).entity(functionConfigJson).build(); + String functionDetailsJson = org.apache.pulsar.functions.utils.Utils.printJson(functionMetaData.getFunctionDetails()); + return Response.status(Status.OK).entity(functionDetailsJson).build(); } @GET @@ -302,8 +302,8 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena log.error("Got Exception Getting Status", e); FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder(); functionStatusBuilder.setRunning(false); - String functionConfigJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build()); - return Response.status(Status.OK).entity(functionConfigJson).build(); + String functionDetailsJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build()); + return Response.status(Status.OK).entity(functionDetailsJson).build(); } String jsonResponse = org.apache.pulsar.functions.utils.Utils.printJson(functionStatus); @@ -344,8 +344,8 @@ public Response getFunctionStatus(final @PathParam("tenant") String tenant, log.error("Got Exception Getting Status", e); FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder(); functionStatusBuilder.setRunning(false); - String functionConfigJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build()); - return Response.status(Status.OK).entity(functionConfigJson).build(); + String functionDetailsJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build()); + return Response.status(Status.OK).entity(functionDetailsJson).build(); } String jsonResponse = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusList); @@ -452,7 +452,7 @@ public Response triggerFunction(final @PathParam("tenant") String tenant, final @PathParam("name") String functionName, final @FormDataParam("data") String input, final @FormDataParam("dataStream") InputStream uploadedInputStream) { - FunctionConfig functionConfig; + FunctionDetails functionDetails; // validate parameters try { validateTriggerRequestParams(tenant, namespace, functionName, input, uploadedInputStream); @@ -475,12 +475,12 @@ public Response triggerFunction(final @PathParam("tenant") String tenant, FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, functionName); String inputTopicToWrite; - if (functionMetaData.getFunctionConfig().getInputsList().size() > 0) { - inputTopicToWrite = functionMetaData.getFunctionConfig().getInputsList().get(0); + if (functionMetaData.getFunctionDetails().getInputsList().size() > 0) { + inputTopicToWrite = functionMetaData.getFunctionDetails().getInputsList().get(0); } else { - inputTopicToWrite = functionMetaData.getFunctionConfig().getCustomSerdeInputs().entrySet().iterator().next().getKey(); + inputTopicToWrite = functionMetaData.getFunctionDetails().getCustomSerdeInputs().entrySet().iterator().next().getKey(); } - String outputTopic = functionMetaData.getFunctionConfig().getOutput(); + String outputTopic = functionMetaData.getFunctionDetails().getOutput(); Reader reader = null; Producer producer = null; try { @@ -577,12 +577,12 @@ private void validateDeregisterRequestParams(String tenant, } } - private FunctionConfig validateUpdateRequestParams(String tenant, + private FunctionDetails validateUpdateRequestParams(String tenant, String namespace, String functionName, InputStream uploadedInputStream, FormDataContentDisposition fileDetail, - String functionConfigJson) throws IllegalArgumentException { + String functionDetailsJson) throws IllegalArgumentException { if (tenant == null) { throw new IllegalArgumentException("Tenant is not provided"); } @@ -595,42 +595,42 @@ private FunctionConfig validateUpdateRequestParams(String tenant, if (uploadedInputStream == null || fileDetail == null) { throw new IllegalArgumentException("Function Package is not provided"); } - if (functionConfigJson == null) { - throw new IllegalArgumentException("FunctionConfig is not provided"); + if (functionDetailsJson == null) { + throw new IllegalArgumentException("FunctionDetails is not provided"); } try { - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - org.apache.pulsar.functions.utils.Utils.mergeJson(functionConfigJson, functionConfigBuilder); - FunctionConfig functionConfig = functionConfigBuilder.build(); + FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); + org.apache.pulsar.functions.utils.Utils.mergeJson(functionDetailsJson, functionDetailsBuilder); + FunctionDetails functionDetails = functionDetailsBuilder.build(); List missingFields = new LinkedList<>(); - if (functionConfig.getTenant() == null || functionConfig.getTenant().isEmpty()) { + if (functionDetails.getTenant() == null || functionDetails.getTenant().isEmpty()) { missingFields.add("Tenant"); } - if (functionConfig.getNamespace() == null || functionConfig.getNamespace().isEmpty()) { + if (functionDetails.getNamespace() == null || functionDetails.getNamespace().isEmpty()) { missingFields.add("Namespace"); } - if (functionConfig.getName() == null || functionConfig.getName().isEmpty()) { + if (functionDetails.getName() == null || functionDetails.getName().isEmpty()) { missingFields.add("Name"); } - if (functionConfig.getClassName() == null || functionConfig.getClassName().isEmpty()) { + if (functionDetails.getClassName() == null || functionDetails.getClassName().isEmpty()) { missingFields.add("ClassName"); } - if (functionConfig.getInputsCount() == 0 && functionConfig.getCustomSerdeInputsCount() == 0) { + if (functionDetails.getInputsCount() == 0 && functionDetails.getCustomSerdeInputsCount() == 0) { missingFields.add("Input"); } if (!missingFields.isEmpty()) { String errorMessage = StringUtils.join(missingFields, ","); throw new IllegalArgumentException(errorMessage + " is not provided"); } - if (functionConfig.getParallelism() <= 0) { + if (functionDetails.getParallelism() <= 0) { throw new IllegalArgumentException("Parallelism needs to be set to a positive number"); } - return functionConfig; + return functionDetails; } catch (IllegalArgumentException ex) { throw ex; } catch (Exception ex) { - throw new IllegalArgumentException("Invalid FunctionConfig"); + throw new IllegalArgumentException("Invalid FunctionDetails"); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java index 1d7a8b67e9b4a..d2347ff3b0128 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java @@ -47,10 +47,10 @@ public Response registerFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("data") FormDataContentDisposition fileDetail, - final @FormDataParam("functionConfig") String functionConfigJson) { + final @FormDataParam("functionDetails") String functionDetailsJson) { return functions.registerFunction( - tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson); + tenant, namespace, functionName, uploadedInputStream, fileDetail, functionDetailsJson); } @@ -62,10 +62,10 @@ public Response updateFunction(final @PathParam("tenant") String tenant, final @PathParam("functionName") String functionName, final @FormDataParam("data") InputStream uploadedInputStream, final @FormDataParam("data") FormDataContentDisposition fileDetail, - final @FormDataParam("functionConfig") String functionConfigJson) { + final @FormDataParam("functionDetails") String functionDetailsJson) { return functions.updateFunction( - tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson); + tenant, namespace, functionName, uploadedInputStream, fileDetail, functionDetailsJson); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java index eb37e77c23505..3d27fb4984b27 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionMetaDataManagerTest.java @@ -50,15 +50,15 @@ public void testListFunctions() throws PulsarClientException { mock(PulsarClient.class))); Map functionMetaDataMap1 = new HashMap<>(); - functionMetaDataMap1.put("func-1", Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder().setName("func-1")).build()); + functionMetaDataMap1.put("func-1", Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder().setName("func-1")).build()); functionMetaDataMap1.put("func-2", - Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder().setName("func-2")).build()); + Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder().setName("func-2")).build()); Map functionMetaDataInfoMap2 = new HashMap<>(); functionMetaDataInfoMap2.put("func-3", - Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder().setName("func-3")).build()); + Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder().setName("func-3")).build()); functionMetaDataManager.functionMetaDataMap.put("tenant-1", new HashMap<>()); @@ -89,7 +89,7 @@ public void updateFunction() throws PulsarClientException { mock(SchedulerManager.class), mock(PulsarClient.class))); Function.FunctionMetaData m1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1")).build(); + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1")).build(); Mockito.doReturn(null).when(functionMetaDataManager).submit(any(Request.ServiceRequest.class)); functionMetaDataManager.updateFunction(m1); @@ -126,7 +126,7 @@ public boolean matches(Object o) { mock(PulsarClient.class))); Map functionMetaDataMap = new HashMap<>(); Function.FunctionMetaData m2 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build(); functionMetaDataMap.put("func-1", m2); functionMetaDataManager.functionMetaDataMap.put("tenant-1", new HashMap<>()); @@ -145,7 +145,7 @@ public boolean matches(Object o) { Request.ServiceRequest.ServiceRequestType.UPDATE)) { return false; } - if (!serviceRequest.getFunctionMetaData().getFunctionConfig().equals(m2.getFunctionConfig())) { + if (!serviceRequest.getFunctionMetaData().getFunctionDetails().equals(m2.getFunctionDetails())) { return false; } if (serviceRequest.getFunctionMetaData().getVersion() != (version + 1)) { @@ -169,7 +169,7 @@ public void deregisterFunction() throws PulsarClientException { mock(SchedulerManager.class), mock(PulsarClient.class))); Function.FunctionMetaData m1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build(); Map functionMetaDataMap = new HashMap<>(); functionMetaDataMap.put("func-1", m1); @@ -190,7 +190,7 @@ public boolean matches(Object o) { Request.ServiceRequest.ServiceRequestType.DELETE)) { return false; } - if (!serviceRequest.getFunctionMetaData().getFunctionConfig().equals(m1.getFunctionConfig())) { + if (!serviceRequest.getFunctionMetaData().getFunctionDetails().equals(m1.getFunctionDetails())) { return false; } if (serviceRequest.getFunctionMetaData().getVersion() != (version + 1)) { @@ -251,7 +251,7 @@ public void processUpdateTest() throws PulsarClientException { // worker has no record of function Function.FunctionMetaData m1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build(); Request.ServiceRequest serviceRequest = Request.ServiceRequest.newBuilder() @@ -278,11 +278,11 @@ public void processUpdateTest() throws PulsarClientException { mock(PulsarClient.class))); Function.FunctionMetaData m3 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build(); functionMetaDataManager.setFunctionMetaData(m3); Function.FunctionMetaData outdated = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version - 1).build(); serviceRequest = Request.ServiceRequest.newBuilder() @@ -299,7 +299,7 @@ public void processUpdateTest() throws PulsarClientException { verify(schedulerManager, times(0)).schedule(); Function.FunctionMetaData outdated2 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build(); serviceRequest = Request.ServiceRequest.newBuilder() @@ -327,11 +327,11 @@ public void processUpdateTest() throws PulsarClientException { mock(PulsarClient.class))); Function.FunctionMetaData m4 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build(); functionMetaDataManager.setFunctionMetaData(m4); Function.FunctionMetaData m5 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version + 1).build(); serviceRequest = Request.ServiceRequest.newBuilder() @@ -364,11 +364,11 @@ public void processDeregister() throws PulsarClientException { mock(PulsarClient.class))); // worker has no record of function Function.FunctionMetaData test = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build(); functionMetaDataManager.setFunctionMetaData(test); Function.FunctionMetaData m1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build(); Request.ServiceRequest serviceRequest = Request.ServiceRequest.newBuilder() .setServiceRequestType(Request.ServiceRequest.ServiceRequestType.UPDATE) @@ -391,7 +391,7 @@ public void processDeregister() throws PulsarClientException { mock(PulsarClient.class))); functionMetaDataManager.setFunctionMetaData(test); Function.FunctionMetaData m2 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version).build(); functionMetaDataManager.setFunctionMetaData(m2); serviceRequest = Request.ServiceRequest.newBuilder() @@ -419,12 +419,12 @@ public void processDeregister() throws PulsarClientException { functionMetaDataManager.setFunctionMetaData(test); Function.FunctionMetaData m3 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version ).build(); functionMetaDataManager.setFunctionMetaData(m3); Function.FunctionMetaData m4 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1")).setVersion(version +1).build(); serviceRequest = Request.ServiceRequest.newBuilder() .setServiceRequestType(Request.ServiceRequest.ServiceRequestType.UPDATE) diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java index 3bca55c319885..690f474150720 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java @@ -55,7 +55,7 @@ public void init(Map conf) { } @Override - public void processRecord(InstanceCommunication.MetricsData record, Function.FunctionConfig functionConfig) { + public void processRecord(InstanceCommunication.MetricsData record, Function.FunctionDetails functionDetails) { } @@ -95,12 +95,12 @@ public void testProcessAssignmentUpdateAddFunctions() throws Exception { mock(MembershipManager.class) )); - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); Function.Assignment assignment1 = Function.Assignment.newBuilder() @@ -188,12 +188,12 @@ public void testProcessAssignmentUpdateDeleteFunctions() throws Exception { mock(MembershipManager.class) )); - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); Function.Assignment assignment1 = Function.Assignment.newBuilder() @@ -285,12 +285,12 @@ public void testProcessAssignmentUpdateModifyFunctions() throws Exception { mock(MembershipManager.class) )); - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); Function.Assignment assignment1 = Function.Assignment.newBuilder() diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionStatsGeneratorTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionStatsGeneratorTest.java index 5bdb81275afd1..7bd85e82a51ee 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionStatsGeneratorTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionStatsGeneratorTest.java @@ -71,8 +71,8 @@ public void testFunctionsStatsGenerate() { RuntimeSpawner runtimeSpawner = mock(RuntimeSpawner.class); doReturn(runtime).when(runtimeSpawner).getRuntime(); - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); Function.Instance instance = Function.Instance.newBuilder() diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java index 4fd72b992683b..14b56acd31731 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java @@ -46,7 +46,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -119,12 +118,12 @@ public void testCheckFailuresNoFailures() throws Exception { Mockito.doReturn(workerInfoList).when(membershipManager).getCurrentMembership(); - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); List metaDataList = new LinkedList<>(); @@ -183,12 +182,12 @@ public void testCheckFailuresSomeFailures() throws Exception { Mockito.doReturn(workerInfoList).when(membershipManager).getCurrentMembership(); - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder() + Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); List metaDataList = new LinkedList<>(); @@ -272,12 +271,12 @@ public void testCheckFailuresSomeUnassigned() throws Exception { Mockito.doReturn(workerInfoList).when(membershipManager).getCurrentMembership(); - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder().setParallelism(1) + Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder().setParallelism(1) .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); - Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig( - Function.FunctionConfig.newBuilder().setParallelism(1) + Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails( + Function.FunctionDetails.newBuilder().setParallelism(1) .setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build(); List metaDataList = new LinkedList<>(); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java index b115b1767a34a..64d693120f330 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/SchedulerManagerTest.java @@ -96,7 +96,7 @@ public void testSchedule() throws Exception { List functionMetaDataList = new LinkedList<>(); long version = 5; Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); functionMetaDataList.add(function1); @@ -140,7 +140,7 @@ public void testNothingNewToSchedule() throws Exception { List functionMetaDataList = new LinkedList<>(); long version = 5; Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); functionMetaDataList.add(function1); @@ -190,12 +190,12 @@ public void testAddingFunctions() throws Exception { List functionMetaDataList = new LinkedList<>(); long version = 5; Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); functionMetaDataList.add(function1); @@ -253,13 +253,13 @@ public void testDeletingFunctions() throws Exception { List functionMetaDataList = new LinkedList<>(); long version = 5; Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); // simulate function2 got removed Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); functionMetaDataList.add(function1); @@ -319,12 +319,12 @@ public void testScalingUp() throws Exception { List functionMetaDataList = new LinkedList<>(); long version = 5; Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); functionMetaDataList.add(function1); @@ -384,7 +384,7 @@ public void testScalingUp() throws Exception { doReturn(producer).when(pulsarClient).createProducer(any(), any()); Function.FunctionMetaData function2Scaled = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version) .build(); functionMetaDataList = new LinkedList<>(); @@ -428,12 +428,12 @@ public void testScalingDown() throws Exception { List functionMetaDataList = new LinkedList<>(); long version = 5; Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version) .build(); functionMetaDataList.add(function1); @@ -504,7 +504,7 @@ public void testScalingDown() throws Exception { doReturn(producer).when(pulsarClient).createProducer(any(), any()); Function.FunctionMetaData function2Scaled = Function.FunctionMetaData.newBuilder() - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); functionMetaDataList = new LinkedList<>(); @@ -539,13 +539,13 @@ public void testUpdate() throws Exception { long version = 5; Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar1")) - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version) .build(); Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder() .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar1")) - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version) .build(); functionMetaDataList.add(function1); @@ -617,7 +617,7 @@ public void testUpdate() throws Exception { Function.FunctionMetaData function2Updated = Function.FunctionMetaData.newBuilder() .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar2")) - .setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2") + .setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2") .setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version) .build(); functionMetaDataList = new LinkedList<>(); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java index 50c6b7b3660f9..5ae9e9932c6a5 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java @@ -47,7 +47,7 @@ import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.utils.DefaultSerDe; import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.Function.FunctionMetaData; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.Utils; @@ -295,40 +295,40 @@ private void testRegisterFunctionMissingArguments( Integer parallelism, String missingFieldName ) throws IOException { - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); + FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); if (tenant != null) { - functionConfigBuilder.setTenant(tenant); + functionDetailsBuilder.setTenant(tenant); } if (namespace != null) { - functionConfigBuilder.setNamespace(namespace); + functionDetailsBuilder.setNamespace(namespace); } if (function != null) { - functionConfigBuilder.setName(function); + functionDetailsBuilder.setName(function); } if (outputTopic != null) { - functionConfigBuilder.setOutput(outputTopic); + functionDetailsBuilder.setOutput(outputTopic); } if (inputTopic != null && inputSerdeClassName != null) { - functionConfigBuilder.putCustomSerdeInputs(inputTopic, inputSerdeClassName); + functionDetailsBuilder.putCustomSerdeInputs(inputTopic, inputSerdeClassName); } if (outputSerdeClassName != null) { - functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName); + functionDetailsBuilder.setOutputSerdeClassName(outputSerdeClassName); } if (className != null) { - functionConfigBuilder.setClassName(className); + functionDetailsBuilder.setClassName(className); } if (parallelism != null) { - functionConfigBuilder.setParallelism(parallelism); + functionDetailsBuilder.setParallelism(parallelism); } - FunctionConfig functionConfig = functionConfigBuilder.build(); + FunctionDetails functionDetails = functionDetailsBuilder.build(); Response response = resource.registerFunction( tenant, namespace, function, inputStream, details, - org.apache.pulsar.functions.utils.Utils.printJson(functionConfig)); + org.apache.pulsar.functions.utils.Utils.printJson(functionDetails)); assertEquals(Status.BAD_REQUEST.getStatusCode(), response.getStatus()); if (missingFieldName.equals("parallelism")) { @@ -339,7 +339,7 @@ private void testRegisterFunctionMissingArguments( } private Response registerDefaultFunction() throws IOException { - FunctionConfig functionConfig = FunctionConfig.newBuilder() + FunctionDetails functionDetails = FunctionDetails.newBuilder() .setTenant(tenant).setNamespace(namespace).setName(function) .setOutput(outputTopic).putCustomSerdeInputs(inputTopic, inputSerdeClassName) .setOutputSerdeClassName(outputSerdeClassName) @@ -351,7 +351,7 @@ private Response registerDefaultFunction() throws IOException { function, mockedInputStream, mockedFormData, - org.apache.pulsar.functions.utils.Utils.printJson(functionConfig)); + org.apache.pulsar.functions.utils.Utils.printJson(functionDetails)); } @Test @@ -615,40 +615,40 @@ private void testUpdateFunctionMissingArguments( Integer parallelism, String missingFieldName ) throws IOException { - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); + FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); if (tenant != null) { - functionConfigBuilder.setTenant(tenant); + functionDetailsBuilder.setTenant(tenant); } if (namespace != null) { - functionConfigBuilder.setNamespace(namespace); + functionDetailsBuilder.setNamespace(namespace); } if (function != null) { - functionConfigBuilder.setName(function); + functionDetailsBuilder.setName(function); } if (outputTopic != null) { - functionConfigBuilder.setOutput(outputTopic); + functionDetailsBuilder.setOutput(outputTopic); } if (inputTopic != null && inputSerdeClassName != null) { - functionConfigBuilder.putCustomSerdeInputs(inputTopic, inputSerdeClassName); + functionDetailsBuilder.putCustomSerdeInputs(inputTopic, inputSerdeClassName); } if (outputSerdeClassName != null) { - functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName); + functionDetailsBuilder.setOutputSerdeClassName(outputSerdeClassName); } if (className != null) { - functionConfigBuilder.setClassName(className); + functionDetailsBuilder.setClassName(className); } if (parallelism != null) { - functionConfigBuilder.setParallelism(parallelism); + functionDetailsBuilder.setParallelism(parallelism); } - FunctionConfig functionConfig = functionConfigBuilder.build(); + FunctionDetails functionDetails = functionDetailsBuilder.build(); Response response = resource.updateFunction( tenant, namespace, function, inputStream, details, - org.apache.pulsar.functions.utils.Utils.printJson(functionConfig)); + org.apache.pulsar.functions.utils.Utils.printJson(functionDetails)); assertEquals(Status.BAD_REQUEST.getStatusCode(), response.getStatus()); if (missingFieldName.equals("parallelism")) { @@ -659,7 +659,7 @@ private void testUpdateFunctionMissingArguments( } private Response updateDefaultFunction() throws IOException { - FunctionConfig functionConfig = FunctionConfig.newBuilder() + FunctionDetails functionDetails = FunctionDetails.newBuilder() .setTenant(tenant).setNamespace(namespace).setName(function) .setOutput(outputTopic).putCustomSerdeInputs(inputTopic, inputSerdeClassName) .setOutputSerdeClassName(outputSerdeClassName) @@ -671,7 +671,7 @@ private Response updateDefaultFunction() throws IOException { function, mockedInputStream, mockedFormData, - org.apache.pulsar.functions.utils.Utils.printJson(functionConfig)); + org.apache.pulsar.functions.utils.Utils.printJson(functionDetails)); } @Test @@ -933,19 +933,19 @@ public void testGetNotExistedFunction() throws IOException { public void testGetFunctionSuccess() throws Exception { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - FunctionConfig functionConfig = FunctionConfig.newBuilder() + FunctionDetails functionDetails = FunctionDetails.newBuilder() .setClassName(className) .putCustomSerdeInputs(inputTopic, inputSerdeClassName) .setOutputSerdeClassName(outputSerdeClassName) .setName(function) .setNamespace(namespace) - .setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATMOST_ONCE) + .setProcessingGuarantees(FunctionDetails.ProcessingGuarantees.ATMOST_ONCE) .setOutput(outputTopic) .setTenant(tenant) .setParallelism(parallelism).build(); FunctionMetaData metaData = FunctionMetaData.newBuilder() .setCreateTime(System.currentTimeMillis()) - .setFunctionConfig(functionConfig) + .setFunctionDetails(functionDetails) .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")) .setVersion(1234) .build(); @@ -954,7 +954,7 @@ public void testGetFunctionSuccess() throws Exception { Response response = getDefaultFunctionInfo(); assertEquals(Status.OK.getStatusCode(), response.getStatus()); assertEquals( - org.apache.pulsar.functions.utils.Utils.printJson(functionConfig), + org.apache.pulsar.functions.utils.Utils.printJson(functionDetails), response.getEntity()); }