Skip to content

Commit

Permalink
Provide a distribution package for pulsar functions (apache#263)
Browse files Browse the repository at this point in the history 
- the package will be generated under `dist/target/`
  • Loading branch information
@sijie
sijie committed Mar 4, 2018
1 parent 9114ddc commit e5ed2ed
Show file tree
Hide file tree
Showing 7 changed files with 294 additions and 14 deletions.
25 changes: 12 additions & 13 deletions pulsar-functions/bin/pulsar
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,45 +45,44 @@ else
fi

# exclude tests jar
RELEASE_JAR=`ls $PULSAR_HOME/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1`
RELEASE_JAR=`ls $PULSAR_HOME/lib/*pulsar-functions-worker*.jar 2> /dev/null | grep -v tests | tail -1`
if [ $? == 0 ]; then
PULSAR_JAR=$RELEASE_JAR
fi

# exclude tests jar
BUILT_JAR=`ls $PULSAR_HOME/worker/target/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1`
if [ $? != 0 ] && [ ! -e "$PULSAR_JAR" ]; then
echo "\nCouldn't find pulsar jar.";
echo "Make sure you've run 'mvn package'\n";
exit 1;
elif [ -e "$BUILT_JAR" ]; then
if [ -z "$PULSAR_JAR" ]; then
BUILT_JAR=`ls $PULSAR_HOME/worker/target/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1`
if [ -z "${BUILT_JAR}" ]; then
echo "\nCouldn't find pulsar jar.";
echo "Make sure you've run 'mvn package'\n";
exit 1;
fi
PULSAR_JAR=$BUILT_JAR
fi

# find the java instance location
if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then
# didn't find a released jar, then search the built jar
BUILT_JAVA_INSTANCE_JAR="${PULSAR_HOME}/runtime/target/java-instance.jar"
if [ -f "${BUILT_JAVA_INSTANCE_JAR}" ]; then
JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR}
else
if [ -z "${BUILT_JAVA_INSTANCE_JAR}" ]; then
echo "\nCouldn't find pulsar java instance jar.";
echo "Make sure you've run 'mvn package'\n";
exit 1;
fi
JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR}
fi

# find the python instance location
if [ ! -f "${PY_INSTANCE_FILE}" ]; then
# didn't find a released python instance, then search the built python instance
BUILT_PY_INSTANCE_FILE="${PULSAR_HOME}/runtime/target/python-instance/python_instance_main.py"
if [ -f "${BUILT_PY_INSTANCE_FILE}" ]; then
PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE}
else
if [ -z "${BUILT_PY_INSTANCE_FILE}" ]; then
echo "\nCouldn't find pulsar python instance.";
echo "Make sure you've run 'mvn package'\n";
exit 1;
fi
PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE}
fi

pulsar_help() {
Expand Down
2 changes: 1 addition & 1 deletion pulsar-functions/bin/pulsar-functions
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ add_maven_deps_to_classpath() {
}

if [ -d "$PULSAR_HOME/lib" ]; then
PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*"
PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*"
else
add_maven_deps_to_classpath
fi
Expand Down
104 changes: 104 additions & 0 deletions pulsar-functions/dist/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-functions</artifactId>
<version>2.0.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>

<artifactId>pulsar-functions-dist</artifactId>
<name>Pulsar Functions :: Distribution</name>
<packaging>pom</packaging>

<dependencies>
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-functions-worker</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-functions-cli</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-functions-api-examples</artifactId>
<version>${project.version}</version>
<!-- make sure the api examples are compiled before assembly -->
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>

</dependencies>

<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>functions-dist-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<attach>true</attach>
<tarLongFileMode>posix</tarLongFileMode>
<finalName>apache-pulsar-functions-${project.version}</finalName>
<descriptors>
<descriptor>src/assemble/bin.xml</descriptor>
<descriptor>src/assemble/src.xml</descriptor>
</descriptors>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
83 changes: 83 additions & 0 deletions pulsar-functions/dist/src/assemble/bin.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>bin</id>
<formats>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<moduleSets>
<moduleSet>
<useAllReactorProjects>true</useAllReactorProjects>
</moduleSet>
</moduleSets>
<fileSets>
<fileSet>
<directory>${basedir}/../conf</directory>
</fileSet>
<fileSet>
<directory>${basedir}/../bin</directory>
<fileMode>755</fileMode>
</fileSet>
<fileSet>
<directory>${basedir}/../runtime/target/python-instance</directory>
<outputDirectory>instances</outputDirectory>
</fileSet>
</fileSets>
<files>
<file>
<source>${basedir}/../runtime/target/java-instance.jar</source>
<destName>java-instance.jar</destName>
<outputDirectory>instances</outputDirectory>
</file>
<file>
<source>${basedir}/../java-examples/target/pulsar-functions-api-examples.jar</source>
<destName>api-examples.jar</destName>
<outputDirectory>examples</outputDirectory>
</file>
</files>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
<scope>compile</scope>
<useProjectArtifact>false</useProjectArtifact>
<!-- Include 'groupId' in the dependencies Jar names to better identify
the provenance of the jar -->
<outputFileNameMapping>${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension}</outputFileNameMapping>

<excludes>
<!-- All these dependencies are already included in netty-all -->
<exclude>io.netty:netty-common</exclude>
<exclude>io.netty:netty-resolver</exclude>
<exclude>io.netty:netty-buffer</exclude>
<exclude>io.netty:netty-codec-http</exclude>
<exclude>io.netty:netty-codec</exclude>
<exclude>io.netty:netty-transport</exclude>
<exclude>io.netty:netty-handler</exclude>
<exclude>io.netty:netty-codec-http</exclude>
</excludes>
</dependencySet>
</dependencySets>
</assembly>
92 changes: 92 additions & 0 deletions pulsar-functions/dist/src/assemble/src.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>src</id>
<formats>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>..</directory>
<useDefaultExcludes>true</useDefaultExcludes>
<includes>
<include>**/README.md</include>
<include>**/DISCLAIMER</include>
<include>**/LICENSE</include>
<include>**/NOTICE</include>
<include>**/pom.xml</include>
<include>**/src/**</include>
<include>**/pulsar-client-cpp/**</include>
<include>**/conf/**</include>
<include>**/bin/**</include>
<include>**/*.txt</include>
<include>docker/**</include>
<include>dashboard/**</include>
<include>deployment/**</include>
</includes>
<excludes>
<exclude>.git/**</exclude>
<exclude>**/.gitignore</exclude>
<exclude>**/.svn</exclude>
<exclude>**/*.iws</exclude>
<exclude>**/*.ipr</exclude>
<exclude>**/*.iml</exclude>
<exclude>**/*.cbp</exclude>
<exclude>**/*.pyc</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/.settings</exclude>
<exclude>**/target/**</exclude>
<exclude>**/CMakeFiles/**</exclude>
<exclude>**/CMakeCache.txt</exclude>
<exclude>**/cmake_install.cmake</exclude>
<exclude>pulsar-client-cpp/**/Makefile</exclude>
<exclude>pulsar-client-cpp/tests/main</exclude>
<exclude>pulsar-client-cpp/examples/SampleAsyncProducer</exclude>
<exclude>pulsar-client-cpp/examples/SampleConsumer</exclude>
<exclude>pulsar-client-cpp/examples/SampleConsumerListener</exclude>
<exclude>pulsar-client-cpp/examples/SampleProducer</exclude>
<exclude>pulsar-client-cpp/perf/perfProducer</exclude>
<exclude>pulsar-client-cpp/perf/perfConsumer</exclude>

<exclude>**/python/dist/**</exclude>
<exclude>**/python/wheelhouse/**</exclude>
<exclude>**/python/MANIFEST</exclude>
<exclude>**/*.egg-info/**</exclude>
<!-- until the code that does this is fixed -->
<exclude>**/*.log</exclude>
<exclude>**/build/**</exclude>
<exclude>**/file:/**</exclude>
<exclude>**/SecurityAuth.audit*</exclude>
<exclude>**/site/**</exclude>
<exclude>**/.idea/**</exclude>
<exclude>**/*.a</exclude>
<exclude>**/*.so</exclude>
<exclude>**/*.so.*</exclude>
<exclude>**/*.dylib</exclude>
</excludes>
</fileSet>
</fileSets>
</assembly>
1 change: 1 addition & 0 deletions pulsar-functions/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
<module>cli</module>
<module>java-examples</module>
<module>api-composition-java</module>
<module>dist</module>
</modules>

<dependencyManagement>
Expand Down
1 change: 1 addition & 0 deletions ...ctions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetadataSetup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public static URI setupFunctionMetadata(WorkerConfig workerConfig)
admin.clusters().getClusters();
break;
} catch (PulsarAdminException e) {
log.warn("Failed to retrieve clusters from pulsar service", e);
log.warn("Retry to connect to Pulsar service at {}", workerConfig.getPulsarWebServiceUrl());
if (retries >= maxRetries) {
log.error("Failed to connect to Pulsar service at {} after {} attempts",
Expand Down

0 comments on commit e5ed2ed

Please sign in to comment.