Skip to content

Package kafka 0.9 shaded jar to pinot-distribution #8569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiangfu0 merged 2 commits into from
Apr 21, 2022
Merged

Package kafka 0.9 shaded jar to pinot-distribution #8569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ae42710
Package kafka 0.9 shaded jar
xiangfu0 Apr 20, 2022
1db2b45
pick higher version of kafka startable
xiangfu0 Apr 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docker/images/pinot/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ RUN wget http://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz -O /
RUN git clone ${PINOT_GIT_URL} ${PINOT_BUILD_DIR} && \
cd ${PINOT_BUILD_DIR} && \
git checkout ${PINOT_BRANCH} && \
mvn install package -DskipTests -Pbin-dist -Pbuild-shaded-jar -Dkafka.version=${KAFKA_VERSION} -Djdk.version=${JDK_VERSION} -T1C && \
mvn install package -DskipTests -Pbin-dist -Pbuild-shaded-jar -Djdk.version=${JDK_VERSION} -T1C && \
mkdir -p ${PINOT_HOME}/configs && \
mkdir -p ${PINOT_HOME}/data && \
cp -r build/* ${PINOT_HOME}/. && \
Expand Down
9 changes: 8 additions & 1 deletion pinot-distribution/pinot-assembly.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,14 @@
</file>
<!-- Start Include Pinot Plugins-->
<!-- Start Include Pinot Stream Ingestion Plugins-->
<!-- Only Include Specified Kafka Version Plugin-->
<file>
<source>
${pinot.root}/pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/target/pinot-kafka-0.9-${project.version}-shaded.jar
</source>
<destName>
plugins/pinot-stream-ingestion/pinot-kafka-0.9/pinot-kafka-0.9-${project.version}-shaded.jar
</destName>
</file>
<file>
<source>
${pinot.root}/pinot-plugins/pinot-stream-ingestion/pinot-kafka-${kafka.version}/target/pinot-kafka-${kafka.version}-${project.version}-shaded.jar
Expand Down
86 changes: 86 additions & 0 deletions pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,4 +72,90 @@
<version>2.10.5</version>
</dependency>
</dependencies>
<profiles>
<profile>
<id>build-shaded-jar</id>
<activation>
<property>
<name>skipShade</name>
<value>!true</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>${phase.prop}</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
</transformers>
<!--
Usually in hadoop environment, there are multiple jars with different versions.
Most of the NoSuchMethodExceptions are caused by class loading conflicts.
Class relocation ensures the reference of certain packages/classes in Pinot code to
shaded libs, e.g. jackson or guava.
Ref: https://maven.apache.org/plugins/maven-shade-plugin/examples/class-relocation.html
-->
<relocations>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>shaded.com.google.common</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
<shadedPattern>shaded.com.fasterxml.jackson</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.http</pattern>
<shadedPattern>shaded.org.apache.http</shadedPattern>
</relocation>
<relocation>
<pattern>software.amazon</pattern>
<shadedPattern>shaded.software.amazon</shadedPattern>
</relocation>
<relocation>
<pattern>org.reflections</pattern>
<shadedPattern>shaded.org.reflections</shadedPattern>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>shaded.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.parquet</pattern>
<shadedPattern>shaded.org.apache.parquet</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.kafka</pattern>
<shadedPattern>shaded.kafka09.org.apache.kafka</shadedPattern>
</relocation>
<relocation>
<pattern>kafka</pattern>
<shadedPattern>shaded.kafka09.kafka</shadedPattern>
</relocation>
<relocation>
<pattern>scala</pattern>
<shadedPattern>shaded.scala_2_10</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
2 changes: 1 addition & 1 deletion pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ A stream plugin for another version of kafka, or another stream, can be added in

* How to build and release Pinot package with Kafka 2.x connector
```$xslt
mvn clean package -DskipTests -Pbin-dist -Dkafka.version=2.0
mvn clean package -DskipTests -Pbin-dist
```

* How to use Kafka 2.x connector
Expand Down
4 changes: 4 additions & 0 deletions pinot-plugins/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@
<pattern>org.apache.parquet</pattern>
<shadedPattern>shaded.org.apache.parquet</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.kafka</pattern>
<shadedPattern>shaded.org.apache.kafka</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
Expand Down
12 changes: 11 additions & 1 deletion pinot-tools/src/main/java/org/apache/pinot/tools/utils/KafkaStarterUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
package org.apache.pinot.tools.utils;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.util.ServiceLoader;
Expand Down Expand Up @@ -51,7 +53,15 @@ private KafkaStarterUtils() {
getKafkaConnectorPackageName() + ".KafkaStreamLevelConsumer";

private static String getKafkaConnectorPackageName() {
return ServiceLoader.load(StreamConsumerFactory.class).iterator().next().getClass().getPackage().getName();
Iterator<StreamConsumerFactory> iterator = ServiceLoader.load(StreamConsumerFactory.class).iterator();
List<String> streamConsumerFactoryList = new ArrayList<>();
while (iterator.hasNext()) {
streamConsumerFactoryList.add(iterator.next().getClass().getPackage().getName());
}
if (streamConsumerFactoryList.size() > 1) {
Collections.sort(streamConsumerFactoryList, Collections.reverseOrder());
}
return streamConsumerFactoryList.get(0);
}

public static final String KAFKA_JSON_MESSAGE_DECODER_CLASS_NAME =
Expand Down
24 changes: 0 additions & 24 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,30 +172,6 @@
</properties>

<profiles>
<profile>
<id>kafka-0.9</id>
<activation>
<property>
<name>kafka.version</name>
<value>0.9</value>
</property>
</activation>
<properties>
<kafka.version>0.9</kafka.version>
</properties>
</profile>
<profile>
<id>kafka-2.0</id>
<activation>
<property>
<name>kafka.version</name>
<value>2.0</value>
</property>
</activation>
<properties>
<kafka.version>2.0</kafka.version>
</properties>
</profile>
<profile>
<id>github-actions</id>
<activation>
Expand Down