Skip to content

Commit

Permalink
扩展Hbase等连接器及改进项目结构
Browse files Browse the repository at this point in the history
  • Loading branch information
aiwenmo committed Oct 18, 2021
1 parent 47355f4 commit 2ea1d40
Show file tree
Hide file tree
Showing 27 changed files with 498 additions and 28 deletions.
16 changes: 16 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,26 @@ DataLink 开源项目及社区正在建设,希望本项目可以帮助你更
```
config/ -- 配置文件
|- application.yml
extends/ -- 扩展
|- clickhouse-jdbc-0.2.6.jar
|- dlink-client-1.11.jar
|- dlink-client-1.13.jar
|- dlink-flink-shaded-hadoop-3-uber.jar
|- flink-sql-connector-hbase-1.4_2.11-1.12.5.jar
|- flink-sql-connector-hbase-2.2_2.11-1.12.5.jar
|- flink-sql-connector-kafka_2.11-1.12.5.jar
|- ojdbc8-12.2.0.1.jar
|- otj-pg-embedded-0.13.3.jar
|- postgresql-42.2.10.jar
lib/ -- 外部依赖及Connector
|- dlink-client-1.12.jar -- 必需
|- dlink-connector-jdbc.jar
|- dlink-function.jar
|- dlink-metadata-clickhouse.jar
|- dlink-metadata-mysql.jar
|- dlink-metadata-oracle.jar
|- dlink-metadata-postgresql.jar
plugins/
|- flink-connector-jdbc_2.11-1.12.5.jar
|- flink-csv-1.12.5.jar
|- flink-json-1.12.5.jar
Expand Down
7 changes: 6 additions & 1 deletion dlink-admin/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>dlink</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down Expand Up @@ -34,6 +34,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down Expand Up @@ -121,6 +125,7 @@
<artifactId>flink-connector-mysql-cdc</artifactId>
<version>1.3.0</version>
</dependency>-->

</dependencies>
<build>
<plugins>
Expand Down
2 changes: 1 addition & 1 deletion dlink-assembly/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>dlink</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
40 changes: 35 additions & 5 deletions dlink-assembly/src/main/assembly/package.xml
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,14 @@
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-client/dlink-client-1.11/target</directory>
<outputDirectory>plugins</outputDirectory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-client-1.11-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-client/dlink-client-1.13/target</directory>
<outputDirectory>plugins</outputDirectory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-client-1.13-${project.version}.jar</include>
</includes>
Expand All @@ -75,11 +75,12 @@
<include>dlink-connector-jdbc-1.12-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-connectors的flink-shaded-hadoop-3-uber文件放到打包目录extends下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-connectors/dlink-connector-jdbc-1.13/target</directory>
<outputDirectory>plugins</outputDirectory>
<directory>${project.parent.basedir}/dlink-connectors/dlink-flink-shaded-hadoop-3-uber/target</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-connector-jdbc-1.13-${project.version}.jar</include>
<include>dlink-flink-shaded-hadoop-3-uber.jar</include>
</includes>
</fileSet>

Expand Down Expand Up @@ -132,5 +133,34 @@
<include>dlink-metadata-postgresql-${project.version}.jar</include>
</includes>
</fileSet>

<!-- 将模块dlink-extends的常用jar文件放到打包目录/plugins下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-extends/target
</directory>
<outputDirectory>plugins</outputDirectory>
<includes>
<include>flink-csv-*.jar</include>
<include>flink-json-*.jar</include>
<include>flink-connector-jdbc_2.11-*.jar</include>
<include>mysql-connector-java-*.jar</include>
</includes>
</fileSet>

<!-- 将模块dlink-extends的jar文件放到打包目录/extends下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-extends/target
</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>flink-sql-connector-hbase-1.4_2.11-*.jar</include>
<include>flink-sql-connector-hbase-2.2_2.11-*.jar</include>
<include>flink-sql-connector-kafka_2.11-*.jar</include>
<include>postgresql-*.jar</include>
<include>otj-pg-embedded-*.jar</include>
<include>ojdbc8-*.jar</include>
<include>clickhouse-jdbc-*.jar</include>
</includes>
</fileSet>
</fileSets>
</assembly>
2 changes: 1 addition & 1 deletion dlink-client/dlink-client-1.11/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>dlink-client</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion dlink-client/dlink-client-1.12/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>dlink-client</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion dlink-client/dlink-client-1.13/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>dlink-client</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion dlink-client/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>dlink</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>

<modelVersion>4.0.0</modelVersion>
Expand Down
2 changes: 1 addition & 1 deletion dlink-common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>dlink</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion dlink-connectors/dlink-connector-jdbc-1.12/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>dlink-connectors</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion dlink-connectors/dlink-connector-jdbc-1.13/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>dlink-connectors</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>dlink-connectors</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dlink-flink-shaded-hadoop-3-uber</artifactId>
<build>
<finalName>${project.artifactId}</finalName>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer>
<resource>reference.conf</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</plugin>
</plugins>
</build>
<properties>
<java.version>1.8</java.version>
<scala.binary.version>2.11</scala.binary.version>
<flink.version>1.12.5</flink.version>
<maven.compiler.source>1.8</maven.compiler.source>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
</project>
105 changes: 105 additions & 0 deletions dlink-connectors/dlink-flink-shaded-hadoop-3-uber/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dlink-connectors</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>dlink-flink-shaded-hadoop-3-uber</artifactId>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>1.12.5</flink.version>
<scala.binary.version>2.11</scala.binary.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded-hadoop-3-uber</artifactId>
<version>3.1.1.7.2.9.0-173-9.0</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<!--打jar包-->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<finalName>${project.artifactId}</finalName>
</build>

</project>
3 changes: 2 additions & 1 deletion dlink-connectors/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@
<parent>
<artifactId>dlink</artifactId>
<groupId>com.dlink</groupId>
<version>0.3.1</version>
<version>0.3.2-SANPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
<modules>
<module>dlink-connector-jdbc-1.12</module>
<module>dlink-connector-jdbc-1.13</module>
<module>dlink-flink-shaded-hadoop-3-uber</module>
</modules>
<artifactId>dlink-connectors</artifactId>
</project>
Loading

0 comments on commit 2ea1d40

Please sign in to comment.