Skip to content

Commit ed2207f

Browse files
dpcollins-googledpcollins-google
authored
fix: Various fixes for flink (#183)
* fix: Various fixes for flink This removes broken interfaces (those that push messages into flink DataStreams can't be serialized properly) and fixes assorted issues. * fix: Various fixes for flink This removes broken interfaces (those that push messages into flink DataStreams can't be serialized properly) and fixes assorted issues.
1 parent a1d06f0 commit ed2207f

30 files changed

+394
-395
lines changed

‎pom.xml‎

Lines changed: 167 additions & 156 deletions
Original file line numberDiff line numberDiff line change
@@ -1,159 +1,170 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<project xmlns="http://maven.apache.org/POM/4.0.0"
3-
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4-
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5-
<parent>
6-
<groupId>com.google.cloud</groupId>
7-
<artifactId>google-cloud-pubsublite-parent</artifactId>
8-
<version>1.9.4</version>
9-
</parent>
10-
<modelVersion>4.0.0</modelVersion>
11-
<packaging>jar</packaging>
12-
<artifactId>google-cloud-pubsublite-flink</artifactId>
13-
<version>0.1.0-SNAPSHOT</version>
14-
<properties>
15-
<flink.version>1.15.0</flink.version>
16-
<pubsublite.version>${project.parent.version}</pubsublite.version>
17-
</properties>
18-
<build>
19-
<extensions>
20-
<extension>
21-
<groupId>kr.motd.maven</groupId>
22-
<artifactId>os-maven-plugin</artifactId>
23-
<version>1.7.1</version>
24-
</extension>
25-
</extensions>
26-
<plugins>
27-
<plugin>
28-
<groupId>org.xolstice.maven.plugins</groupId>
29-
<artifactId>protobuf-maven-plugin</artifactId>
30-
<version>0.6.1</version>
31-
<executions>
32-
<execution>
33-
<goals>
34-
<goal>compile</goal>
35-
<goal>test-compile</goal>
36-
</goals>
37-
</execution>
38-
</executions>
39-
<configuration>
40-
<protocArtifact>com.google.protobuf:protoc:3.4.0:exe:${os.detected.classifier}</protocArtifact>
41-
</configuration>
42-
</plugin>
43-
</plugins>
44-
</build>
45-
<dependencies>
46-
<dependency>
47-
<groupId>com.google.auto.value</groupId>
48-
<artifactId>auto-value-annotations</artifactId>
49-
</dependency>
50-
<dependency>
51-
<groupId>com.google.protobuf</groupId>
52-
<artifactId>protobuf-java</artifactId>
53-
</dependency>
54-
<dependency>
55-
<groupId>com.google.cloud</groupId>
56-
<artifactId>google-cloud-core</artifactId>
57-
</dependency>
58-
<dependency>
59-
<groupId>com.google.cloud</groupId>
60-
<artifactId>google-cloud-pubsublite</artifactId>
61-
<version>${pubsublite.version}</version>
62-
</dependency>
63-
<dependency>
64-
<groupId>com.google.api.grpc</groupId>
65-
<artifactId>proto-google-cloud-pubsublite-v1</artifactId>
66-
<version>${pubsublite.version}</version>
67-
</dependency>
68-
<dependency>
69-
<groupId>com.google.api</groupId>
70-
<artifactId>api-common</artifactId>
71-
</dependency>
72-
<dependency>
73-
<groupId>com.google.errorprone</groupId>
74-
<artifactId>error_prone_annotations</artifactId>
75-
</dependency>
76-
<dependency>
77-
<groupId>com.google.code.findbugs</groupId>
78-
<artifactId>jsr305</artifactId>
79-
</dependency>
80-
<dependency>
81-
<groupId>com.google.guava</groupId>
82-
<artifactId>guava</artifactId>
83-
</dependency>
84-
<dependency>
85-
<groupId>com.google.api</groupId>
86-
<artifactId>gax</artifactId>
87-
</dependency>
88-
<dependency>
89-
<groupId>org.slf4j</groupId>
90-
<artifactId>slf4j-api</artifactId>
91-
<version>2.0.6</version>
92-
</dependency>
93-
<dependency>
94-
<groupId>org.apache.flink</groupId>
95-
<artifactId>flink-core</artifactId>
96-
<version>${flink.version}</version>
97-
</dependency>
98-
<dependency>
99-
<groupId>org.apache.flink</groupId>
100-
<artifactId>flink-connector-base</artifactId>
101-
<version>${flink.version}</version>
102-
</dependency>
103-
<dependency>
104-
<groupId>org.apache.flink</groupId>
105-
<artifactId>flink-metrics-core</artifactId>
106-
<version>${flink.version}</version>
107-
</dependency>
108-
<dependency>
109-
<groupId>org.apache.flink</groupId>
110-
<artifactId>flink-streaming-java</artifactId>
111-
<version>${flink.version}</version>
112-
</dependency>
113-
<dependency>
114-
<groupId>org.apache.flink</groupId>
115-
<artifactId>flink-runtime</artifactId>
116-
<version>${flink.version}</version>
117-
</dependency>
118-
<dependency>
119-
<groupId>junit</groupId>
120-
<artifactId>junit</artifactId>
121-
<scope>test</scope>
122-
</dependency>
123-
<dependency>
124-
<groupId>com.google.truth</groupId>
125-
<artifactId>truth</artifactId>
126-
<scope>test</scope>
127-
</dependency>
128-
<dependency>
129-
<groupId>org.mockito</groupId>
130-
<artifactId>mockito-core</artifactId>
131-
<scope>test</scope>
132-
</dependency>
133-
<dependency>
134-
<groupId>org.apache.flink</groupId>
135-
<artifactId>flink-connector-test-utils</artifactId>
136-
<version>${flink.version}</version>
137-
<scope>test</scope>
138-
</dependency>
139-
<dependency>
140-
<groupId>org.apache.flink</groupId>
141-
<artifactId>flink-test-utils-junit</artifactId>
142-
<version>${flink.version}</version>
143-
<scope>test</scope>
144-
</dependency>
145-
<dependency>
146-
<groupId>org.apache.flink</groupId>
147-
<artifactId>flink-test-utils</artifactId>
148-
<version>${flink.version}</version>
149-
<scope>test</scope>
150-
</dependency>
151-
<dependency>
152-
<groupId>org.apache.flink</groupId>
153-
<artifactId>flink-runtime</artifactId>
154-
<version>${flink.version}</version>
155-
<type>test-jar</type>
156-
<scope>test</scope>
157-
</dependency>
158-
</dependencies>
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<groupId>com.google.cloud</groupId>
7+
<artifactId>google-cloud-pubsublite-parent</artifactId>
8+
<version>1.9.4</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
<packaging>jar</packaging>
12+
<artifactId>google-cloud-pubsublite-flink</artifactId>
13+
<version>0.1.0-SNAPSHOT</version>
14+
<properties>
15+
<flink.version>1.15.0</flink.version>
16+
<pubsublite.version>${project.parent.version}</pubsublite.version>
17+
</properties>
18+
<build>
19+
<extensions>
20+
<extension>
21+
<groupId>kr.motd.maven</groupId>
22+
<artifactId>os-maven-plugin</artifactId>
23+
<version>1.7.1</version>
24+
</extension>
25+
</extensions>
26+
<plugins>
27+
<plugin>
28+
<groupId>org.xolstice.maven.plugins</groupId>
29+
<artifactId>protobuf-maven-plugin</artifactId>
30+
<version>0.6.1</version>
31+
<executions>
32+
<execution>
33+
<goals>
34+
<goal>compile</goal>
35+
<goal>test-compile</goal>
36+
</goals>
37+
</execution>
38+
</executions>
39+
<configuration>
40+
<protocArtifact>
41+
com.google.protobuf:protoc:3.4.0:exe:${os.detected.classifier}
42+
</protocArtifact>
43+
</configuration>
44+
</plugin>
45+
</plugins>
46+
</build>
47+
<dependencies>
48+
<dependency>
49+
<groupId>com.google.auto.value</groupId>
50+
<artifactId>auto-value-annotations</artifactId>
51+
</dependency>
52+
<dependency>
53+
<groupId>com.google.protobuf</groupId>
54+
<artifactId>protobuf-java</artifactId>
55+
</dependency>
56+
<dependency>
57+
<groupId>com.google.protobuf</groupId>
58+
<artifactId>protobuf-java-util</artifactId>
59+
</dependency>
60+
<dependency>
61+
<groupId>com.google.cloud</groupId>
62+
<artifactId>google-cloud-core</artifactId>
63+
</dependency>
64+
<dependency>
65+
<groupId>com.google.cloud</groupId>
66+
<artifactId>google-cloud-pubsublite</artifactId>
67+
<version>${pubsublite.version}</version>
68+
</dependency>
69+
<dependency>
70+
<groupId>com.google.api.grpc</groupId>
71+
<artifactId>proto-google-cloud-pubsublite-v1</artifactId>
72+
<version>${pubsublite.version}</version>
73+
</dependency>
74+
<dependency>
75+
<groupId>com.google.api</groupId>
76+
<artifactId>api-common</artifactId>
77+
</dependency>
78+
<dependency>
79+
<groupId>com.google.errorprone</groupId>
80+
<artifactId>error_prone_annotations</artifactId>
81+
</dependency>
82+
<dependency>
83+
<groupId>com.google.code.findbugs</groupId>
84+
<artifactId>jsr305</artifactId>
85+
</dependency>
86+
<dependency>
87+
<groupId>com.google.guava</groupId>
88+
<artifactId>guava</artifactId>
89+
</dependency>
90+
<dependency>
91+
<groupId>com.google.api</groupId>
92+
<artifactId>gax</artifactId>
93+
</dependency>
94+
<dependency>
95+
<groupId>org.slf4j</groupId>
96+
<artifactId>slf4j-api</artifactId>
97+
<version>2.0.6</version>
98+
</dependency>
99+
<dependency>
100+
<groupId>org.apache.flink</groupId>
101+
<artifactId>flink-core</artifactId>
102+
<version>${flink.version}</version>
103+
<scope>provided</scope>
104+
</dependency>
105+
<dependency>
106+
<groupId>org.apache.flink</groupId>
107+
<artifactId>flink-connector-base</artifactId>
108+
<version>${flink.version}</version>
109+
<scope>provided</scope>
110+
</dependency>
111+
<dependency>
112+
<groupId>org.apache.flink</groupId>
113+
<artifactId>flink-metrics-core</artifactId>
114+
<version>${flink.version}</version>
115+
<scope>provided</scope>
116+
</dependency>
117+
<dependency>
118+
<groupId>org.apache.flink</groupId>
119+
<artifactId>flink-streaming-java</artifactId>
120+
<version>${flink.version}</version>
121+
<scope>provided</scope>
122+
</dependency>
123+
<dependency>
124+
<groupId>org.apache.flink</groupId>
125+
<artifactId>flink-runtime</artifactId>
126+
<version>${flink.version}</version>
127+
<scope>provided</scope>
128+
</dependency>
129+
<dependency>
130+
<groupId>junit</groupId>
131+
<artifactId>junit</artifactId>
132+
<scope>test</scope>
133+
</dependency>
134+
<dependency>
135+
<groupId>com.google.truth</groupId>
136+
<artifactId>truth</artifactId>
137+
<scope>test</scope>
138+
</dependency>
139+
<dependency>
140+
<groupId>org.mockito</groupId>
141+
<artifactId>mockito-core</artifactId>
142+
<scope>test</scope>
143+
</dependency>
144+
<dependency>
145+
<groupId>org.apache.flink</groupId>
146+
<artifactId>flink-connector-test-utils</artifactId>
147+
<version>${flink.version}</version>
148+
<scope>test</scope>
149+
</dependency>
150+
<dependency>
151+
<groupId>org.apache.flink</groupId>
152+
<artifactId>flink-test-utils-junit</artifactId>
153+
<version>${flink.version}</version>
154+
<scope>test</scope>
155+
</dependency>
156+
<dependency>
157+
<groupId>org.apache.flink</groupId>
158+
<artifactId>flink-test-utils</artifactId>
159+
<version>${flink.version}</version>
160+
<scope>test</scope>
161+
</dependency>
162+
<dependency>
163+
<groupId>org.apache.flink</groupId>
164+
<artifactId>flink-runtime</artifactId>
165+
<version>${flink.version}</version>
166+
<type>test-jar</type>
167+
<scope>test</scope>
168+
</dependency>
169+
</dependencies>
159170
</project>

‎src/main/java/com/google/cloud/pubsublite/flink/MessageTimestampExtractor.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,21 +16,21 @@
1616
package com.google.cloud.pubsublite.flink;
1717

1818
import com.google.cloud.Timestamp;
19-
import com.google.cloud.pubsublite.SequencedMessage;
19+
import com.google.cloud.pubsublite.proto.SequencedMessage;
2020
import java.io.Serializable;
2121
import java.time.Instant;
2222

2323
public interface MessageTimestampExtractor extends Serializable {
2424
static MessageTimestampExtractor publishTimeExtractor() {
25-
return m -> Timestamp.fromProto(m.publishTime()).toDate().toInstant();
25+
return m -> Timestamp.fromProto(m.getPublishTime()).toDate().toInstant();
2626
}
2727

2828
static MessageTimestampExtractor eventTimeExtractor() {
2929
return m -> {
30-
if (m.message().eventTime().isPresent()) {
31-
return Timestamp.fromProto(m.message().eventTime().get()).toDate().toInstant();
30+
if (m.getMessage().hasEventTime()) {
31+
return Timestamp.fromProto(m.getMessage().getEventTime()).toDate().toInstant();
3232
}
33-
return Timestamp.fromProto(m.publishTime()).toDate().toInstant();
33+
return Timestamp.fromProto(m.getPublishTime()).toDate().toInstant();
3434
};
3535
}
3636

0 commit comments

Comments
 (0)