Skip to content

Commit 95460a1

Browse files
rafael-tellesAbner Eduardo FerreirajduojcralmeidaJrJuscelino
authored and
kou
committed
ARROW-14421: [C++] Implement Flight SQL
Implement Flight SQL in C++ and Java, and add the protocol definitions. This is a combination of multiple pull requests, merged into one branch before merging into master. Closes apache#10906 (ARROW-12922). Closes apache#11507 (ARROW-14421). Closes apache#11989 (ARROW-15112). Closes apache#12021 (ARROW-15187). Closes apache#12035 (ARROW-15198). Closes apache#12013 from apache/flight-sql Lead-authored-by: Rafael Telles <rafael@telles.dev> Co-authored-by: Abner Eduardo Ferreira <abenaru@protonmail.ch> Co-authored-by: James Duong <duong.james@gmail.com> Co-authored-by: Jose Almeida <almeidajcr90@gmail.com> Co-authored-by: Juscelino Junior <juscelinojunior@id.uff.br> Co-authored-by: Kyle Porter <kporter@dremio.com> Co-authored-by: Ryan Nicholson <rnicholson@dremio.com> Co-authored-by: Vinicius Fraga <sxvinifp@gmail.com> Co-authored-by: tifflhl <tiffanylamhl@gmail.com> Signed-off-by: David Li <li.davidm96@gmail.com>
1 parent dd49fdf commit 95460a1

File tree

32 files changed

+6352
-98
lines changed

32 files changed

+6352
-98
lines changed

‎adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java‎

Lines changed: 8 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,12 @@
1717

1818
package org.apache.arrow.adapter.jdbc;
1919

20-
import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE;
21-
import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE;
22-
23-
import java.sql.Types;
2420
import java.util.Calendar;
2521
import java.util.Map;
2622
import java.util.function.Function;
2723

2824
import org.apache.arrow.memory.BufferAllocator;
2925
import org.apache.arrow.util.Preconditions;
30-
import org.apache.arrow.vector.types.DateUnit;
31-
import org.apache.arrow.vector.types.TimeUnit;
3226
import org.apache.arrow.vector.types.pojo.ArrowType;
3327

3428
/**
@@ -55,16 +49,14 @@
5549
*/
5650
public final class JdbcToArrowConfig {
5751

52+
public static final int DEFAULT_TARGET_BATCH_SIZE = 1024;
53+
public static final int NO_LIMIT_BATCH_SIZE = -1;
5854
private final Calendar calendar;
5955
private final BufferAllocator allocator;
6056
private final boolean includeMetadata;
6157
private final boolean reuseVectorSchemaRoot;
6258
private final Map<Integer, JdbcFieldInfo> arraySubTypesByColumnIndex;
6359
private final Map<String, JdbcFieldInfo> arraySubTypesByColumnName;
64-
65-
public static final int DEFAULT_TARGET_BATCH_SIZE = 1024;
66-
public static final int NO_LIMIT_BATCH_SIZE = -1;
67-
6860
/**
6961
* The maximum rowCount to read each time when partially convert data.
7062
* Default value is 1024 and -1 means disable partial read.
@@ -82,7 +74,7 @@ public final class JdbcToArrowConfig {
8274
/**
8375
* Constructs a new configuration from the provided allocator and calendar. The <code>allocator</code>
8476
* is used when constructing the Arrow vectors from the ResultSet, and the calendar is used to define
85-
* Arrow Timestamp fields, and to read time-based fields from the JDBC <code>ResultSet</code>.
77+
* Arrow Timestamp fields, and to read time-based fields from the JDBC <code>ResultSet</code>.
8678
*
8779
* @param allocator The memory allocator to construct the Arrow vectors with.
8880
* @param calendar The calendar to use when constructing Timestamp fields and reading time-based results.
@@ -99,7 +91,7 @@ public final class JdbcToArrowConfig {
9991
/**
10092
* Constructs a new configuration from the provided allocator and calendar. The <code>allocator</code>
10193
* is used when constructing the Arrow vectors from the ResultSet, and the calendar is used to define
102-
* Arrow Timestamp fields, and to read time-based fields from the JDBC <code>ResultSet</code>.
94+
* Arrow Timestamp fields, and to read time-based fields from the JDBC <code>ResultSet</code>.
10395
*
10496
* @param allocator The memory allocator to construct the Arrow vectors with.
10597
* @param calendar The calendar to use when constructing Timestamp fields and reading time-based results.
@@ -134,6 +126,8 @@ public final class JdbcToArrowConfig {
134126
* <li>TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, calendar timezone)</li>
135127
* <li>CLOB --> ArrowType.Utf8</li>
136128
* <li>BLOB --> ArrowType.Binary</li>
129+
* <li>ARRAY --> ArrowType.List</li>
130+
* <li>STRUCT --> ArrowType.Struct</li>
137131
* <li>NULL --> ArrowType.Null</li>
138132
* </ul>
139133
*/
@@ -157,64 +151,7 @@ public final class JdbcToArrowConfig {
157151

158152
// set up type converter
159153
this.jdbcToArrowTypeConverter = jdbcToArrowTypeConverter != null ? jdbcToArrowTypeConverter :
160-
fieldInfo -> {
161-
final String timezone;
162-
if (calendar != null) {
163-
timezone = calendar.getTimeZone().getID();
164-
} else {
165-
timezone = null;
166-
}
167-
168-
switch (fieldInfo.getJdbcType()) {
169-
case Types.BOOLEAN:
170-
case Types.BIT:
171-
return new ArrowType.Bool();
172-
case Types.TINYINT:
173-
return new ArrowType.Int(8, true);
174-
case Types.SMALLINT:
175-
return new ArrowType.Int(16, true);
176-
case Types.INTEGER:
177-
return new ArrowType.Int(32, true);
178-
case Types.BIGINT:
179-
return new ArrowType.Int(64, true);
180-
case Types.NUMERIC:
181-
case Types.DECIMAL:
182-
int precision = fieldInfo.getPrecision();
183-
int scale = fieldInfo.getScale();
184-
return new ArrowType.Decimal(precision, scale, 128);
185-
case Types.REAL:
186-
case Types.FLOAT:
187-
return new ArrowType.FloatingPoint(SINGLE);
188-
case Types.DOUBLE:
189-
return new ArrowType.FloatingPoint(DOUBLE);
190-
case Types.CHAR:
191-
case Types.NCHAR:
192-
case Types.VARCHAR:
193-
case Types.NVARCHAR:
194-
case Types.LONGVARCHAR:
195-
case Types.LONGNVARCHAR:
196-
case Types.CLOB:
197-
return new ArrowType.Utf8();
198-
case Types.DATE:
199-
return new ArrowType.Date(DateUnit.DAY);
200-
case Types.TIME:
201-
return new ArrowType.Time(TimeUnit.MILLISECOND, 32);
202-
case Types.TIMESTAMP:
203-
return new ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone);
204-
case Types.BINARY:
205-
case Types.VARBINARY:
206-
case Types.LONGVARBINARY:
207-
case Types.BLOB:
208-
return new ArrowType.Binary();
209-
case Types.ARRAY:
210-
return new ArrowType.List();
211-
case Types.NULL:
212-
return new ArrowType.Null();
213-
default:
214-
// no-op, shouldn't get here
215-
return null;
216-
}
217-
};
154+
jdbcFieldInfo -> JdbcToArrowUtils.getArrowTypeFromJdbcType(jdbcFieldInfo, calendar);
218155
}
219156

220157
/**
@@ -230,6 +167,7 @@ public Calendar getCalendar() {
230167

231168
/**
232169
* The Arrow memory allocator.
170+
*
233171
* @return the allocator.
234172
*/
235173
public BufferAllocator getAllocator() {

‎adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,18 @@
1717

1818
package org.apache.arrow.adapter.jdbc;
1919

20+
import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE;
21+
import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE;
22+
2023
import java.io.IOException;
2124
import java.sql.Date;
25+
import java.sql.ParameterMetaData;
2226
import java.sql.ResultSet;
2327
import java.sql.ResultSetMetaData;
2428
import java.sql.SQLException;
2529
import java.sql.Time;
2630
import java.sql.Timestamp;
31+
import java.sql.Types;
2732
import java.util.ArrayList;
2833
import java.util.Calendar;
2934
import java.util.HashMap;
@@ -70,6 +75,8 @@
7075
import org.apache.arrow.vector.VarCharVector;
7176
import org.apache.arrow.vector.VectorSchemaRoot;
7277
import org.apache.arrow.vector.complex.ListVector;
78+
import org.apache.arrow.vector.types.DateUnit;
79+
import org.apache.arrow.vector.types.TimeUnit;
7380
import org.apache.arrow.vector.types.pojo.ArrowType;
7481
import org.apache.arrow.vector.types.pojo.Field;
7582
import org.apache.arrow.vector.types.pojo.FieldType;
@@ -106,6 +113,101 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar
106113
return jdbcToArrowSchema(rsmd, new JdbcToArrowConfig(new RootAllocator(0), calendar));
107114
}
108115

116+
/**
117+
* Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}.
118+
*
119+
* @param parameterMetaData The ResultSetMetaData containing the results, to read the JDBC metadata from.
120+
* @param calendar The calendar to use the time zone field of, to construct Timestamp fields from.
121+
* @return {@link Schema}
122+
* @throws SQLException on error
123+
*/
124+
public static Schema jdbcToArrowSchema(final ParameterMetaData parameterMetaData, final Calendar calendar)
125+
throws SQLException {
126+
Preconditions.checkNotNull(calendar, "Calendar object can't be null");
127+
Preconditions.checkNotNull(parameterMetaData);
128+
final List<Field> parameterFields = new ArrayList<>(parameterMetaData.getParameterCount());
129+
for (int parameterCounter = 1; parameterCounter <= parameterMetaData.getParameterCount();
130+
parameterCounter++) {
131+
final int jdbcDataType = parameterMetaData.getParameterType(parameterCounter);
132+
final int jdbcIsNullable = parameterMetaData.isNullable(parameterCounter);
133+
final boolean arrowIsNullable = jdbcIsNullable != ParameterMetaData.parameterNoNulls;
134+
final int precision = parameterMetaData.getPrecision(parameterCounter);
135+
final int scale = parameterMetaData.getScale(parameterCounter);
136+
final ArrowType arrowType = getArrowTypeFromJdbcType(new JdbcFieldInfo(jdbcDataType, precision, scale), calendar);
137+
final FieldType fieldType = new FieldType(arrowIsNullable, arrowType, /*dictionary=*/null);
138+
parameterFields.add(new Field(null, fieldType, null));
139+
}
140+
141+
return new Schema(parameterFields);
142+
}
143+
144+
/**
145+
* Converts the provided JDBC type to its respective {@link ArrowType} counterpart.
146+
*
147+
* @param fieldInfo the {@link JdbcFieldInfo} with information about the original JDBC type.
148+
* @param calendar the {@link Calendar} to use for datetime data types.
149+
* @return a new {@link ArrowType}.
150+
*/
151+
public static ArrowType getArrowTypeFromJdbcType(final JdbcFieldInfo fieldInfo, final Calendar calendar) {
152+
switch (fieldInfo.getJdbcType()) {
153+
case Types.BOOLEAN:
154+
case Types.BIT:
155+
return new ArrowType.Bool();
156+
case Types.TINYINT:
157+
return new ArrowType.Int(8, true);
158+
case Types.SMALLINT:
159+
return new ArrowType.Int(16, true);
160+
case Types.INTEGER:
161+
return new ArrowType.Int(32, true);
162+
case Types.BIGINT:
163+
return new ArrowType.Int(64, true);
164+
case Types.NUMERIC:
165+
case Types.DECIMAL:
166+
int precision = fieldInfo.getPrecision();
167+
int scale = fieldInfo.getScale();
168+
return new ArrowType.Decimal(precision, scale, 128);
169+
case Types.REAL:
170+
case Types.FLOAT:
171+
return new ArrowType.FloatingPoint(SINGLE);
172+
case Types.DOUBLE:
173+
return new ArrowType.FloatingPoint(DOUBLE);
174+
case Types.CHAR:
175+
case Types.NCHAR:
176+
case Types.VARCHAR:
177+
case Types.NVARCHAR:
178+
case Types.LONGVARCHAR:
179+
case Types.LONGNVARCHAR:
180+
case Types.CLOB:
181+
return new ArrowType.Utf8();
182+
case Types.DATE:
183+
return new ArrowType.Date(DateUnit.DAY);
184+
case Types.TIME:
185+
return new ArrowType.Time(TimeUnit.MILLISECOND, 32);
186+
case Types.TIMESTAMP:
187+
final String timezone;
188+
if (calendar != null) {
189+
timezone = calendar.getTimeZone().getID();
190+
} else {
191+
timezone = null;
192+
}
193+
return new ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone);
194+
case Types.BINARY:
195+
case Types.VARBINARY:
196+
case Types.LONGVARBINARY:
197+
case Types.BLOB:
198+
return new ArrowType.Binary();
199+
case Types.ARRAY:
200+
return new ArrowType.List();
201+
case Types.NULL:
202+
return new ArrowType.Null();
203+
case Types.STRUCT:
204+
return new ArrowType.Struct();
205+
default:
206+
// no-op, shouldn't get here
207+
return null;
208+
}
209+
}
210+
109211
/**
110212
* Create Arrow {@link Schema} object for the given JDBC {@link java.sql.ResultSetMetaData}.
111213
*

‎flight/flight-core/pom.xml‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@
1212
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
1313
<modelVersion>4.0.0</modelVersion>
1414
<parent>
15+
<artifactId>arrow-flight</artifactId>
1516
<groupId>org.apache.arrow</groupId>
16-
<artifactId>arrow-java-root</artifactId>
1717
<version>7.0.0-SNAPSHOT</version>
18-
<relativePath>../../pom.xml</relativePath>
18+
<relativePath>../pom.xml</relativePath>
1919
</parent>
2020

2121
<artifactId>flight-core</artifactId>
@@ -24,8 +24,6 @@
2424
<packaging>jar</packaging>
2525

2626
<properties>
27-
<dep.grpc.version>1.41.0</dep.grpc.version>
28-
<dep.protobuf.version>3.7.1</dep.protobuf.version>
2927
<forkCount>1</forkCount>
3028
</properties>
3129

@@ -95,11 +93,6 @@
9593
<groupId>com.google.guava</groupId>
9694
<artifactId>guava</artifactId>
9795
</dependency>
98-
<dependency>
99-
<groupId>commons-cli</groupId>
100-
<artifactId>commons-cli</artifactId>
101-
<version>1.4</version>
102-
</dependency>
10396
<dependency>
10497
<groupId>io.grpc</groupId>
10598
<artifactId>grpc-stub</artifactId>

‎flight/flight-grpc/pom.xml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@
1111
language governing permissions and limitations under the License. -->
1212
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
1313
<parent>
14-
<artifactId>arrow-java-root</artifactId>
14+
<artifactId>arrow-flight</artifactId>
1515
<groupId>org.apache.arrow</groupId>
1616
<version>7.0.0-SNAPSHOT</version>
17-
<relativePath>../../pom.xml</relativePath>
17+
<relativePath>../pom.xml</relativePath>
1818
</parent>
1919
<modelVersion>4.0.0</modelVersion>
2020

@@ -24,8 +24,6 @@
2424
<packaging>jar</packaging>
2525

2626
<properties>
27-
<dep.grpc.version>1.41.0</dep.grpc.version>
28-
<dep.protobuf.version>3.7.1</dep.protobuf.version>
2927
<forkCount>1</forkCount>
3028
</properties>
3129

0 commit comments

Comments
 (0)