Improved usage of use arrow in ballista. (#447)

Author: jorgecarleitao

ballista/rust/client/Cargo.toml

@@ -31,5 +31,4 @@ futures = "0.3"
 log = "0.4"
 tokio = "1.0"
 
-arrow = { version = "4.0"  }
 datafusion = { path = "../../../datafusion" }

ballista/rust/client/src/columnar_batch.rs

@@ -19,7 +19,7 @@ use std::{collections::HashMap, sync::Arc};
 
 use ballista_core::error::{ballista_error, Result};
 
-use arrow::{
+use datafusion::arrow::{
     array::ArrayRef,
     datatypes::{DataType, Schema},
     record_batch::RecordBatch,

ballista/rust/client/src/context.rs

@@ -33,7 +33,7 @@ use ballista_core::{
     utils::create_datafusion_context,
 };
 
-use arrow::datatypes::Schema;
+use datafusion::arrow::datatypes::Schema;
 use datafusion::catalog::TableReference;
 use datafusion::error::{DataFusionError, Result};
 use datafusion::logical_plan::LogicalPlan;

ballista/rust/core/Cargo.toml

@@ -40,7 +40,6 @@ tokio = "1.0"
 tonic = "0.4"
 uuid = { version = "0.8", features = ["v4"] }
 
-arrow = { version = "4.0" }
 arrow-flight = { version = "4.0"  }
 
 datafusion = { path = "../../../datafusion" }

ballista/rust/core/src/client.rs

@@ -31,15 +31,15 @@ use crate::serde::scheduler::{
     Action, ExecutePartition, ExecutePartitionResult, PartitionId, PartitionStats,
 };
 
-use arrow::record_batch::RecordBatch;
-use arrow::{
-    array::{StringArray, StructArray},
-    error::{ArrowError, Result as ArrowResult},
-};
-use arrow::{datatypes::Schema, datatypes::SchemaRef};
 use arrow_flight::utils::flight_data_to_arrow_batch;
 use arrow_flight::Ticket;
 use arrow_flight::{flight_service_client::FlightServiceClient, FlightData};
+use datafusion::arrow::{
+    array::{StringArray, StructArray},
+    datatypes::{Schema, SchemaRef},
+    error::{ArrowError, Result as ArrowResult},
+    record_batch::RecordBatch,
+};
 use datafusion::physical_plan::common::collect;
 use datafusion::physical_plan::{ExecutionPlan, SendableRecordBatchStream};
 use datafusion::{logical_plan::LogicalPlan, physical_plan::RecordBatchStream};

ballista/rust/core/src/datasource.rs

@@ -17,7 +17,7 @@
 
 use std::{any::Any, sync::Arc};
 
-use arrow::datatypes::SchemaRef;
+use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::error::Result as DFResult;
 use datafusion::{
     datasource::{datasource::Statistics, TableProvider},

ballista/rust/core/src/error.rs

@@ -23,7 +23,7 @@ use std::{
     io, result,
 };
 
-use arrow::error::ArrowError;
+use datafusion::arrow::error::ArrowError;
 use datafusion::error::DataFusionError;
 use sqlparser::parser;
 

ballista/rust/core/src/execution_plans/query_stage.rs

@@ -18,8 +18,8 @@
 use std::sync::Arc;
 use std::{any::Any, pin::Pin};
 
-use arrow::datatypes::SchemaRef;
 use async_trait::async_trait;
+use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::physical_plan::{ExecutionPlan, Partitioning};
 use datafusion::{error::Result, physical_plan::RecordBatchStream};
 use uuid::Uuid;

ballista/rust/core/src/execution_plans/shuffle_reader.rs

@@ -22,8 +22,8 @@ use crate::client::BallistaClient;
 use crate::memory_stream::MemoryStream;
 use crate::serde::scheduler::PartitionLocation;
 
-use arrow::datatypes::SchemaRef;
 use async_trait::async_trait;
+use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::physical_plan::{DisplayFormatType, ExecutionPlan, Partitioning};
 use datafusion::{
     error::{DataFusionError, Result},

ballista/rust/core/src/execution_plans/unresolved_shuffle.rs

@@ -21,8 +21,8 @@ use std::{any::Any, pin::Pin};
 use crate::memory_stream::MemoryStream;
 use crate::serde::scheduler::PartitionLocation;
 
-use arrow::datatypes::SchemaRef;
 use async_trait::async_trait;
+use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::physical_plan::{ExecutionPlan, Partitioning};
 use datafusion::{
     error::{DataFusionError, Result},