move to expr

Signed-off-by: jayzhan211 <jayzhan211@gmail.com>
apache · Mar 2, 2024 · adcb2e2 · adcb2e2
1 parent b69b33f
commit adcb2e2
Show file tree

Hide file tree

Showing 35 changed files with 70 additions and 61 deletions.
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-examples/examples/expr_api.rs b/datafusion-examples/examples/expr_api.rs
@@ -20,15 +20,16 @@ use arrow::record_batch::RecordBatch;
 use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit};
 use datafusion::common::{DFField, DFSchema};
 use datafusion::error::Result;
-use datafusion::optimizer::simplify_expressions::{ExprSimplifier, SimplifyContext};
-use datafusion::physical_expr::execution_props::ExecutionProps;
+use datafusion::optimizer::simplify_expressions::ExprSimplifier;
 use datafusion::physical_expr::{
     analyze, create_physical_expr, AnalysisContext, ExprBoundaries, PhysicalExpr,
 };
 use datafusion::prelude::*;
 use datafusion_common::{ScalarValue, ToDFSchema};
+use datafusion_expr::execution_props::ExecutionProps;
 use datafusion_expr::expr::BinaryExpr;
 use datafusion_expr::interval_arithmetic::Interval;
+use datafusion_expr::simplify::SimplifyContext;
 use datafusion_expr::{ColumnarValue, ExprSchemable, Operator};
 use std::collections::HashMap;
 use std::sync::Arc;

diff --git a/datafusion-examples/examples/simple_udtf.rs b/datafusion-examples/examples/simple_udtf.rs
@@ -28,8 +28,9 @@ use datafusion::physical_plan::memory::MemoryExec;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::SessionContext;
 use datafusion_common::{plan_err, DataFusionError, ScalarValue};
+use datafusion_expr::simplify::SimplifyContext;
 use datafusion_expr::{Expr, TableType};
-use datafusion_optimizer::simplify_expressions::{ExprSimplifier, SimplifyContext};
+use datafusion_optimizer::simplify_expressions::ExprSimplifier;
 use std::fs::File;
 use std::io::Seek;
 use std::path::Path;

diff --git a/datafusion/core/src/datasource/listing/helpers.rs b/datafusion/core/src/datasource/listing/helpers.rs
@@ -39,9 +39,9 @@ use crate::datasource::listing::ListingTableUrl;
 use crate::execution::context::SessionState;
 use datafusion_common::tree_node::{TreeNode, VisitRecursion};
 use datafusion_common::{internal_err, Column, DFField, DFSchema, DataFusionError};
+use datafusion_expr::execution_props::ExecutionProps;
 use datafusion_expr::{Expr, ScalarFunctionDefinition, Volatility};
 use datafusion_physical_expr::create_physical_expr;
-use datafusion_physical_expr::execution_props::ExecutionProps;
 use object_store::path::Path;
 use object_store::{ObjectMeta, ObjectStore};
 

diff --git a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs
@@ -792,9 +792,9 @@ mod tests {
     use chrono::{TimeZone, Utc};
     use datafusion_common::{assert_contains, ToDFSchema};
     use datafusion_common::{FileType, GetExt, ScalarValue};
+    use datafusion_expr::execution_props::ExecutionProps;
     use datafusion_expr::{col, lit, when, Expr};
     use datafusion_physical_expr::create_physical_expr;
-    use datafusion_physical_expr::execution_props::ExecutionProps;
     use futures::StreamExt;
     use object_store::local::LocalFileSystem;
     use object_store::path::Path;

diff --git a/datafusion/core/src/datasource/physical_plan/parquet/row_filter.rs b/datafusion/core/src/datasource/physical_plan/parquet/row_filter.rs
@@ -394,9 +394,9 @@ mod test {
     use super::*;
     use arrow::datatypes::Field;
     use datafusion_common::ToDFSchema;
+    use datafusion_expr::execution_props::ExecutionProps;
     use datafusion_expr::{cast, col, lit, Expr};
     use datafusion_physical_expr::create_physical_expr;
-    use datafusion_physical_expr::execution_props::ExecutionProps;
     use parquet::arrow::parquet_to_arrow_schema;
     use parquet::file::reader::{FileReader, SerializedFileReader};
     use rand::prelude::*;

diff --git a/datafusion/core/src/datasource/physical_plan/parquet/row_groups.rs b/datafusion/core/src/datasource/physical_plan/parquet/row_groups.rs
@@ -346,8 +346,8 @@ mod tests {
     use arrow::datatypes::Schema;
     use arrow::datatypes::{DataType, Field};
     use datafusion_common::{Result, ToDFSchema};
+    use datafusion_expr::execution_props::ExecutionProps;
     use datafusion_expr::{cast, col, lit, Expr};
-    use datafusion_physical_expr::execution_props::ExecutionProps;
     use datafusion_physical_expr::{create_physical_expr, PhysicalExpr};
     use parquet::arrow::arrow_to_parquet_schema;
     use parquet::arrow::async_reader::ParquetObjectReader;

diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs
@@ -43,12 +43,12 @@ use datafusion_common::{
     tree_node::{TreeNode, TreeNodeVisitor, VisitRecursion},
 };
 use datafusion_execution::registry::SerializerRegistry;
+pub use datafusion_expr::execution_props::ExecutionProps;
+use datafusion_expr::var_provider::is_system_variables;
 use datafusion_expr::{
     logical_plan::{DdlStatement, Statement},
     Expr, StringifiedPlan, UserDefinedLogicalNode, WindowUDF,
 };
-pub use datafusion_physical_expr::execution_props::ExecutionProps;
-use datafusion_physical_expr::var_provider::is_system_variables;
 use parking_lot::RwLock;
 use std::collections::hash_map::Entry;
 use std::string::String;

diff --git a/datafusion/core/src/physical_optimizer/pruning.rs b/datafusion/core/src/physical_optimizer/pruning.rs
@@ -1338,10 +1338,10 @@ mod tests {
         datatypes::{DataType, TimeUnit},
     };
     use datafusion_common::{ScalarValue, ToDFSchema};
+    use datafusion_expr::execution_props::ExecutionProps;
     use datafusion_expr::expr::InList;
     use datafusion_expr::{cast, is_null, try_cast, Expr};
     use datafusion_physical_expr::create_physical_expr;
-    use datafusion_physical_expr::execution_props::ExecutionProps;
     use std::collections::HashMap;
     use std::ops::{Not, Rem};
 

diff --git a/datafusion/core/src/test_util/parquet.rs b/datafusion/core/src/test_util/parquet.rs
@@ -28,9 +28,10 @@ use crate::datasource::listing::{ListingTableUrl, PartitionedFile};
 use crate::datasource::object_store::ObjectStoreUrl;
 use crate::datasource::physical_plan::{FileScanConfig, ParquetExec};
 use crate::error::Result;
-use crate::optimizer::simplify_expressions::{ExprSimplifier, SimplifyContext};
+use crate::logical_expr::execution_props::ExecutionProps;
+use crate::logical_expr::simplify::SimplifyContext;
+use crate::optimizer::simplify_expressions::ExprSimplifier;
 use crate::physical_expr::create_physical_expr;
-use crate::physical_expr::execution_props::ExecutionProps;
 use crate::physical_plan::filter::FilterExec;
 use crate::physical_plan::metrics::MetricsSet;
 use crate::physical_plan::ExecutionPlan;

diff --git a/datafusion/core/src/variable/mod.rs b/datafusion/core/src/variable/mod.rs
@@ -17,4 +17,4 @@
 
 //! Variable provider for `@name` and `@@name` style runtime values.
 
-pub use datafusion_physical_expr::var_provider::{VarProvider, VarType};
+pub use datafusion_expr::var_provider::{VarProvider, VarType};
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs
@@ -44,13 +44,13 @@ use datafusion_common::{assert_contains, DataFusionError, ScalarValue, UnnestOpt
 use datafusion_execution::config::SessionConfig;
 use datafusion_execution::runtime_env::RuntimeEnv;
 use datafusion_expr::expr::{GroupingSet, Sort};
+use datafusion_expr::var_provider::{VarProvider, VarType};
 use datafusion_expr::{
     array_agg, avg, cast, col, count, exists, expr, in_subquery, lit, max, out_ref_col,
     placeholder, scalar_subquery, sum, when, wildcard, AggregateFunction, Expr,
     ExprSchemable, WindowFrame, WindowFrameBound, WindowFrameUnits,
     WindowFunctionDefinition,
 };
-use datafusion_physical_expr::var_provider::{VarProvider, VarType};
 
 #[tokio::test]
 async fn test_count_wildcard_on_sort() -> Result<()> {

diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rs
@@ -28,9 +28,9 @@ use datafusion::physical_plan::metrics::MetricValue;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::SessionContext;
 use datafusion_common::{ScalarValue, Statistics, ToDFSchema};
+use datafusion_expr::execution_props::ExecutionProps;
 use datafusion_expr::{col, lit, Expr};
 use datafusion_physical_expr::create_physical_expr;
-use datafusion_physical_expr::execution_props::ExecutionProps;
 
 use futures::StreamExt;
 use object_store::path::Path;

diff --git a/datafusion/core/tests/simplification.rs b/datafusion/core/tests/simplification.rs
@@ -20,8 +20,9 @@
 use arrow::datatypes::{DataType, Field, Schema};
 use datafusion::{error::Result, execution::context::ExecutionProps, prelude::*};
 use datafusion_common::{DFSchemaRef, ToDFSchema};
+use datafusion_expr::simplify::SimplifyInfo;
 use datafusion_expr::{Expr, ExprSchemable};
-use datafusion_optimizer::simplify_expressions::{ExprSimplifier, SimplifyInfo};
+use datafusion_optimizer::simplify_expressions::ExprSimplifier;
 
 /// In order to simplify expressions, DataFusion must have information
 /// about the expressions.

diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
@@ -40,6 +40,7 @@ ahash = { version = "0.8", default-features = false, features = [
 ] }
 arrow = { workspace = true }
 arrow-array = { workspace = true }
+chrono = { workspace = true }
 datafusion-common = { workspace = true }
 paste = "^1.0"
 sqlparser = { workspace = true }

diff --git a/...sion/physical-expr/src/execution_props.rs → datafusion/expr/src/execution_props.rs b/...sion/physical-expr/src/execution_props.rs → datafusion/expr/src/execution_props.rs
diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rs
@@ -40,6 +40,7 @@ mod udwf;
 
 pub mod aggregate_function;
 pub mod conditional_expressions;
+pub mod execution_props;
 pub mod expr;
 pub mod expr_fn;
 pub mod expr_rewriter;
@@ -49,9 +50,11 @@ pub mod function;
 pub mod groups_accumulator;
 pub mod interval_arithmetic;
 pub mod logical_plan;
+pub mod simplify;
 pub mod tree_node;
 pub mod type_coercion;
 pub mod utils;
+pub mod var_provider;
 pub mod window_frame;
 pub mod window_state;
 

diff --git a/...mizer/src/simplify_expressions/context.rs → datafusion/expr/src/simplify.rs b/...mizer/src/simplify_expressions/context.rs → datafusion/expr/src/simplify.rs
@@ -19,8 +19,8 @@
 
 use arrow::datatypes::DataType;
 use datafusion_common::{DFSchemaRef, DataFusionError, Result};
-use datafusion_expr::{Expr, ExprSchemable};
-use datafusion_physical_expr::execution_props::ExecutionProps;
+
+use crate::{execution_props::ExecutionProps, Expr, ExprSchemable};
 
 #[allow(rustdoc::private_intra_doc_links)]
 /// The information necessary to apply algebraic simplification to an
@@ -54,8 +54,9 @@ pub trait SimplifyInfo {
 /// use arrow::datatypes::{Schema, Field, DataType};
 /// use datafusion_expr::{col, lit};
 /// use datafusion_common::{DataFusionError, ToDFSchema};
-/// use datafusion_physical_expr::execution_props::ExecutionProps;
-/// use datafusion_optimizer::simplify_expressions::{SimplifyContext, ExprSimplifier};
+/// use datafusion_expr::execution_props::ExecutionProps;
+/// use datafusion_expr::simplify::SimplifyContext;
+/// use datafusion_optimizer::simplify_expressions::ExprSimplifier;
 ///
 /// // Create the schema
 /// let schema = Schema::new(vec![

diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs
@@ -30,6 +30,7 @@ use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::sync::Arc;
 
+// TODO(In this PR): Move to simplify.rs
 /// Was the expression simplified?
 pub enum Simplified {
     /// The function call was simplified to an entirely new Expr

diff --git a/datafusion/physical-expr/src/var_provider.rs → datafusion/expr/src/var_provider.rs b/datafusion/physical-expr/src/var_provider.rs → datafusion/expr/src/var_provider.rs
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs
@@ -15,17 +15,18 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::simplify_expressions::{ExprSimplifier, SimplifyContext};
+use crate::simplify_expressions::ExprSimplifier;
 use crate::utils::collect_subquery_cols;
 use datafusion_common::tree_node::{
     RewriteRecursion, Transformed, TreeNode, TreeNodeRewriter,
 };
 use datafusion_common::{plan_err, Result};
 use datafusion_common::{Column, DFSchemaRef, DataFusionError, ScalarValue};
+use datafusion_expr::execution_props::ExecutionProps;
 use datafusion_expr::expr::{AggregateFunctionDefinition, Alias};
+use datafusion_expr::simplify::SimplifyContext;
 use datafusion_expr::utils::{conjunction, find_join_exprs, split_conjunction};
 use datafusion_expr::{expr, EmptyRelation, Expr, LogicalPlan, LogicalPlanBuilder};
-use datafusion_physical_expr::execution_props::ExecutionProps;
 use std::collections::{BTreeSet, HashMap};
 use std::ops::Deref;
 

diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -25,7 +25,6 @@ use super::utils::*;
 use crate::analyzer::type_coercion::TypeCoercionRewriter;
 use crate::simplify_expressions::guarantees::GuaranteeRewriter;
 use crate::simplify_expressions::regex::simplify_regex_expr;
-use crate::simplify_expressions::SimplifyInfo;
 
 use arrow::{
     array::{new_null_array, AsArray},
@@ -39,12 +38,14 @@ use datafusion_common::{
 use datafusion_common::{
     internal_err, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue,
 };
+use datafusion_expr::execution_props::ExecutionProps;
+use datafusion_expr::simplify::SimplifyInfo;
 use datafusion_expr::{
     and, lit, or, BinaryExpr, BuiltinScalarFunction, Case, ColumnarValue, Expr, Like,
     ScalarFunctionDefinition, Volatility,
 };
 use datafusion_expr::{expr::ScalarFunction, interval_arithmetic::NullableInterval};
-use datafusion_physical_expr::{create_physical_expr, execution_props::ExecutionProps};
+use datafusion_physical_expr::create_physical_expr;
 
 /// This structure handles API for expression simplification
 pub struct ExprSimplifier<S> {
@@ -92,8 +93,9 @@ impl<S: SimplifyInfo> ExprSimplifier<S> {
     /// use arrow::datatypes::DataType;
     /// use datafusion_expr::{col, lit, Expr};
     /// use datafusion_common::Result;
-    /// use datafusion_physical_expr::execution_props::ExecutionProps;
-    /// use datafusion_optimizer::simplify_expressions::{ExprSimplifier, SimplifyInfo};
+    /// use datafusion_expr::execution_props::ExecutionProps;
+    /// use datafusion_expr::simplify::SimplifyContext;
+    /// use datafusion_optimizer::simplify_expressions::ExprSimplifier;
     /// use datafusion_common::DFSchema;
     /// use std::sync::Arc;
     ///
@@ -193,9 +195,9 @@ impl<S: SimplifyInfo> ExprSimplifier<S> {
     /// use datafusion_expr::{col, lit, Expr};
     /// use datafusion_expr::interval_arithmetic::{Interval, NullableInterval};
     /// use datafusion_common::{Result, ScalarValue, ToDFSchema};
-    /// use datafusion_physical_expr::execution_props::ExecutionProps;
-    /// use datafusion_optimizer::simplify_expressions::{
-    ///     ExprSimplifier, SimplifyContext};
+    /// use datafusion_expr::execution_props::ExecutionProps;
+    /// use datafusion_expr::simplify::SimplifyContext;
+    /// use datafusion_optimizer::simplify_expressions::ExprSimplifier;
     ///
     /// let schema = Schema::new(vec![
     ///   Field::new("x", DataType::Int64, false),
@@ -252,9 +254,9 @@ impl<S: SimplifyInfo> ExprSimplifier<S> {
     /// use datafusion_expr::{col, lit, Expr};
     /// use datafusion_expr::interval_arithmetic::{Interval, NullableInterval};
     /// use datafusion_common::{Result, ScalarValue, ToDFSchema};
-    /// use datafusion_physical_expr::execution_props::ExecutionProps;
-    /// use datafusion_optimizer::simplify_expressions::{
-    ///     ExprSimplifier, SimplifyContext};
+    /// use datafusion_expr::execution_props::ExecutionProps;
+    /// use datafusion_expr::simplify::SimplifyContext;
+    /// use datafusion_optimizer::simplify_expressions::ExprSimplifier;
     ///
     /// let schema = Schema::new(vec![
     ///   Field::new("a", DataType::Int64, false),
@@ -1338,9 +1340,8 @@ mod tests {
     };
 
     use super::*;
-    use crate::simplify_expressions::{
-        utils::for_test::{cast_to_int64_expr, now_expr, to_timestamp_expr},
-        SimplifyContext,
+    use crate::simplify_expressions::utils::for_test::{
+        cast_to_int64_expr, now_expr, to_timestamp_expr,
     };
     use crate::test::test_table_scan_with_name;
 
@@ -1349,8 +1350,9 @@ mod tests {
         datatypes::{DataType, Field, Schema},
     };
     use datafusion_common::{assert_contains, cast::as_int32_array, DFField, ToDFSchema};
+    use datafusion_expr::execution_props::ExecutionProps;
+    use datafusion_expr::simplify::SimplifyContext;
     use datafusion_expr::{interval_arithmetic::Interval, *};
-    use datafusion_physical_expr::execution_props::ExecutionProps;
 
     use chrono::{DateTime, TimeZone, Utc};
 

diff --git a/datafusion/optimizer/src/simplify_expressions/function_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/function_simplifier.rs
@@ -19,11 +19,10 @@
 
 use datafusion_common::tree_node::TreeNodeRewriter;
 use datafusion_common::{DFSchema, Result};
+use datafusion_expr::simplify::SimplifyInfo;
 use datafusion_expr::Simplified;
 use datafusion_expr::{expr::ScalarFunction, Expr, ScalarFunctionDefinition};
 
-use super::SimplifyInfo;
-
 pub(super) struct FunctionSimplifier<'a, S> {
     info: &'a S,
 }

diff --git a/datafusion/optimizer/src/simplify_expressions/mod.rs b/datafusion/optimizer/src/simplify_expressions/mod.rs
@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-pub mod context;
 pub mod expr_simplifier;
 mod function_simplifier;
 mod guarantees;
@@ -24,6 +23,5 @@ mod regex;
 pub mod simplify_exprs;
 mod utils;
 
-pub use context::*;
 pub use expr_simplifier::*;
 pub use simplify_exprs::*;
diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs
@@ -19,12 +19,13 @@
 
 use std::sync::Arc;
 
-use super::{ExprSimplifier, SimplifyContext};
+use super::ExprSimplifier;
 use crate::{OptimizerConfig, OptimizerRule};
 use datafusion_common::{DFSchema, DFSchemaRef, Result};
+use datafusion_expr::execution_props::ExecutionProps;
 use datafusion_expr::logical_plan::LogicalPlan;
+use datafusion_expr::simplify::SimplifyContext;
 use datafusion_expr::utils::merge_schema;
-use datafusion_physical_expr::execution_props::ExecutionProps;
 
 /// Optimizer Pass that simplifies [`LogicalPlan`]s by rewriting
 /// [`Expr`]`s evaluating constants and applying algebraic

diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs
@@ -17,11 +17,10 @@
 
 //! Utility functions for expression simplification
 
-use crate::simplify_expressions::SimplifyInfo;
 use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue};
-use datafusion_expr::expr::ScalarFunction;
+use datafusion_expr::simplify::SimplifyInfo;
 use datafusion_expr::{
-    expr::{Between, BinaryExpr, InList},
+    expr::{Between, BinaryExpr, InList, ScalarFunction},
     expr_fn::{and, bitwise_and, bitwise_or, concat_ws, or},
     lit, BuiltinScalarFunction, Expr, Like, Operator, ScalarFunctionDefinition,
 };