From 2f07c568107b2e466a6d6e199eaff7068100bb3c Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Thu, 12 Nov 2020 14:59:22 +0000 Subject: [PATCH] [SPARK-33278][SQL] Improve the performance for FIRST_VALUE ### What changes were proposed in this pull request? https://github.com/apache/spark/pull/29800 provides a performance improvement for `NTH_VALUE`. `FIRST_VALUE` also could use the `UnboundedOffsetWindowFunctionFrame` and `UnboundedPrecedingOffsetWindowFunctionFrame`. ### Why are the changes needed? Improve the performance for `FIRST_VALUE`. ### Does this PR introduce _any_ user-facing change? 'No'. ### How was this patch tested? Jenkins test. Closes #30178 from beliefer/SPARK-33278. Lead-authored-by: gengjiaan Co-authored-by: beliefer Co-authored-by: Jiaan Geng Signed-off-by: Wenchen Fan --- .../sql/catalyst/optimizer/Optimizer.scala | 13 + .../OptimizeWindowFunctionsSuite.scala | 76 ++++ .../resources/sql-tests/inputs/window.sql | 66 +-- .../sql-tests/results/window.sql.out | 426 +++++++++--------- 4 files changed, 339 insertions(+), 242 deletions(-) create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWindowFunctionsSuite.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 51f7799b1e427..e492d01650097 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -82,6 +82,7 @@ abstract class Optimizer(catalogManager: CatalogManager) // Operator combine CollapseRepartition, CollapseProject, + OptimizeWindowFunctions, CollapseWindow, CombineFilters, CombineLimits, @@ -806,6 +807,18 @@ object CollapseRepartition extends Rule[LogicalPlan] { } } +/** + * Replaces first(col) to nth_value(col, 1) for better performance. + */ +object OptimizeWindowFunctions extends Rule[LogicalPlan] { + def apply(plan: LogicalPlan): LogicalPlan = plan resolveExpressions { + case we @ WindowExpression(AggregateExpression(first: First, _, _, _, _), spec) + if spec.orderSpec.nonEmpty && + spec.frameSpecification.asInstanceOf[SpecifiedWindowFrame].frameType == RowFrame => + we.copy(windowFunction = NthValue(first.child, Literal(1), first.ignoreNulls)) + } +} + /** * Collapse Adjacent Window Expression. * - If the partition specs and order specs are the same and the window expression are diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWindowFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWindowFunctionsSuite.scala new file mode 100644 index 0000000000000..389aaeafe655f --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWindowFunctionsSuite.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.optimizer + +import org.apache.spark.sql.catalyst.dsl.expressions._ +import org.apache.spark.sql.catalyst.dsl.plans._ +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.aggregate.First +import org.apache.spark.sql.catalyst.plans.PlanTest +import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan} +import org.apache.spark.sql.catalyst.rules.RuleExecutor + +class OptimizeWindowFunctionsSuite extends PlanTest { + object Optimize extends RuleExecutor[LogicalPlan] { + val batches = Batch("OptimizeWindowFunctions", FixedPoint(10), + OptimizeWindowFunctions) :: Nil + } + + val testRelation = LocalRelation('a.double, 'b.double, 'c.string) + val a = testRelation.output(0) + val b = testRelation.output(1) + val c = testRelation.output(2) + + test("replace first(col) by nth_value(col, 1)") { + val inputPlan = testRelation.select( + WindowExpression( + First(a, false).toAggregateExpression(), + WindowSpecDefinition(b :: Nil, c.asc :: Nil, + SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow)))) + val correctAnswer = testRelation.select( + WindowExpression( + NthValue(a, Literal(1), false), + WindowSpecDefinition(b :: Nil, c.asc :: Nil, + SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow)))) + + val optimized = Optimize.execute(inputPlan) + assert(optimized == correctAnswer) + } + + test("can't replace first(col) by nth_value(col, 1) if the window frame type is range") { + val inputPlan = testRelation.select( + WindowExpression( + First(a, false).toAggregateExpression(), + WindowSpecDefinition(b :: Nil, c.asc :: Nil, + SpecifiedWindowFrame(RangeFrame, UnboundedPreceding, CurrentRow)))) + + val optimized = Optimize.execute(inputPlan) + assert(optimized == inputPlan) + } + + test("can't replace first(col) by nth_value(col, 1) if the window frame isn't ordered") { + val inputPlan = testRelation.select( + WindowExpression( + First(a, false).toAggregateExpression(), + WindowSpecDefinition(b :: Nil, Nil, + SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow)))) + + val optimized = Optimize.execute(inputPlan) + assert(optimized == inputPlan) + } +} diff --git a/sql/core/src/test/resources/sql-tests/inputs/window.sql b/sql/core/src/test/resources/sql-tests/inputs/window.sql index c1be5fb27e6fa..f5223af9125f6 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/window.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/window.sql @@ -146,104 +146,108 @@ SELECT val, cate, count(val) FILTER (WHERE val > 1) OVER(PARTITION BY cate) FROM testData ORDER BY cate, val; --- nth_value() over () +-- nth_value()/first_value() over () SELECT employee_name, salary, - nth_value(employee_name, 2) OVER (ORDER BY salary DESC) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC) ORDER BY salary DESC; SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) ORDER BY salary DESC; SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) ORDER BY salary DESC; SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary - RANGE BETWEEN 2000 PRECEDING AND 1000 FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary RANGE BETWEEN 2000 PRECEDING AND 1000 FOLLOWING) ORDER BY salary; SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) ORDER BY salary DESC; SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) ORDER BY salary DESC; SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) ORDER BY salary DESC; SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) ORDER BY salary DESC; SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) ORDER BY salary DESC; SELECT employee_name, department, salary, - NTH_VALUE(employee_name, 2) OVER ( - PARTITION BY department - ORDER BY salary DESC - RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) second_highest_salary + FIRST_VALUE(employee_name) OVER w highest_salary, + NTH_VALUE(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS ( + PARTITION BY department + ORDER BY salary DESC + RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +) ORDER BY department; \ No newline at end of file diff --git a/sql/core/src/test/resources/sql-tests/results/window.sql.out b/sql/core/src/test/resources/sql-tests/results/window.sql.out index f6506a77e239c..1304dcf21d0b3 100644 --- a/sql/core/src/test/resources/sql-tests/results/window.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/window.sql.out @@ -421,286 +421,288 @@ window aggregate function with filter predicate is not supported yet.; SELECT employee_name, salary, - nth_value(employee_name, 2) OVER (ORDER BY salary DESC) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC) ORDER BY salary DESC -- !query schema -struct +struct -- !query output -Larry Bott 11798 NULL -Gerard Bondur 11472 Gerard Bondur -Pamela Castillo 11303 Gerard Bondur -Barry Jones 10586 Gerard Bondur -George Vanauf 10563 Gerard Bondur -Loui Bondur 10449 Gerard Bondur -Mary Patterson 9998 Gerard Bondur -Steve Patterson 9441 Gerard Bondur -Julie Firrelli 9181 Gerard Bondur -Jeff Firrelli 8992 Gerard Bondur -William Patterson 8870 Gerard Bondur -Diane Murphy 8435 Gerard Bondur -Leslie Jennings 8113 Gerard Bondur -Gerard Hernandez 6949 Gerard Bondur -Foon Yue Tseng 6660 Gerard Bondur -Anthony Bow 6627 Gerard Bondur -Leslie Thompson 5186 Gerard Bondur +Larry Bott 11798 Larry Bott NULL +Gerard Bondur 11472 Larry Bott Gerard Bondur +Pamela Castillo 11303 Larry Bott Gerard Bondur +Barry Jones 10586 Larry Bott Gerard Bondur +George Vanauf 10563 Larry Bott Gerard Bondur +Loui Bondur 10449 Larry Bott Gerard Bondur +Mary Patterson 9998 Larry Bott Gerard Bondur +Steve Patterson 9441 Larry Bott Gerard Bondur +Julie Firrelli 9181 Larry Bott Gerard Bondur +Jeff Firrelli 8992 Larry Bott Gerard Bondur +William Patterson 8870 Larry Bott Gerard Bondur +Diane Murphy 8435 Larry Bott Gerard Bondur +Leslie Jennings 8113 Larry Bott Gerard Bondur +Gerard Hernandez 6949 Larry Bott Gerard Bondur +Foon Yue Tseng 6660 Larry Bott Gerard Bondur +Anthony Bow 6627 Larry Bott Gerard Bondur +Leslie Thompson 5186 Larry Bott Gerard Bondur -- !query SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) ORDER BY salary DESC -- !query schema -struct +struct -- !query output -Larry Bott 11798 NULL -Gerard Bondur 11472 Gerard Bondur -Pamela Castillo 11303 Gerard Bondur -Barry Jones 10586 Gerard Bondur -George Vanauf 10563 Gerard Bondur -Loui Bondur 10449 Gerard Bondur -Mary Patterson 9998 Gerard Bondur -Steve Patterson 9441 Gerard Bondur -Julie Firrelli 9181 Gerard Bondur -Jeff Firrelli 8992 Gerard Bondur -William Patterson 8870 Gerard Bondur -Diane Murphy 8435 Gerard Bondur -Leslie Jennings 8113 Gerard Bondur -Gerard Hernandez 6949 Gerard Bondur -Foon Yue Tseng 6660 Gerard Bondur -Anthony Bow 6627 Gerard Bondur -Leslie Thompson 5186 Gerard Bondur +Larry Bott 11798 Larry Bott NULL +Gerard Bondur 11472 Larry Bott Gerard Bondur +Pamela Castillo 11303 Larry Bott Gerard Bondur +Barry Jones 10586 Larry Bott Gerard Bondur +George Vanauf 10563 Larry Bott Gerard Bondur +Loui Bondur 10449 Larry Bott Gerard Bondur +Mary Patterson 9998 Larry Bott Gerard Bondur +Steve Patterson 9441 Larry Bott Gerard Bondur +Julie Firrelli 9181 Larry Bott Gerard Bondur +Jeff Firrelli 8992 Larry Bott Gerard Bondur +William Patterson 8870 Larry Bott Gerard Bondur +Diane Murphy 8435 Larry Bott Gerard Bondur +Leslie Jennings 8113 Larry Bott Gerard Bondur +Gerard Hernandez 6949 Larry Bott Gerard Bondur +Foon Yue Tseng 6660 Larry Bott Gerard Bondur +Anthony Bow 6627 Larry Bott Gerard Bondur +Leslie Thompson 5186 Larry Bott Gerard Bondur -- !query SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) ORDER BY salary DESC -- !query schema -struct +struct -- !query output -Larry Bott 11798 NULL -Gerard Bondur 11472 Gerard Bondur -Pamela Castillo 11303 Gerard Bondur -Barry Jones 10586 Gerard Bondur -George Vanauf 10563 Gerard Bondur -Loui Bondur 10449 Gerard Bondur -Mary Patterson 9998 Gerard Bondur -Steve Patterson 9441 Gerard Bondur -Julie Firrelli 9181 Gerard Bondur -Jeff Firrelli 8992 Gerard Bondur -William Patterson 8870 Gerard Bondur -Diane Murphy 8435 Gerard Bondur -Leslie Jennings 8113 Gerard Bondur -Gerard Hernandez 6949 Gerard Bondur -Foon Yue Tseng 6660 Gerard Bondur -Anthony Bow 6627 Gerard Bondur -Leslie Thompson 5186 Gerard Bondur +Larry Bott 11798 Larry Bott NULL +Gerard Bondur 11472 Larry Bott Gerard Bondur +Pamela Castillo 11303 Larry Bott Gerard Bondur +Barry Jones 10586 Larry Bott Gerard Bondur +George Vanauf 10563 Larry Bott Gerard Bondur +Loui Bondur 10449 Larry Bott Gerard Bondur +Mary Patterson 9998 Larry Bott Gerard Bondur +Steve Patterson 9441 Larry Bott Gerard Bondur +Julie Firrelli 9181 Larry Bott Gerard Bondur +Jeff Firrelli 8992 Larry Bott Gerard Bondur +William Patterson 8870 Larry Bott Gerard Bondur +Diane Murphy 8435 Larry Bott Gerard Bondur +Leslie Jennings 8113 Larry Bott Gerard Bondur +Gerard Hernandez 6949 Larry Bott Gerard Bondur +Foon Yue Tseng 6660 Larry Bott Gerard Bondur +Anthony Bow 6627 Larry Bott Gerard Bondur +Leslie Thompson 5186 Larry Bott Gerard Bondur -- !query SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary - RANGE BETWEEN 2000 PRECEDING AND 1000 FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary RANGE BETWEEN 2000 PRECEDING AND 1000 FOLLOWING) ORDER BY salary -- !query schema -struct +struct -- !query output -Leslie Thompson 5186 NULL -Anthony Bow 6627 Anthony Bow -Foon Yue Tseng 6660 Anthony Bow -Gerard Hernandez 6949 Anthony Bow -Leslie Jennings 8113 Foon Yue Tseng -Diane Murphy 8435 Foon Yue Tseng -William Patterson 8870 Leslie Jennings -Jeff Firrelli 8992 Diane Murphy -Julie Firrelli 9181 Diane Murphy -Steve Patterson 9441 Diane Murphy -Mary Patterson 9998 Diane Murphy -Loui Bondur 10449 Jeff Firrelli -George Vanauf 10563 Jeff Firrelli -Barry Jones 10586 Jeff Firrelli -Pamela Castillo 11303 Mary Patterson -Gerard Bondur 11472 Loui Bondur -Larry Bott 11798 Loui Bondur +Leslie Thompson 5186 Leslie Thompson NULL +Anthony Bow 6627 Leslie Thompson Anthony Bow +Foon Yue Tseng 6660 Leslie Thompson Anthony Bow +Gerard Hernandez 6949 Leslie Thompson Anthony Bow +Leslie Jennings 8113 Anthony Bow Foon Yue Tseng +Diane Murphy 8435 Anthony Bow Foon Yue Tseng +William Patterson 8870 Gerard Hernandez Leslie Jennings +Jeff Firrelli 8992 Leslie Jennings Diane Murphy +Julie Firrelli 9181 Leslie Jennings Diane Murphy +Steve Patterson 9441 Leslie Jennings Diane Murphy +Mary Patterson 9998 Leslie Jennings Diane Murphy +Loui Bondur 10449 William Patterson Jeff Firrelli +George Vanauf 10563 William Patterson Jeff Firrelli +Barry Jones 10586 William Patterson Jeff Firrelli +Pamela Castillo 11303 Steve Patterson Mary Patterson +Gerard Bondur 11472 Mary Patterson Loui Bondur +Larry Bott 11798 Mary Patterson Loui Bondur -- !query SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) ORDER BY salary DESC -- !query schema -struct +struct -- !query output -Larry Bott 11798 Gerard Bondur -Gerard Bondur 11472 Gerard Bondur -Pamela Castillo 11303 Gerard Bondur -Barry Jones 10586 Pamela Castillo -George Vanauf 10563 Barry Jones -Loui Bondur 10449 George Vanauf -Mary Patterson 9998 Loui Bondur -Steve Patterson 9441 Mary Patterson -Julie Firrelli 9181 Steve Patterson -Jeff Firrelli 8992 Julie Firrelli -William Patterson 8870 Jeff Firrelli -Diane Murphy 8435 William Patterson -Leslie Jennings 8113 Diane Murphy -Gerard Hernandez 6949 Leslie Jennings -Foon Yue Tseng 6660 Gerard Hernandez -Anthony Bow 6627 Foon Yue Tseng -Leslie Thompson 5186 Anthony Bow +Larry Bott 11798 Larry Bott Gerard Bondur +Gerard Bondur 11472 Larry Bott Gerard Bondur +Pamela Castillo 11303 Larry Bott Gerard Bondur +Barry Jones 10586 Gerard Bondur Pamela Castillo +George Vanauf 10563 Pamela Castillo Barry Jones +Loui Bondur 10449 Barry Jones George Vanauf +Mary Patterson 9998 George Vanauf Loui Bondur +Steve Patterson 9441 Loui Bondur Mary Patterson +Julie Firrelli 9181 Mary Patterson Steve Patterson +Jeff Firrelli 8992 Steve Patterson Julie Firrelli +William Patterson 8870 Julie Firrelli Jeff Firrelli +Diane Murphy 8435 Jeff Firrelli William Patterson +Leslie Jennings 8113 William Patterson Diane Murphy +Gerard Hernandez 6949 Diane Murphy Leslie Jennings +Foon Yue Tseng 6660 Leslie Jennings Gerard Hernandez +Anthony Bow 6627 Gerard Hernandez Foon Yue Tseng +Leslie Thompson 5186 Foon Yue Tseng Anthony Bow -- !query SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) ORDER BY salary DESC -- !query schema -struct +struct -- !query output -Larry Bott 11798 Gerard Bondur -Gerard Bondur 11472 Pamela Castillo -Pamela Castillo 11303 Barry Jones -Barry Jones 10586 George Vanauf -George Vanauf 10563 Loui Bondur -Loui Bondur 10449 Mary Patterson -Mary Patterson 9998 Steve Patterson -Steve Patterson 9441 Julie Firrelli -Julie Firrelli 9181 Jeff Firrelli -Jeff Firrelli 8992 William Patterson -William Patterson 8870 Diane Murphy -Diane Murphy 8435 Leslie Jennings -Leslie Jennings 8113 Gerard Hernandez -Gerard Hernandez 6949 Foon Yue Tseng -Foon Yue Tseng 6660 Anthony Bow -Anthony Bow 6627 Leslie Thompson -Leslie Thompson 5186 NULL +Larry Bott 11798 Larry Bott Gerard Bondur +Gerard Bondur 11472 Gerard Bondur Pamela Castillo +Pamela Castillo 11303 Pamela Castillo Barry Jones +Barry Jones 10586 Barry Jones George Vanauf +George Vanauf 10563 George Vanauf Loui Bondur +Loui Bondur 10449 Loui Bondur Mary Patterson +Mary Patterson 9998 Mary Patterson Steve Patterson +Steve Patterson 9441 Steve Patterson Julie Firrelli +Julie Firrelli 9181 Julie Firrelli Jeff Firrelli +Jeff Firrelli 8992 Jeff Firrelli William Patterson +William Patterson 8870 William Patterson Diane Murphy +Diane Murphy 8435 Diane Murphy Leslie Jennings +Leslie Jennings 8113 Leslie Jennings Gerard Hernandez +Gerard Hernandez 6949 Gerard Hernandez Foon Yue Tseng +Foon Yue Tseng 6660 Foon Yue Tseng Anthony Bow +Anthony Bow 6627 Anthony Bow Leslie Thompson +Leslie Thompson 5186 Leslie Thompson NULL -- !query SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) ORDER BY salary DESC -- !query schema -struct +struct -- !query output -Larry Bott 11798 Gerard Bondur -Gerard Bondur 11472 Gerard Bondur -Pamela Castillo 11303 Gerard Bondur -Barry Jones 10586 Gerard Bondur -George Vanauf 10563 Gerard Bondur -Loui Bondur 10449 Gerard Bondur -Mary Patterson 9998 Gerard Bondur -Steve Patterson 9441 Gerard Bondur -Julie Firrelli 9181 Gerard Bondur -Jeff Firrelli 8992 Gerard Bondur -William Patterson 8870 Gerard Bondur -Diane Murphy 8435 Gerard Bondur -Leslie Jennings 8113 Gerard Bondur -Gerard Hernandez 6949 Gerard Bondur -Foon Yue Tseng 6660 Gerard Bondur -Anthony Bow 6627 Gerard Bondur -Leslie Thompson 5186 Gerard Bondur +Larry Bott 11798 Larry Bott Gerard Bondur +Gerard Bondur 11472 Larry Bott Gerard Bondur +Pamela Castillo 11303 Larry Bott Gerard Bondur +Barry Jones 10586 Larry Bott Gerard Bondur +George Vanauf 10563 Larry Bott Gerard Bondur +Loui Bondur 10449 Larry Bott Gerard Bondur +Mary Patterson 9998 Larry Bott Gerard Bondur +Steve Patterson 9441 Larry Bott Gerard Bondur +Julie Firrelli 9181 Larry Bott Gerard Bondur +Jeff Firrelli 8992 Larry Bott Gerard Bondur +William Patterson 8870 Larry Bott Gerard Bondur +Diane Murphy 8435 Larry Bott Gerard Bondur +Leslie Jennings 8113 Larry Bott Gerard Bondur +Gerard Hernandez 6949 Larry Bott Gerard Bondur +Foon Yue Tseng 6660 Larry Bott Gerard Bondur +Anthony Bow 6627 Larry Bott Gerard Bondur +Leslie Thompson 5186 Larry Bott Gerard Bondur -- !query SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) ORDER BY salary DESC -- !query schema -struct +struct -- !query output -Larry Bott 11798 Gerard Bondur -Gerard Bondur 11472 Gerard Bondur -Pamela Castillo 11303 Gerard Bondur -Barry Jones 10586 Gerard Bondur -George Vanauf 10563 Gerard Bondur -Loui Bondur 10449 Gerard Bondur -Mary Patterson 9998 Gerard Bondur -Steve Patterson 9441 Gerard Bondur -Julie Firrelli 9181 Gerard Bondur -Jeff Firrelli 8992 Gerard Bondur -William Patterson 8870 Gerard Bondur -Diane Murphy 8435 Gerard Bondur -Leslie Jennings 8113 Gerard Bondur -Gerard Hernandez 6949 Gerard Bondur -Foon Yue Tseng 6660 Gerard Bondur -Anthony Bow 6627 Gerard Bondur -Leslie Thompson 5186 Gerard Bondur +Larry Bott 11798 Larry Bott Gerard Bondur +Gerard Bondur 11472 Larry Bott Gerard Bondur +Pamela Castillo 11303 Larry Bott Gerard Bondur +Barry Jones 10586 Larry Bott Gerard Bondur +George Vanauf 10563 Larry Bott Gerard Bondur +Loui Bondur 10449 Larry Bott Gerard Bondur +Mary Patterson 9998 Larry Bott Gerard Bondur +Steve Patterson 9441 Larry Bott Gerard Bondur +Julie Firrelli 9181 Larry Bott Gerard Bondur +Jeff Firrelli 8992 Larry Bott Gerard Bondur +William Patterson 8870 Larry Bott Gerard Bondur +Diane Murphy 8435 Larry Bott Gerard Bondur +Leslie Jennings 8113 Larry Bott Gerard Bondur +Gerard Hernandez 6949 Larry Bott Gerard Bondur +Foon Yue Tseng 6660 Larry Bott Gerard Bondur +Anthony Bow 6627 Larry Bott Gerard Bondur +Leslie Thompson 5186 Larry Bott Gerard Bondur -- !query SELECT employee_name, salary, - nth_value(employee_name, 2) OVER ( - ORDER BY salary DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) second_highest_salary + first_value(employee_name) OVER w highest_salary, + nth_value(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) ORDER BY salary DESC -- !query schema -struct +struct -- !query output -Larry Bott 11798 Gerard Bondur -Gerard Bondur 11472 Gerard Bondur -Pamela Castillo 11303 Gerard Bondur -Barry Jones 10586 Gerard Bondur -George Vanauf 10563 Gerard Bondur -Loui Bondur 10449 Gerard Bondur -Mary Patterson 9998 Gerard Bondur -Steve Patterson 9441 Gerard Bondur -Julie Firrelli 9181 Gerard Bondur -Jeff Firrelli 8992 Gerard Bondur -William Patterson 8870 Gerard Bondur -Diane Murphy 8435 Gerard Bondur -Leslie Jennings 8113 Gerard Bondur -Gerard Hernandez 6949 Gerard Bondur -Foon Yue Tseng 6660 Gerard Bondur -Anthony Bow 6627 Gerard Bondur -Leslie Thompson 5186 Gerard Bondur +Larry Bott 11798 Larry Bott Gerard Bondur +Gerard Bondur 11472 Larry Bott Gerard Bondur +Pamela Castillo 11303 Larry Bott Gerard Bondur +Barry Jones 10586 Larry Bott Gerard Bondur +George Vanauf 10563 Larry Bott Gerard Bondur +Loui Bondur 10449 Larry Bott Gerard Bondur +Mary Patterson 9998 Larry Bott Gerard Bondur +Steve Patterson 9441 Larry Bott Gerard Bondur +Julie Firrelli 9181 Larry Bott Gerard Bondur +Jeff Firrelli 8992 Larry Bott Gerard Bondur +William Patterson 8870 Larry Bott Gerard Bondur +Diane Murphy 8435 Larry Bott Gerard Bondur +Leslie Jennings 8113 Larry Bott Gerard Bondur +Gerard Hernandez 6949 Larry Bott Gerard Bondur +Foon Yue Tseng 6660 Larry Bott Gerard Bondur +Anthony Bow 6627 Larry Bott Gerard Bondur +Leslie Thompson 5186 Larry Bott Gerard Bondur -- !query @@ -708,31 +710,33 @@ SELECT employee_name, department, salary, - NTH_VALUE(employee_name, 2) OVER ( - PARTITION BY department - ORDER BY salary DESC - RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) second_highest_salary + FIRST_VALUE(employee_name) OVER w highest_salary, + NTH_VALUE(employee_name, 2) OVER w second_highest_salary FROM basic_pays +WINDOW w AS ( + PARTITION BY department + ORDER BY salary DESC + RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +) ORDER BY department -- !query schema -struct --- !query output -Gerard Bondur Accounting 11472 Mary Patterson -Mary Patterson Accounting 9998 Mary Patterson -Jeff Firrelli Accounting 8992 Mary Patterson -William Patterson Accounting 8870 Mary Patterson -Diane Murphy Accounting 8435 Mary Patterson -Anthony Bow Accounting 6627 Mary Patterson -Leslie Jennings IT 8113 Leslie Thompson -Leslie Thompson IT 5186 Leslie Thompson -Larry Bott SCM 11798 Pamela Castillo -Pamela Castillo SCM 11303 Pamela Castillo -Barry Jones SCM 10586 Pamela Castillo -Loui Bondur SCM 10449 Pamela Castillo -Gerard Hernandez SCM 6949 Pamela Castillo -George Vanauf Sales 10563 Steve Patterson -Steve Patterson Sales 9441 Steve Patterson -Julie Firrelli Sales 9181 Steve Patterson -Foon Yue Tseng Sales 6660 Steve Patterson \ No newline at end of file +struct +-- !query output +Gerard Bondur Accounting 11472 Gerard Bondur Mary Patterson +Mary Patterson Accounting 9998 Gerard Bondur Mary Patterson +Jeff Firrelli Accounting 8992 Gerard Bondur Mary Patterson +William Patterson Accounting 8870 Gerard Bondur Mary Patterson +Diane Murphy Accounting 8435 Gerard Bondur Mary Patterson +Anthony Bow Accounting 6627 Gerard Bondur Mary Patterson +Leslie Jennings IT 8113 Leslie Jennings Leslie Thompson +Leslie Thompson IT 5186 Leslie Jennings Leslie Thompson +Larry Bott SCM 11798 Larry Bott Pamela Castillo +Pamela Castillo SCM 11303 Larry Bott Pamela Castillo +Barry Jones SCM 10586 Larry Bott Pamela Castillo +Loui Bondur SCM 10449 Larry Bott Pamela Castillo +Gerard Hernandez SCM 6949 Larry Bott Pamela Castillo +George Vanauf Sales 10563 George Vanauf Steve Patterson +Steve Patterson Sales 9441 George Vanauf Steve Patterson +Julie Firrelli Sales 9181 George Vanauf Steve Patterson +Foon Yue Tseng Sales 6660 George Vanauf Steve Patterson \ No newline at end of file