Skip to content

Commit 962e330

Browse files
cxzl25wangyum
authored andcommitted
[SPARK-26598][SQL] Fix HiveThriftServer2 cannot be modified hiveconf/hivevar variables
### What changes were proposed in this pull request? The intent to use the --hiveconf/--hivevar parameter is just an initialization value, so setting it once in ```SparkSQLSessionManager#openSession``` is sufficient, and each time the ```SparkExecuteStatementOperation``` setting causes the variable to not be modified. ### Why are the changes needed? It is wrong to set the --hivevar/--hiveconf variable in every ```SparkExecuteStatementOperation```, which prevents variable updates. ### Does this PR introduce any user-facing change? ``` cat <<EOF > test.sql select '\${a}', '\${b}'; set b=bvalue_MOD_VALUE; set b; EOF beeline -u jdbc:hive2://localhost:10000 --hiveconf a=avalue --hivevar b=bvalue -f test.sql ``` current result: ``` +-----------------+-----------------+--+ | avalue | bvalue | +-----------------+-----------------+--+ | avalue | bvalue | +-----------------+-----------------+--+ +-----------------+-----------------+--+ | key | value | +-----------------+-----------------+--+ | b | bvalue | +-----------------+-----------------+--+ 1 row selected (0.022 seconds) ``` after modification: ``` +-----------------+-----------------+--+ | avalue | bvalue | +-----------------+-----------------+--+ | avalue | bvalue | +-----------------+-----------------+--+ +-----------------+-----------------+--+ | key | value | +-----------------+-----------------+--+ | b | bvalue_MOD_VALUE| +-----------------+-----------------+--+ 1 row selected (0.022 seconds) ``` ### How was this patch tested? modified the existing unit test Closes #25722 from cxzl25/fix_SPARK-26598. Authored-by: sychen <sychen@ctrip.com> Signed-off-by: Yuming Wang <wgyumg@gmail.com>
1 parent 580c626 commit 962e330

File tree

3 files changed

+21
-15
lines changed

3 files changed

+21
-15
lines changed

sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext:
6363
sqlContext.newSession()
6464
}
6565
ctx.setConf(HiveUtils.FAKE_HIVE_VERSION.key, HiveUtils.builtinHiveVersion)
66+
val hiveSessionState = session.getSessionState
67+
setConfMap(ctx, hiveSessionState.getOverriddenConfigurations)
68+
setConfMap(ctx, hiveSessionState.getHiveVariables)
6669
if (sessionConf != null && sessionConf.containsKey("use:database")) {
6770
ctx.sql(s"use ${sessionConf.get("use:database")}")
6871
}
@@ -76,4 +79,12 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext:
7679
sparkSqlOperationManager.sessionToActivePool.remove(sessionHandle)
7780
sparkSqlOperationManager.sessionToContexts.remove(sessionHandle)
7881
}
82+
83+
def setConfMap(conf: SQLContext, confMap: java.util.Map[String, String]): Unit = {
84+
val iterator = confMap.entrySet().iterator()
85+
while (iterator.hasNext) {
86+
val kv = iterator.next()
87+
conf.setConf(kv.getKey, kv.getValue)
88+
}
89+
}
7990
}

sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ import org.apache.spark.internal.Logging
2828
import org.apache.spark.sql.SQLContext
2929
import org.apache.spark.sql.hive.HiveUtils
3030
import org.apache.spark.sql.hive.thriftserver._
31-
import org.apache.spark.sql.internal.SQLConf
3231

3332
/**
3433
* Executes queries using Spark SQL, and maintains a list of handles to active queries.
@@ -51,9 +50,6 @@ private[thriftserver] class SparkSQLOperationManager()
5150
require(sqlContext != null, s"Session handle: ${parentSession.getSessionHandle} has not been" +
5251
s" initialized or had already closed.")
5352
val conf = sqlContext.sessionState.conf
54-
val hiveSessionState = parentSession.getSessionState
55-
setConfMap(conf, hiveSessionState.getOverriddenConfigurations)
56-
setConfMap(conf, hiveSessionState.getHiveVariables)
5753
val runInBackground = async && conf.getConf(HiveUtils.HIVE_THRIFT_SERVER_ASYNC)
5854
val operation = new SparkExecuteStatementOperation(parentSession, statement, confOverlay,
5955
runInBackground)(sqlContext, sessionToActivePool)
@@ -144,12 +140,4 @@ private[thriftserver] class SparkSQLOperationManager()
144140
logDebug(s"Created GetFunctionsOperation with session=$parentSession.")
145141
operation
146142
}
147-
148-
def setConfMap(conf: SQLConf, confMap: java.util.Map[String, String]): Unit = {
149-
val iterator = confMap.entrySet().iterator()
150-
while (iterator.hasNext) {
151-
val kv = iterator.next()
152-
conf.setConfString(kv.getKey, kv.getValue)
153-
}
154-
}
155143
}

sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,10 +144,17 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
144144
def executeTest(hiveList: String): Unit = {
145145
hiveList.split(";").foreach{ m =>
146146
val kv = m.split("=")
147-
// select "${a}"; ---> avalue
148-
val resultSet = statement.executeQuery("select \"${" + kv(0) + "}\"")
147+
val k = kv(0)
148+
val v = kv(1)
149+
val modValue = s"${v}_MOD_VALUE"
150+
// select '${a}'; ---> avalue
151+
val resultSet = statement.executeQuery(s"select '$${$k}'")
149152
resultSet.next()
150-
assert(resultSet.getString(1) === kv(1))
153+
assert(resultSet.getString(1) === v)
154+
statement.executeQuery(s"set $k=$modValue")
155+
val modResultSet = statement.executeQuery(s"select '$${$k}'")
156+
modResultSet.next()
157+
assert(modResultSet.getString(1) === s"$modValue")
151158
}
152159
}
153160
}

0 commit comments

Comments
 (0)