From f2ba4523f87d7cff2a559fb50839cd5fb64d84dc Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 10 Apr 2021 02:45:45 +0800 Subject: [PATCH] [KYUUBI #502][SPARK-35002][INFRA] Fix the java.net.BindException when testing with Github Action ![turboFei](https://badgen.net/badge/Hello/turboFei/green) [![Closes #503](https://badgen.net/badge/Preview/Closes%20%23503/blue)](https://github.com/yaooqinn/kyuubi/pull/503) ![6](https://badgen.net/badge/%2B/6/red) ![1](https://badgen.net/badge/-/1/green) ![10](https://badgen.net/badge/commits/10/yellow) ![Test Plan](https://badgen.net/badge/Missing/Test%20Plan/ff0000) [Powered by Pull Request Badge](https://pullrequestbadge.com/?utm_medium=github&utm_source=yaooqinn&utm_campaign=badge_info) ### _Why are the changes needed?_ Refer https://github.com/apache/spark/pull/32096, https://github.com/apache/spark/pull/32102, this PR tries to fix the java.net.BindException when testing with Github Action. ``` SparkOperationSuite: *** RUN ABORTED *** java.net.BindException: Cannot assign requested address: Service 'sparkDriver' failed after 16 retries (on a random free port)! Consider explicitly setting the appropriate binding address for the service 'sparkDriver' (for example spark.driver.bindAddress for SparkDriver) to the correct binding address. at sun.nio.ch.Net.bind0(Native Method) at sun.nio.ch.Net.bind(Net.java:461) at sun.nio.ch.Net.bind(Net.java:453) at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:222) at io.netty.channel.socket.nio.NioServerSocketChannel.doBind(NioServerSocketChannel.java:134) at io.netty.channel.AbstractChannel$AbstractUnsafe.bind(AbstractChannel.java:550) at io.netty.channel.DefaultChannelPipeline$HeadContext.bind(DefaultChannelPipeline.java:1334) at io.netty.channel.AbstractChannelHandlerContext.invokeBind(AbstractChannelHandlerContext.java:506) at io.netty.channel.AbstractChannelHandlerContext.bind(AbstractChannelHandlerContext.java:491) at io.netty.channel.DefaultChannelPipeline.bind(DefaultChannelPipeline.java:973) ``` Also transfer FRONTEND_BIND_HOST by connection string to fix similar issue. ``` Cause: java.lang.RuntimeException: org.apache.kyuubi.KyuubiSQLException:org.apache.kyuubi.KyuubiException: Failed to initialize frontend service on fv-az207-19/10.1.1.0:0. at org.apache.kyuubi.service.FrontendService.initialize(FrontendService.scala:102) at org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1(CompositeService.scala:40) at org.apache.kyuubi.service.CompositeService.$anonfun$initialize$1$adapted(CompositeService.scala:40) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.kyuubi.service.CompositeService.initialize(CompositeService.scala:40) at org.apache.kyuubi.service.Serverable.initialize(Serverable.scala:44) at org.apache.kyuubi.engine.spark.SparkSQLEngine.initialize(SparkSQLEngine.scala:49) at org.apache.kyuubi.engine.spark.SparkSQLEngine$.startEngine(SparkSQLEngine.scala:105) at org.apache.kyuubi.engine.spark.SparkSQLEngine$.main(SparkSQLEngine.scala:118) at org.apache.kyuubi.engine.spark.SparkSQLEngine.main(SparkSQLEngine.scala) ``` ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes #503 from turboFei/KYUUBI-502. Closes #502 1b10253a [fwang12] use localhost instead of 127.0.0.1 c104ce3f [fwang12] address comments 1e549c16 [fwang12] revert install shade 457ce2f5 [fwang12] try set frontend bind host in connection string 8bcd5a0d [fwang12] revert env KYUUBI_FRONTEND_BIND_HOST and set kyuubi.frontend.bind.host to 127.0.0.1 in scalatest-maven-plugin 717a992f [fwang12] update doc d5ba05a4 [fwang12] add install shaded jars in release.yml e8b23725 [fwang12] involve KYUUBI_FRONTEND_BIND_HOST 5eb7cdb2 [fwang12] also set KYUUBI_FRONTEND_BIND_HOST env to 127.0.0.1 7d708198 [fwang12] [KYUUBI #502][SPARK-35002][INFRA] Fix the java.net.BindException when testing with Github Action Authored-by: fwang12 Signed-off-by: Cheng Pan <379377944@qq.com> --- .github/workflows/master.yml | 2 ++ .../scala/org/apache/kyuubi/operation/WithKyuubiServer.scala | 4 +++- pom.xml | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 767db5ab289..e357e48bae5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -38,6 +38,8 @@ jobs: - '-Pspark-3.1' - '-Pspark-3.1 -Dhadoop.binary.version=3.2' - '-Pspark-3.2-snapshot -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.DataLakeTest' + env: + SPARK_LOCAL_IP: localhost steps: - uses: actions/checkout@v2 - name: Setup JDK 1.8 diff --git a/kyuubi-main/src/test/scala/org/apache/kyuubi/operation/WithKyuubiServer.scala b/kyuubi-main/src/test/scala/org/apache/kyuubi/operation/WithKyuubiServer.scala index 518ad8665b0..f3629bd5b23 100644 --- a/kyuubi-main/src/test/scala/org/apache/kyuubi/operation/WithKyuubiServer.scala +++ b/kyuubi-main/src/test/scala/org/apache/kyuubi/operation/WithKyuubiServer.scala @@ -72,5 +72,7 @@ trait WithKyuubiServer extends KyuubiFunSuite { super.afterAll() } - protected def getJdbcUrl: String = s"jdbc:hive2://${server.connectionUrl}/;" + // TODO: KYUUBI-504: forbid setting FRONTEND_BIND_HOST by connection string in engine side + protected def getJdbcUrl: String = + s"jdbc:hive2://${server.connectionUrl}/;#${KyuubiConf.FRONTEND_BIND_HOST.key}=localhost" } diff --git a/pom.xml b/pom.xml index 07c910a2137..3531df342ea 100644 --- a/pom.xml +++ b/pom.xml @@ -1169,6 +1169,7 @@ ${project.build.directory}/tmp 2g ${project.build.directory}/metrics + localhost ${maven.plugin.scalatest.exclude.tags}