cache AppClassLoader

sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala

@@ -89,6 +89,9 @@ class ArtifactManager(session: SparkSession) extends Logging {
 
   private var initialContextResourcesCopied = false
 
+  /** Cached generated classloader for the system classloader, aka., AppClassLoader. */
+  private var classLoaderForSystemClassLoader: Option[ClassLoader] = None
+
   /**
    * The number of JARs for the last [[classloader]] call, and all generated class loaders.
    * Value of the HashMap is not used.
@@ -311,15 +314,24 @@ class ArtifactManager(session: SparkSession) extends Logging {
     val thread = Thread.currentThread().toString
     val (lastNumberOfJars, classLoaderMap) = numberOfJarsAndCachedClassLoaders.get()
     if (lastNumberOfJars != jarsList.size()) {
-      // If the number of jars has changed, we need to generate new class loaders.
+      // If the number of jars has changed, clear the cache and generate new class loaders.
+      classLoaderForSystemClassLoader = None
       numberOfJarsAndCachedClassLoaders.remove()
       buildAndCacheClassLoader(fallbackClassLoader, urls)
     } else {
-      // If the fallback class loader is generated by us, we can reuse it to avoid layering.
       if (classLoaderMap.contains(fallbackClassLoader)) {
+        // If the fallback class loader is built by us, reuse it to avoid layering.
         fallbackClassLoader
+      } else if (fallbackClassLoader == ClassLoader.getSystemClassLoader) {
+        // If the fallback class loader is the system class loader, build a new one and cache it.
+        classLoaderForSystemClassLoader.getOrElse {
+          val loader = buildAndCacheClassLoader(fallbackClassLoader, urls)
+          classLoaderForSystemClassLoader = Some(loader)
+          loader
+        }
       } else {
-        // Otherwise we need to layer one new class loader.
+        logWarning(s"Generating new class loader for classloader $fallbackClassLoader. $thread")
+        // Otherwise build one new class loader.
         buildAndCacheClassLoader(fallbackClassLoader, urls)
       }
     }