Cleanup and add an env

Author: HyukjinKwon

python/pyspark/install.py

@@ -100,7 +100,10 @@ def install_spark(dest, spark_version, hadoop_version, hive_version):
 
     package_name = checked_package_name(spark_version, hadoop_version, hive_version)
     package_local_path = os.path.join(dest, "%s.tgz" % package_name)
-    sites = get_preferred_mirrors()
+    if "PYSPARK_RELEASE_MIRROR" in os.environ:
+        sites = [os.environ["PYSPARK_RELEASE_MIRROR"]]
+    else:
+        sites = get_preferred_mirrors()
     print("Trying to download Spark %s from [%s]" % (spark_version, ", ".join(sites)))
 
     pretty_pkg_name = "%s for Hadoop %s" % (

python/setup.py

@@ -120,6 +120,11 @@ class InstallCommand(install):
 
     def run(self):
         install.run(self)
+
+        # Make sure the destination is always clean.
+        spark_dist = os.path.join(self.install_lib, "pyspark", "spark-distribution")
+        rmtree(spark_dist, ignore_errors=True)
+
         if ("HADOOP_VERSION" in os.environ) or ("HIVE_VERSION" in os.environ):
             # Note that SPARK_VERSION environment is just a testing purpose.
             spark_version, hadoop_version, hive_version = install_module.checked_versions(
@@ -134,7 +139,7 @@ def run(self):
                 return
 
             install_module.install_spark(
-                dest=os.path.join(self.install_lib, "pyspark", "spark-distribution"),
+                dest=spark_dist,
                 spark_version=spark_version,
                 hadoop_version=hadoop_version,
                 hive_version=hive_version)