Address nits and fix documentations to be more clear

Author: HyukjinKwon

python/docs/source/getting_started/installation.rst

@@ -48,7 +48,12 @@ For PySpark with different Hadoop and/or Hive, you can install it by using ``HIV
     HIVE_VERSION=1.2 HADOOP_VERSION=2.7 pip install pyspark
 
 The default distribution has built-in Hadoop 3.2 and Hive 2.3. If users specify different versions, the pip installation automatically
-downloads a different version and use it in PySpark.
+downloads a different version and use it in PySpark. Downloading it can take a while depending on the network and the mirror chosen.
+It is recommended to use `-v` option in `pip` to track the installation and download status.
+
+.. code-block:: bash
+
+    HADOOP_VERSION=2.7 pip install pyspark -v
 
 Supported versions are as below:
 
@@ -134,4 +139,4 @@ Package       Minimum supported version Note
 ============= ========================= ================
 
 **Note**: PySpark requires Java 8 or later with ``JAVA_HOME`` properly set.  
-If using JDK 11, set ``-Dio.netty.tryReflectionSetAccessible=true`` for Arrow related features and refer to `Downloading <https://spark.apache.org/docs/latest/#downloading>`_
+If using JDK 11, set ``-Dio.netty.tryReflectionSetAccessible=true`` for Arrow related features and refer to `Downloading <https://spark.apache.org/docs/latest/#downloading>`_

python/pyspark/find_spark_home.py

@@ -43,7 +43,7 @@ def is_spark_home(path):
     from importlib.util import find_spec
     try:
         # Spark distribution can be downloaded when HADOOP_VERSION environment variable is set.
-        # We should look up this directory first.
+        # We should look up this directory first, see also SPARK-32017.
         spark_dist_dir = "spark-distribution"
         module_home = os.path.dirname(find_spec("pyspark").origin)
         paths.append(os.path.join(module_home, spark_dist_dir))

python/pyspark/tests/test_install_spark.py

@@ -25,12 +25,11 @@
 class SparkInstallationTestCase(unittest.TestCase):
 
     def test_install_spark(self):
-        # Just pick one combination to test.
+        # Test only one case. Testing this is expensive because it needs to download
+        # the Spark distribution.
         spark_version, hadoop_version, hive_version = checked_versions("3.0.1", "3.2", "2.3")
 
         with tempfile.TemporaryDirectory() as tmp_dir:
-            # Test only default case. Testing this is expensive because it needs to download
-            # the Spark distribution.
             install_spark(
                 dest=tmp_dir,
                 spark_version=spark_version,

python/setup.py

@@ -126,8 +126,9 @@ def run(self):
                 os.environ.get("HADOOP_VERSION", install_module.DEFAULT_HADOOP).lower(),
                 os.environ.get("HIVE_VERSION", install_module.DEFAULT_HIVE).lower())
 
-            if ((install_module.DEFAULT_HADOOP, install_module.DEFAULT_HIVE) ==
-                    (hadoop_version, hive_version)):
+            if ("SPARK_VERSION" not in os.environ and
+                ((install_module.DEFAULT_HADOOP, install_module.DEFAULT_HIVE) ==
+                    (hadoop_version, hive_version))):
                 # Do not download and install if they are same as default.
                 return