use set-pythonpath only for impala-python

* Don't add PYTHONPATH to environment in impala-config.sh,
  it is done automatically by the impala-python script anyway.
  I think this is legacy from when we ran some things with
  the system python.
* Remove unnecessary set-pythonpath.sh invocations where all
  calls go via impala-python anyway.
* Remove impala-shell eggs from python path. All these packages
  are installed into the virtualenv.
* testdata path entry was not needed - it's imported via the root

Testing:
Ran core tests

Change-Id: Iff98eb261ab48c592e8d323aa409c6a65317b95a
Reviewed-on: http://gerrit.cloudera.org:8080/14238
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Tim Armstrong <tarmstrong@cloudera.com>

bin/impala-config.sh

@@ -735,9 +735,6 @@ export IMPALA_MAKE_FLAGS=${IMPALA_MAKE_FLAGS-}
 # Some environments (like the packaging build) might not have $USER set.  Fix that here.
 export USER="${USER-`id -un`}"
 
-# Configure python path
-. "$IMPALA_HOME/bin/set-pythonpath.sh"
-
 # These arguments are, despite the name, passed to every JVM created
 # by an impalad.
 # - Enable JNI check
@@ -803,7 +800,6 @@ echo "HADOOP_LZO              = $HADOOP_LZO"
 echo "IMPALA_LZO              = $IMPALA_LZO"
 echo "CLASSPATH               = $CLASSPATH"
 echo "LIBHDFS_OPTS            = $LIBHDFS_OPTS"
-echo "PYTHONPATH              = $PYTHONPATH"
 echo "JAVA_HOME               = $JAVA_HOME"
 echo "LD_LIBRARY_PATH         = $LD_LIBRARY_PATH"
 echo "LD_PRELOAD              = $LD_PRELOAD"

bin/impala-shell.sh

@@ -19,6 +19,5 @@
 
 
 # This script runs the impala shell from a dev environment.
-. ${IMPALA_HOME}/bin/set-pythonpath.sh
 SHELL_HOME=${IMPALA_SHELL_HOME:-${IMPALA_HOME}/shell}
 exec impala-python ${SHELL_HOME}/impala_shell.py "$@"

bin/run-all-tests.sh

@@ -25,8 +25,6 @@ set -euo pipefail
 . $IMPALA_HOME/bin/report_build_error.sh
 setup_report_build_error
 
-. "$IMPALA_HOME/bin/set-pythonpath.sh"
-
 # Allow picking up strategy from environment
 : ${EXPLORATION_STRATEGY:=core}
 : ${NUM_TEST_ITERATIONS:=1}

bin/set-pythonpath.sh

@@ -15,16 +15,21 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# set the python path for test modules and beeswax
-# setting USE_THRIFT11_GEN_PY will add Thrift 11 Python generated code rather than the
-# default Thrift Python code
-PYTHONPATH=${IMPALA_HOME}
+# Sets up the python path for impala-python. This is needed because tests and other
+# utility scripts depend on some modules external to infra/python/env.
+# TODO: we should try to reduce our reliance on PYTHONPATH if possible.
+#
+# Setting USE_THRIFT11_GEN_PY will add Thrift 11 Python generated code rather than the
+# default Thrift Python code.
+# Used to allow importing testdata, test, etc modules from other scripts.
+export PYTHONPATH=${IMPALA_HOME}
+
+# Generated Thrift files are used by tests and other scripts.
 if [ -n "${USE_THRIFT11_GEN_PY:-}" ]; then
   PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/shell/build/thrift-11-gen/gen-py
 else
   PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/shell/gen-py
 fi
-PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/testdata/
 
 # There should be just a single version of python that created the
 # site-packages directory. We find it by performing shell independent expansion
@@ -42,10 +47,3 @@ done
 
 # Add Hive after Thrift because Hive supplies its own Thrift modules
 PYTHONPATH=${PYTHONPATH}:${HIVE_HOME}/lib/py
-
-# Add all the built eggs to the python path
-for EGG in `find ${IMPALA_HOME}/shell/ext-py/ -name '*\.egg'`; do
-  PYTHONPATH=${PYTHONPATH}:${EGG}
-done
-
-export PYTHONPATH

buildall.sh

@@ -380,9 +380,6 @@ bootstrap_dependencies() {
     echo ">>> Downloading and extracting toolchain dependencies."
     "$IMPALA_HOME/bin/bootstrap_toolchain.py"
     echo "Toolchain bootstrap complete."
-
-    # Reset python path to include thrift
-    . "$IMPALA_HOME/bin/set-pythonpath.sh"
   fi
 }
 

testdata/bin/run-hive-server.sh

@@ -21,8 +21,6 @@ set -euo pipefail
 . $IMPALA_HOME/bin/report_build_error.sh
 setup_report_build_error
 
-. ${IMPALA_HOME}/bin/set-pythonpath.sh
-
 HIVE_SERVER_PORT=10000
 export HIVE_SERVER2_THRIFT_PORT=11050
 HIVE_METASTORE_PORT=9083

tests/query_test/test_nested_types.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information