Remove strong dependency on PySpark

pystellardb/stellar_hive.py

@@ -305,7 +305,7 @@ def _convertData(self, type, data):
         elif type == 'int':
             return int(data)
         elif type == 'long':
-            return long(data)
+            return int(data)
         elif type == 'float' or type == 'double':
             return float(data)
         elif type == 'CruxType:Node' or type == 'GraphNode':

pystellardb/stellar_rdd.py

@@ -6,8 +6,13 @@
 import abc
 from future.utils import with_metaclass
 import logging
-from pyspark import RDD, SparkContext
-from pyspark.serializers import BatchedSerializer
+
+try:
+    import pyspark
+    from pyspark import RDD, SparkContext
+    from pyspark.serializers import BatchedSerializer
+except ImportError:
+    pyspark = None
 
 _logger = logging.getLogger(__name__)
 
@@ -20,6 +25,9 @@ def transformToRDD(cursor, sc, parallelism=1):
     param sc: SparkContext
     param parallelism: Parallelism of RDD
     """
+    if not pyspark:
+        raise ImportError("Could not import pyspark! Please run `pip install pyspark` first in your environment!")
+
     # Get all data from cursor
     data = cursor.fetchall()
 

setup.py

@@ -47,14 +47,14 @@
         'pyhive',
         'sasl',
         'thrift',
-        'thrift-sasl>=0.3.0',
-        'pyspark>=2.4.0'
+        'thrift-sasl>=0.3.0'
     ],
     extras_require={
         'presto': ['requests>=1.0.0'],
         'hive': ['sasl>=0.2.1', 'thrift>=0.10.0'],
         'sqlalchemy': ['sqlalchemy>=1.3.0'],
         'kerberos': ['requests_kerberos>=0.12.0'],
+        'pyspark': ['pyspark>=2.4.0']
     },
     tests_require=[
         'mock>=1.0.0',