Added configuration to make test environment and outcomes consistent

dev/build.sh

@@ -21,5 +21,6 @@ env | tee "target/buildinfo"
 pushd  ${FWDIR}/python
 python setup.py sdist
 python setup.py bdist_egg
+python -m unittest discover -v
 popd 
 

dev/dev-requirements.txt

@@ -1,4 +1,5 @@
 Sphinx>=1.3.5
 pylint>=1.7.4
 decorator>=4.1.2
-typedecorator
+pyspark>=2.2.1
+typedecorator>=0.0.5

python/variants/test/test_core.py

@@ -1,14 +1,31 @@
 import os
 import unittest
 
+from pyspark import SparkConf
 from pyspark.sql import SparkSession
-from pyspark.tests import ReusedPySparkTestCase
 
 from variants import VariantsContext
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 
-class VariantSparkAPITestCase(ReusedPySparkTestCase):
+
+class VariantSparkPySparkTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(self):
+        sconf = SparkConf(loadDefaults=False)\
+            .set("spark.sql.files.openCostInBytes", 53687091200L)\
+            .set("spark.sql.files.maxPartitionBytes", 53687091200L)
+        spark = SparkSession.builder.config(conf=sconf)\
+            .appName("test").master("local").getOrCreate()
+        self.sc = spark.sparkContext
+
+    @classmethod
+    def tearDownClass(self):
+        self.sc.stop()
+
+
+class VariantSparkAPITestCase(VariantSparkPySparkTestCase):
 
     def setUp(self):
         self.spark = SparkSession(self.sc)
@@ -34,7 +51,7 @@ def test_importance_analysis_from_vcf(self):
         self.assertEqual('22_16050408',
                          str(df.orderBy('importance', ascending=False).collect()[0][0]))
         oob_error = imp_analysis.oob_error()
-        self.assertAlmostEqual(0.014652014652014652, oob_error, 3)
+        self.assertAlmostEqual(0.016483516483516484, oob_error, 4)
 
 
 if __name__ == '__main__':