Do not set fileSystem globally to hdfs

tylerwmarrs · Aug 10, 2017 · 54a2346 · 54a2346
1 parent c739f88
commit 54a2346
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 5 deletions.
diff --git a/Code/MRS/2-Train-Test.r b/Code/MRS/2-Train-Test.r
@@ -42,7 +42,7 @@ trainDS <- RxHiveData(table = "flightsweathertrain", colInfo = colInfoFull)
 testDS <- RxHiveData(table = "flightsweathertest", colInfo = colInfoFull)
 
 # save the test data as XDF
-airWeatherTestXdf <- RxXdfData(file.path(dataDir, "airWeatherTestXdf"))
+airWeatherTestXdf <- RxXdfData(file.path(dataDir, "airWeatherTestXdf"), fileSystem = hdfs)
 rxDataStep(inData = testDS, outFile = airWeatherTestXdf, overwrite = TRUE)
 
 ################################################
@@ -67,7 +67,7 @@ save(logitModel, file = "logitModelSubset.RData")
 
 # Predict over test data (Logistic Regression).
 
-logitPredict <- RxXdfData(file.path(dataDir, "logitPredictSubset"))
+logitPredict <- RxXdfData(file.path(dataDir, "logitPredictSubset"), fileSystem = hdfs)
 
 # Use the scalable rxPredict() function
 
@@ -99,7 +99,7 @@ summary(fastTreesEnsembleModel)
 save(fastTreesEnsembleModel, file = "fastTreesEnsembleModelSubset.RData")
 
 # Test
-fastTreesEnsemblePredict <- RxXdfData(file.path(dataDir, "fastTreesEnsemblePredictSubset"))
+fastTreesEnsemblePredict <- RxXdfData(file.path(dataDir, "fastTreesEnsemblePredictSubset"), fileSystem = hdfs)
 
 # Experimental feature to parallelize rxPredict when using a MicrosoftML model
 assign("predictMethod", "useDataStep", envir = MicrosoftML:::rxHashEnv)

diff --git a/Code/MRS/3-Deploy-Score.r b/Code/MRS/3-Deploy-Score.r
@@ -14,7 +14,7 @@ rxSetComputeContext("local")
 load("logitModelSubset.RData") # loads logitModel
 
 # Reference the test data to be scored
-airWeatherTestXdf <- RxXdfData( file.path(dataDir, "airWeatherTestXdf") )
+airWeatherTestXdf <- RxXdfData(file.path(dataDir, "airWeatherTestXdf"), fileSystem = hdfs)
 
 # Read the first 6 rows and remove the ArrDel15 column
 dataToBeScored <- base::subset(head(airWeatherTestXdf), select = -ArrDel15)

diff --git a/Code/MRS/SetComputeContext.r b/Code/MRS/SetComputeContext.r
@@ -19,7 +19,7 @@ if(file.exists("/dsvm"))
 # N.B. Can be used with local or RxSpark compute contexts
 ################################################
 
-rxOptions(fileSystem = RxHdfsFileSystem())
+hdfs <- RxHdfsFileSystem()
 
 dataDir <- "/user/RevoShare/remoteuser/Data"