Skip to content

Commit d17cb2b

Browse files
jkbradleymengxr
jkbradley
authored and
mengxr
committed
[SPARK-4587] [mllib] [docs] Fixed save,load calls in ML guide examples
Should pass spark context to save/load CC: mengxr Author: Joseph K. Bradley <joseph@databricks.com> Closes #4816 from jkbradley/ml-io-doc-fix and squashes the following commits: 83d369d [Joseph K. Bradley] added comment to save,load parts of ML guide examples 2841170 [Joseph K. Bradley] Fixed save,load calls in ML guide examples
1 parent 57566d0 commit d17cb2b

File tree

5 files changed

+60
-40
lines changed

5 files changed

+60
-40
lines changed

‎docs/mllib-collaborative-filtering.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,9 @@ val MSE = ratesAndPreds.map { case ((user, product), (r1, r2)) =>
9797
}.mean()
9898
println("Mean Squared Error = " + MSE)
9999

100-
model.save("myModelPath")
101-
val sameModel = MatrixFactorizationModel.load("myModelPath")
100+
// Save and load model
101+
model.save(sc, "myModelPath")
102+
val sameModel = MatrixFactorizationModel.load(sc, "myModelPath")
102103
{% endhighlight %}
103104

104105
If the rating matrix is derived from another source of information (e.g., it is inferred from
@@ -186,8 +187,9 @@ public class CollaborativeFiltering {
186187
).rdd()).mean();
187188
System.out.println("Mean Squared Error = " + MSE);
188189

189-
model.save("myModelPath");
190-
MatrixFactorizationModel sameModel = MatrixFactorizationModel.load("myModelPath");
190+
// Save and load model
191+
model.save(sc.sc(), "myModelPath");
192+
MatrixFactorizationModel sameModel = MatrixFactorizationModel.load(sc.sc(), "myModelPath");
191193
}
192194
}
193195
{% endhighlight %}

‎docs/mllib-decision-tree.md

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -223,8 +223,9 @@ val testErr = labelAndPreds.filter(r => r._1 != r._2).count.toDouble / testData.
223223
println("Test Error = " + testErr)
224224
println("Learned classification tree model:\n" + model.toDebugString)
225225

226-
model.save("myModelPath")
227-
val sameModel = DecisionTreeModel.load("myModelPath")
226+
// Save and load model
227+
model.save(sc, "myModelPath")
228+
val sameModel = DecisionTreeModel.load(sc, "myModelPath")
228229
{% endhighlight %}
229230
</div>
230231

@@ -284,8 +285,9 @@ Double testErr =
284285
System.out.println("Test Error: " + testErr);
285286
System.out.println("Learned classification tree model:\n" + model.toDebugString());
286287

287-
model.save("myModelPath");
288-
DecisionTreeModel sameModel = DecisionTreeModel.load("myModelPath");
288+
// Save and load model
289+
model.save(sc.sc(), "myModelPath");
290+
DecisionTreeModel sameModel = DecisionTreeModel.load(sc.sc(), "myModelPath");
289291
{% endhighlight %}
290292
</div>
291293

@@ -362,8 +364,9 @@ val testMSE = labelsAndPredictions.map{ case(v, p) => math.pow((v - p), 2)}.mean
362364
println("Test Mean Squared Error = " + testMSE)
363365
println("Learned regression tree model:\n" + model.toDebugString)
364366

365-
model.save("myModelPath")
366-
val sameModel = DecisionTreeModel.load("myModelPath")
367+
// Save and load model
368+
model.save(sc, "myModelPath")
369+
val sameModel = DecisionTreeModel.load(sc, "myModelPath")
367370
{% endhighlight %}
368371
</div>
369372

@@ -429,8 +432,9 @@ Double testMSE =
429432
System.out.println("Test Mean Squared Error: " + testMSE);
430433
System.out.println("Learned regression tree model:\n" + model.toDebugString());
431434

432-
model.save("myModelPath");
433-
DecisionTreeModel sameModel = DecisionTreeModel.load("myModelPath");
435+
// Save and load model
436+
model.save(sc.sc(), "myModelPath");
437+
DecisionTreeModel sameModel = DecisionTreeModel.load(sc.sc(), "myModelPath");
434438
{% endhighlight %}
435439
</div>
436440

‎docs/mllib-ensembles.md

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -129,8 +129,9 @@ val testErr = labelAndPreds.filter(r => r._1 != r._2).count.toDouble / testData.
129129
println("Test Error = " + testErr)
130130
println("Learned classification forest model:\n" + model.toDebugString)
131131

132-
model.save("myModelPath")
133-
val sameModel = RandomForestModel.load("myModelPath")
132+
// Save and load model
133+
model.save(sc, "myModelPath")
134+
val sameModel = RandomForestModel.load(sc, "myModelPath")
134135
{% endhighlight %}
135136
</div>
136137

@@ -193,8 +194,9 @@ Double testErr =
193194
System.out.println("Test Error: " + testErr);
194195
System.out.println("Learned classification forest model:\n" + model.toDebugString());
195196

196-
model.save("myModelPath");
197-
RandomForestModel sameModel = RandomForestModel.load("myModelPath");
197+
// Save and load model
198+
model.save(sc.sc(), "myModelPath");
199+
RandomForestModel sameModel = RandomForestModel.load(sc.sc(), "myModelPath");
198200
{% endhighlight %}
199201
</div>
200202

@@ -276,8 +278,9 @@ val testMSE = labelsAndPredictions.map{ case(v, p) => math.pow((v - p), 2)}.mean
276278
println("Test Mean Squared Error = " + testMSE)
277279
println("Learned regression forest model:\n" + model.toDebugString)
278280

279-
model.save("myModelPath")
280-
val sameModel = RandomForestModel.load("myModelPath")
281+
// Save and load model
282+
model.save(sc, "myModelPath")
283+
val sameModel = RandomForestModel.load(sc, "myModelPath")
281284
{% endhighlight %}
282285
</div>
283286

@@ -343,8 +346,9 @@ Double testMSE =
343346
System.out.println("Test Mean Squared Error: " + testMSE);
344347
System.out.println("Learned regression forest model:\n" + model.toDebugString());
345348

346-
model.save("myModelPath");
347-
RandomForestModel sameModel = RandomForestModel.load("myModelPath");
349+
// Save and load model
350+
model.save(sc.sc(), "myModelPath");
351+
RandomForestModel sameModel = RandomForestModel.load(sc.sc(), "myModelPath");
348352
{% endhighlight %}
349353
</div>
350354

@@ -504,8 +508,9 @@ val testErr = labelAndPreds.filter(r => r._1 != r._2).count.toDouble / testData.
504508
println("Test Error = " + testErr)
505509
println("Learned classification GBT model:\n" + model.toDebugString)
506510

507-
model.save("myModelPath")
508-
val sameModel = GradientBoostedTreesModel.load("myModelPath")
511+
// Save and load model
512+
model.save(sc, "myModelPath")
513+
val sameModel = GradientBoostedTreesModel.load(sc, "myModelPath")
509514
{% endhighlight %}
510515
</div>
511516

@@ -568,8 +573,9 @@ Double testErr =
568573
System.out.println("Test Error: " + testErr);
569574
System.out.println("Learned classification GBT model:\n" + model.toDebugString());
570575

571-
model.save("myModelPath");
572-
GradientBoostedTreesModel sameModel = GradientBoostedTreesModel.load("myModelPath");
576+
// Save and load model
577+
model.save(sc.sc(), "myModelPath");
578+
GradientBoostedTreesModel sameModel = GradientBoostedTreesModel.load(sc.sc(), "myModelPath");
573579
{% endhighlight %}
574580
</div>
575581

@@ -647,8 +653,9 @@ val testMSE = labelsAndPredictions.map{ case(v, p) => math.pow((v - p), 2)}.mean
647653
println("Test Mean Squared Error = " + testMSE)
648654
println("Learned regression GBT model:\n" + model.toDebugString)
649655

650-
model.save("myModelPath")
651-
val sameModel = GradientBoostedTreesModel.load("myModelPath")
656+
// Save and load model
657+
model.save(sc, "myModelPath")
658+
val sameModel = GradientBoostedTreesModel.load(sc, "myModelPath")
652659
{% endhighlight %}
653660
</div>
654661

@@ -717,8 +724,9 @@ Double testMSE =
717724
System.out.println("Test Mean Squared Error: " + testMSE);
718725
System.out.println("Learned regression GBT model:\n" + model.toDebugString());
719726

720-
model.save("myModelPath");
721-
GradientBoostedTreesModel sameModel = GradientBoostedTreesModel.load("myModelPath");
727+
// Save and load model
728+
model.save(sc.sc(), "myModelPath");
729+
GradientBoostedTreesModel sameModel = GradientBoostedTreesModel.load(sc.sc(), "myModelPath");
722730
{% endhighlight %}
723731
</div>
724732

‎docs/mllib-linear-methods.md

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -223,8 +223,9 @@ val auROC = metrics.areaUnderROC()
223223

224224
println("Area under ROC = " + auROC)
225225

226-
model.save("myModelPath")
227-
val sameModel = SVMModel.load("myModelPath")
226+
// Save and load model
227+
model.save(sc, "myModelPath")
228+
val sameModel = SVMModel.load(sc, "myModelPath")
228229
{% endhighlight %}
229230

230231
The `SVMWithSGD.train()` method by default performs L2 regularization with the
@@ -308,8 +309,9 @@ public class SVMClassifier {
308309

309310
System.out.println("Area under ROC = " + auROC);
310311

311-
model.save("myModelPath");
312-
SVMModel sameModel = SVMModel.load("myModelPath");
312+
// Save and load model
313+
model.save(sc.sc(), "myModelPath");
314+
SVMModel sameModel = SVMModel.load(sc.sc(), "myModelPath");
313315
}
314316
}
315317
{% endhighlight %}
@@ -423,8 +425,9 @@ val valuesAndPreds = parsedData.map { point =>
423425
val MSE = valuesAndPreds.map{case(v, p) => math.pow((v - p), 2)}.mean()
424426
println("training Mean Squared Error = " + MSE)
425427

426-
model.save("myModelPath")
427-
val sameModel = LinearRegressionModel.load("myModelPath")
428+
// Save and load model
429+
model.save(sc, "myModelPath")
430+
val sameModel = LinearRegressionModel.load(sc, "myModelPath")
428431
{% endhighlight %}
429432

430433
[`RidgeRegressionWithSGD`](api/scala/index.html#org.apache.spark.mllib.regression.RidgeRegressionWithSGD)
@@ -496,8 +499,9 @@ public class LinearRegression {
496499
).rdd()).mean();
497500
System.out.println("training Mean Squared Error = " + MSE);
498501

499-
model.save("myModelPath");
500-
LinearRegressionModel sameModel = LinearRegressionModel.load("myModelPath");
502+
// Save and load model
503+
model.save(sc.sc(), "myModelPath");
504+
LinearRegressionModel sameModel = LinearRegressionModel.load(sc.sc(), "myModelPath");
501505
}
502506
}
503507
{% endhighlight %}

‎docs/mllib-naive-bayes.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,9 @@ val model = NaiveBayes.train(training, lambda = 1.0)
5656
val predictionAndLabel = test.map(p => (model.predict(p.features), p.label))
5757
val accuracy = 1.0 * predictionAndLabel.filter(x => x._1 == x._2).count() / test.count()
5858

59-
model.save("myModelPath")
60-
val sameModel = NaiveBayesModel.load("myModelPath")
59+
// Save and load model
60+
model.save(sc, "myModelPath")
61+
val sameModel = NaiveBayesModel.load(sc, "myModelPath")
6162
{% endhighlight %}
6263
</div>
6364

@@ -97,8 +98,9 @@ double accuracy = predictionAndLabel.filter(new Function<Tuple2<Double, Double>,
9798
}
9899
}).count() / (double) test.count();
99100

100-
model.save("myModelPath");
101-
NaiveBayesModel sameModel = NaiveBayesModel.load("myModelPath");
101+
// Save and load model
102+
model.save(sc.sc(), "myModelPath");
103+
NaiveBayesModel sameModel = NaiveBayesModel.load(sc.sc(), "myModelPath");
102104
{% endhighlight %}
103105
</div>
104106

0 commit comments

Comments
 (0)