Skip to content

Commit c34b546

Browse files
felixcheungshivaram
felixcheung
authored and
shivaram
committed
[SPARK-16519][SPARKR] Handle SparkR RDD generics that create warnings in R CMD check
## What changes were proposed in this pull request? Rename RDD functions for now to avoid CRAN check warnings. Some RDD functions are sharing generics with DataFrame functions (hence the problem) so after the renames we need to add new generics, for now. ## How was this patch tested? unit tests Author: Felix Cheung <felixcheung_m@hotmail.com> Closes #14626 from felixcheung/rrddfunctions.
1 parent d37ea3c commit c34b546

17 files changed

+312
-287
lines changed

‎R/pkg/R/RDD.R

Lines changed: 50 additions & 50 deletions
Large diffs are not rendered by default.

‎R/pkg/R/SQLContext.R

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ createDataFrame.default <- function(data, schema = NULL, samplingRatio = 1.0) {
218218
}
219219

220220
if (is.null(schema) || (!inherits(schema, "structType") && is.null(names(schema)))) {
221-
row <- first(rdd)
221+
row <- firstRDD(rdd)
222222
names <- if (is.null(schema)) {
223223
names(row)
224224
} else {

‎R/pkg/R/context.R

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ spark.lapply <- function(list, func) {
267267
sc <- getSparkContext()
268268
rdd <- parallelize(sc, list, length(list))
269269
results <- map(rdd, func)
270-
local <- collect(results)
270+
local <- collectRDD(results)
271271
local
272272
}
273273

‎R/pkg/R/generics.R

Lines changed: 58 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,7 @@
2323
setGeneric("aggregateRDD",
2424
function(x, zeroValue, seqOp, combOp) { standardGeneric("aggregateRDD") })
2525

26-
# @rdname cache-methods
27-
# @export
28-
setGeneric("cache", function(x) { standardGeneric("cache") })
26+
setGeneric("cacheRDD", function(x) { standardGeneric("cacheRDD") })
2927

3028
# @rdname coalesce
3129
# @seealso repartition
@@ -36,9 +34,7 @@ setGeneric("coalesce", function(x, numPartitions, ...) { standardGeneric("coales
3634
# @export
3735
setGeneric("checkpoint", function(x) { standardGeneric("checkpoint") })
3836

39-
# @rdname collect-methods
40-
# @export
41-
setGeneric("collect", function(x, ...) { standardGeneric("collect") })
37+
setGeneric("collectRDD", function(x, ...) { standardGeneric("collectRDD") })
4238

4339
# @rdname collect-methods
4440
# @export
@@ -51,9 +47,9 @@ setGeneric("collectPartition",
5147
standardGeneric("collectPartition")
5248
})
5349

54-
# @rdname nrow
55-
# @export
56-
setGeneric("count", function(x) { standardGeneric("count") })
50+
setGeneric("countRDD", function(x) { standardGeneric("countRDD") })
51+
52+
setGeneric("lengthRDD", function(x) { standardGeneric("lengthRDD") })
5753

5854
# @rdname countByValue
5955
# @export
@@ -74,17 +70,13 @@ setGeneric("approxQuantile",
7470
standardGeneric("approxQuantile")
7571
})
7672

77-
# @rdname distinct
78-
# @export
79-
setGeneric("distinct", function(x, numPartitions = 1) { standardGeneric("distinct") })
73+
setGeneric("distinctRDD", function(x, numPartitions = 1) { standardGeneric("distinctRDD") })
8074

8175
# @rdname filterRDD
8276
# @export
8377
setGeneric("filterRDD", function(x, f) { standardGeneric("filterRDD") })
8478

85-
# @rdname first
86-
# @export
87-
setGeneric("first", function(x, ...) { standardGeneric("first") })
79+
setGeneric("firstRDD", function(x, ...) { standardGeneric("firstRDD") })
8880

8981
# @rdname flatMap
9082
# @export
@@ -110,6 +102,8 @@ setGeneric("glom", function(x) { standardGeneric("glom") })
110102
# @export
111103
setGeneric("histogram", function(df, col, nbins=10) { standardGeneric("histogram") })
112104

105+
setGeneric("joinRDD", function(x, y, ...) { standardGeneric("joinRDD") })
106+
113107
# @rdname keyBy
114108
# @export
115109
setGeneric("keyBy", function(x, func) { standardGeneric("keyBy") })
@@ -152,9 +146,7 @@ setGeneric("getNumPartitions", function(x) { standardGeneric("getNumPartitions")
152146
# @export
153147
setGeneric("numPartitions", function(x) { standardGeneric("numPartitions") })
154148

155-
# @rdname persist
156-
# @export
157-
setGeneric("persist", function(x, newLevel) { standardGeneric("persist") })
149+
setGeneric("persistRDD", function(x, newLevel) { standardGeneric("persistRDD") })
158150

159151
# @rdname pipeRDD
160152
# @export
@@ -168,10 +160,7 @@ setGeneric("pivot", function(x, colname, values = list()) { standardGeneric("piv
168160
# @export
169161
setGeneric("reduce", function(x, func) { standardGeneric("reduce") })
170162

171-
# @rdname repartition
172-
# @seealso coalesce
173-
# @export
174-
setGeneric("repartition", function(x, ...) { standardGeneric("repartition") })
163+
setGeneric("repartitionRDD", function(x, ...) { standardGeneric("repartitionRDD") })
175164

176165
# @rdname sampleRDD
177166
# @export
@@ -193,16 +182,16 @@ setGeneric("saveAsTextFile", function(x, path) { standardGeneric("saveAsTextFile
193182
# @export
194183
setGeneric("setName", function(x, name) { standardGeneric("setName") })
195184

185+
setGeneric("showRDD", function(object, ...) { standardGeneric("showRDD") })
186+
196187
# @rdname sortBy
197188
# @export
198189
setGeneric("sortBy",
199190
function(x, func, ascending = TRUE, numPartitions = 1) {
200191
standardGeneric("sortBy")
201192
})
202193

203-
# @rdname take
204-
# @export
205-
setGeneric("take", function(x, num) { standardGeneric("take") })
194+
setGeneric("takeRDD", function(x, num) { standardGeneric("takeRDD") })
206195

207196
# @rdname takeOrdered
208197
# @export
@@ -223,9 +212,7 @@ setGeneric("top", function(x, num) { standardGeneric("top") })
223212
# @export
224213
setGeneric("unionRDD", function(x, y) { standardGeneric("unionRDD") })
225214

226-
# @rdname unpersist-methods
227-
# @export
228-
setGeneric("unpersist", function(x, ...) { standardGeneric("unpersist") })
215+
setGeneric("unpersistRDD", function(x, ...) { standardGeneric("unpersistRDD") })
229216

230217
# @rdname zipRDD
231218
# @export
@@ -343,9 +330,7 @@ setGeneric("join", function(x, y, ...) { standardGeneric("join") })
343330
# @export
344331
setGeneric("leftOuterJoin", function(x, y, numPartitions) { standardGeneric("leftOuterJoin") })
345332

346-
#' @rdname partitionBy
347-
#' @export
348-
setGeneric("partitionBy", function(x, ...) { standardGeneric("partitionBy") })
333+
setGeneric("partitionByRDD", function(x, ...) { standardGeneric("partitionByRDD") })
349334

350335
# @rdname reduceByKey
351336
# @seealso groupByKey
@@ -414,6 +399,14 @@ setGeneric("as.data.frame",
414399
#' @export
415400
setGeneric("attach")
416401

402+
#' @rdname cache
403+
#' @export
404+
setGeneric("cache", function(x) { standardGeneric("cache") })
405+
406+
#' @rdname collect
407+
#' @export
408+
setGeneric("collect", function(x, ...) { standardGeneric("collect") })
409+
417410
#' @rdname columns
418411
#' @export
419412
setGeneric("colnames", function(x, do.NULL = TRUE, prefix = "col") { standardGeneric("colnames") })
@@ -434,6 +427,10 @@ setGeneric("coltypes<-", function(x, value) { standardGeneric("coltypes<-") })
434427
#' @export
435428
setGeneric("columns", function(x) {standardGeneric("columns") })
436429

430+
#' @rdname nrow
431+
#' @export
432+
setGeneric("count", function(x) { standardGeneric("count") })
433+
437434
#' @rdname cov
438435
#' @export
439436
setGeneric("cov", function(x, ...) {standardGeneric("cov") })
@@ -477,6 +474,10 @@ setGeneric("gapplyCollect", function(x, ...) { standardGeneric("gapplyCollect")
477474
#' @export
478475
setGeneric("describe", function(x, col, ...) { standardGeneric("describe") })
479476

477+
#' @rdname distinct
478+
#' @export
479+
setGeneric("distinct", function(x) { standardGeneric("distinct") })
480+
480481
#' @rdname drop
481482
#' @export
482483
setGeneric("drop", function(x, ...) { standardGeneric("drop") })
@@ -519,6 +520,10 @@ setGeneric("fillna", function(x, value, cols = NULL) { standardGeneric("fillna")
519520
#' @export
520521
setGeneric("filter", function(x, condition) { standardGeneric("filter") })
521522

523+
#' @rdname first
524+
#' @export
525+
setGeneric("first", function(x, ...) { standardGeneric("first") })
526+
522527
#' @rdname groupBy
523528
#' @export
524529
setGeneric("group_by", function(x, ...) { standardGeneric("group_by") })
@@ -555,17 +560,25 @@ setGeneric("mutate", function(.data, ...) {standardGeneric("mutate") })
555560
#' @export
556561
setGeneric("orderBy", function(x, col, ...) { standardGeneric("orderBy") })
557562

563+
#' @rdname persist
564+
#' @export
565+
setGeneric("persist", function(x, newLevel) { standardGeneric("persist") })
566+
558567
#' @rdname printSchema
559568
#' @export
560569
setGeneric("printSchema", function(x) { standardGeneric("printSchema") })
561570

571+
#' @rdname registerTempTable-deprecated
572+
#' @export
573+
setGeneric("registerTempTable", function(x, tableName) { standardGeneric("registerTempTable") })
574+
562575
#' @rdname rename
563576
#' @export
564577
setGeneric("rename", function(x, ...) { standardGeneric("rename") })
565578

566-
#' @rdname registerTempTable-deprecated
579+
#' @rdname repartition
567580
#' @export
568-
setGeneric("registerTempTable", function(x, tableName) { standardGeneric("registerTempTable") })
581+
setGeneric("repartition", function(x, ...) { standardGeneric("repartition") })
569582

570583
#' @rdname sample
571584
#' @export
@@ -592,6 +605,10 @@ setGeneric("saveAsTable", function(df, tableName, source = NULL, mode = "error",
592605
#' @export
593606
setGeneric("str")
594607

608+
#' @rdname take
609+
#' @export
610+
setGeneric("take", function(x, num) { standardGeneric("take") })
611+
595612
#' @rdname mutate
596613
#' @export
597614
setGeneric("transform", function(`_data`, ...) {standardGeneric("transform") })
@@ -674,6 +691,10 @@ setGeneric("union", function(x, y) { standardGeneric("union") })
674691
#' @export
675692
setGeneric("unionAll", function(x, y) { standardGeneric("unionAll") })
676693

694+
#' @rdname unpersist-methods
695+
#' @export
696+
setGeneric("unpersist", function(x, ...) { standardGeneric("unpersist") })
697+
677698
#' @rdname filter
678699
#' @export
679700
setGeneric("where", function(x, condition) { standardGeneric("where") })
@@ -771,6 +792,10 @@ setGeneric("over", function(x, window) { standardGeneric("over") })
771792

772793
###################### WindowSpec Methods ##########################
773794

795+
#' @rdname partitionBy
796+
#' @export
797+
setGeneric("partitionBy", function(x, ...) { standardGeneric("partitionBy") })
798+
774799
#' @rdname rowsBetween
775800
#' @export
776801
setGeneric("rowsBetween", function(x, start, end) { standardGeneric("rowsBetween") })

0 commit comments

Comments
 (0)