Skip to content

Commit edcde23

Browse files
felixcheungcmonkey
authored andcommitted
[SPARK-19130][SPARKR] Support setting literal value as column implicitly
## What changes were proposed in this pull request? ``` df$foo <- 1 ``` instead of ``` df$foo <- lit(1) ``` ## How was this patch tested? unit tests Author: Felix Cheung <felixcheung_m@hotmail.com> Closes apache#16510 from felixcheung/rlitcol.
1 parent 2887cdf commit edcde23

File tree

3 files changed

+39
-5
lines changed

3 files changed

+39
-5
lines changed

R/pkg/R/DataFrame.R

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1727,14 +1727,21 @@ setMethod("$", signature(x = "SparkDataFrame"),
17271727
getColumn(x, name)
17281728
})
17291729

1730-
#' @param value a Column or \code{NULL}. If \code{NULL}, the specified Column is dropped.
1730+
#' @param value a Column or an atomic vector in the length of 1 as literal value, or \code{NULL}.
1731+
#' If \code{NULL}, the specified Column is dropped.
17311732
#' @rdname select
17321733
#' @name $<-
17331734
#' @aliases $<-,SparkDataFrame-method
17341735
#' @note $<- since 1.4.0
17351736
setMethod("$<-", signature(x = "SparkDataFrame"),
17361737
function(x, name, value) {
1737-
stopifnot(class(value) == "Column" || is.null(value))
1738+
if (class(value) != "Column" && !is.null(value)) {
1739+
if (isAtomicLengthOne(value)) {
1740+
value <- lit(value)
1741+
} else {
1742+
stop("value must be a Column, literal value as atomic in length of 1, or NULL")
1743+
}
1744+
}
17381745

17391746
if (is.null(value)) {
17401747
nx <- drop(x, name)
@@ -1947,10 +1954,10 @@ setMethod("selectExpr",
19471954
#'
19481955
#' @param x a SparkDataFrame.
19491956
#' @param colName a column name.
1950-
#' @param col a Column expression.
1957+
#' @param col a Column expression, or an atomic vector in the length of 1 as literal value.
19511958
#' @return A SparkDataFrame with the new column added or the existing column replaced.
19521959
#' @family SparkDataFrame functions
1953-
#' @aliases withColumn,SparkDataFrame,character,Column-method
1960+
#' @aliases withColumn,SparkDataFrame,character-method
19541961
#' @rdname withColumn
19551962
#' @name withColumn
19561963
#' @seealso \link{rename} \link{mutate}
@@ -1963,11 +1970,16 @@ setMethod("selectExpr",
19631970
#' newDF <- withColumn(df, "newCol", df$col1 * 5)
19641971
#' # Replace an existing column
19651972
#' newDF2 <- withColumn(newDF, "newCol", newDF$col1)
1973+
#' newDF3 <- withColumn(newDF, "newCol", 42)
19661974
#' }
19671975
#' @note withColumn since 1.4.0
19681976
setMethod("withColumn",
1969-
signature(x = "SparkDataFrame", colName = "character", col = "Column"),
1977+
signature(x = "SparkDataFrame", colName = "character"),
19701978
function(x, colName, col) {
1979+
if (class(col) != "Column") {
1980+
if (!isAtomicLengthOne(col)) stop("Literal value must be atomic in length of 1")
1981+
col <- lit(col)
1982+
}
19711983
sdf <- callJMethod(x@sdf, "withColumn", colName, col@jc)
19721984
dataFrame(sdf)
19731985
})

R/pkg/R/utils.R

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -863,3 +863,7 @@ basenameSansExtFromUrl <- function(url) {
863863
# then, strip extension by the last '.'
864864
sub("([^.]+)\\.[[:alnum:]]+$", "\\1", filename)
865865
}
866+
867+
isAtomicLengthOne <- function(x) {
868+
is.atomic(x) && length(x) == 1
869+
}

R/pkg/inst/tests/testthat/test_sparkSQL.R

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1001,6 +1001,17 @@ test_that("select operators", {
10011001
expect_equal(columns(df), c("name", "age", "age2"))
10021002
expect_equal(count(where(df, df$age2 == df$age * 2)), 2)
10031003

1004+
df$age2 <- 21
1005+
expect_equal(columns(df), c("name", "age", "age2"))
1006+
expect_equal(count(where(df, df$age2 == 21)), 3)
1007+
1008+
df$age2 <- c(22)
1009+
expect_equal(columns(df), c("name", "age", "age2"))
1010+
expect_equal(count(where(df, df$age2 == 22)), 3)
1011+
1012+
expect_error(df$age3 <- c(22, NA),
1013+
"value must be a Column, literal value as atomic in length of 1, or NULL")
1014+
10041015
# Test parameter drop
10051016
expect_equal(class(df[, 1]) == "SparkDataFrame", T)
10061017
expect_equal(class(df[, 1, drop = T]) == "Column", T)
@@ -1778,6 +1789,13 @@ test_that("withColumn() and withColumnRenamed()", {
17781789
expect_equal(length(columns(newDF)), 2)
17791790
expect_equal(first(filter(newDF, df$name != "Michael"))$age, 32)
17801791

1792+
newDF <- withColumn(df, "age", 18)
1793+
expect_equal(length(columns(newDF)), 2)
1794+
expect_equal(first(newDF)$age, 18)
1795+
1796+
expect_error(withColumn(df, "age", list("a")),
1797+
"Literal value must be atomic in length of 1")
1798+
17811799
newDF2 <- withColumnRenamed(df, "age", "newerAge")
17821800
expect_equal(length(columns(newDF2)), 2)
17831801
expect_equal(columns(newDF2)[1], "newerAge")

0 commit comments

Comments
 (0)