Skip to content

Commit 4927f63

Browse files
yaooqinnzhengruifeng
authored andcommitted
[SPARK-48878][PYTHON][DOCS] Add doctests for options in json functions
### What changes were proposed in this pull request? Add doctests for `options` in json functions ### Why are the changes needed? test coverage, we never test `options` in `from_json` and `to_json` before since it is a new underlying implementation in Spark Connect, we should explicitly test it ### Does this PR introduce _any_ user-facing change? doc changes ### How was this patch tested? CI ### Was this patch authored or co-authored using generative AI tooling? No Closes apache#47319 from zhengruifeng/from_json_option. Lead-authored-by: Kent Yao <yao@apache.org> Co-authored-by: Ruifeng Zheng <ruifengz@apache.org> Signed-off-by: Ruifeng Zheng <ruifengz@apache.org>
1 parent 5a03410 commit 4927f63

File tree

1 file changed

+36
-6
lines changed

1 file changed

+36
-6
lines changed

python/pyspark/sql/functions/builtin.py

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15793,6 +15793,20 @@ def from_json(
1579315793
+---------+
1579415794
|[1, 2, 3]|
1579515795
+---------+
15796+
15797+
Example 6: Parsing JSON with specified options
15798+
15799+
>>> import pyspark.sql.functions as sf
15800+
>>> df = spark.createDataFrame([(1, '''{a:123}'''), (2, '''{"a":456}''')], ("key", "value"))
15801+
>>> parsed1 = sf.from_json(df.value, "a INT")
15802+
>>> parsed2 = sf.from_json(df.value, "a INT", {"allowUnquotedFieldNames": "true"})
15803+
>>> df.select("value", parsed1, parsed2).show()
15804+
+---------+----------------+----------------+
15805+
| value|from_json(value)|from_json(value)|
15806+
+---------+----------------+----------------+
15807+
| {a:123}| {NULL}| {123}|
15808+
|{"a":456}| {456}| {456}|
15809+
+---------+----------------+----------------+
1579615810
"""
1579715811
from pyspark.sql.classic.column import _to_java_column
1579815812

@@ -16113,6 +16127,19 @@ def to_json(col: "ColumnOrName", options: Optional[Dict[str, str]] = None) -> Co
1611316127
+---------------+
1611416128
|["Alice","Bob"]|
1611516129
+---------------+
16130+
16131+
Example 6: Converting to JSON with specified options
16132+
16133+
>>> import pyspark.sql.functions as sf
16134+
>>> df = spark.sql("SELECT (DATE('2022-02-22'), 1) AS date")
16135+
>>> json1 = sf.to_json(df.date)
16136+
>>> json2 = sf.to_json(df.date, {"dateFormat": "yyyy/MM/dd"})
16137+
>>> df.select("date", json1, json2).show(truncate=False)
16138+
+---------------+------------------------------+------------------------------+
16139+
|date |to_json(date) |to_json(date) |
16140+
+---------------+------------------------------+------------------------------+
16141+
|{2022-02-22, 1}|{"col1":"2022-02-22","col2":1}|{"col1":"2022/02/22","col2":1}|
16142+
+---------------+------------------------------+------------------------------+
1611616143
"""
1611716144
from pyspark.sql.classic.column import _to_java_column
1611816145

@@ -16150,12 +16177,15 @@ def schema_of_json(json: Union[Column, str], options: Optional[Dict[str, str]] =
1615016177

1615116178
Examples
1615216179
--------
16153-
>>> df = spark.range(1)
16154-
>>> df.select(schema_of_json(lit('{"a": 0}')).alias("json")).collect()
16155-
[Row(json='STRUCT<a: BIGINT>')]
16156-
>>> schema = schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'})
16157-
>>> df.select(schema.alias("json")).collect()
16158-
[Row(json='STRUCT<a: BIGINT>')]
16180+
>>> import pyspark.sql.functions as sf
16181+
>>> parsed1 = sf.schema_of_json(sf.lit('{"a": 0}'))
16182+
>>> parsed2 = sf.schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'})
16183+
>>> spark.range(1).select(parsed1, parsed2).show()
16184+
+------------------------+----------------------+
16185+
|schema_of_json({"a": 0})|schema_of_json({a: 1})|
16186+
+------------------------+----------------------+
16187+
| STRUCT<a: BIGINT>| STRUCT<a: BIGINT>|
16188+
+------------------------+----------------------+
1615916189
"""
1616016190
from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
1616116191

0 commit comments

Comments
 (0)