Skip to content

Commit 890348f

Browse files
authored
Merge pull request byzer-org#1480 from lwz9103/MLSQL-1417
byzer-org#1417 Add suggestions when using delta path mode.
2 parents 1317d00 + d1d077e commit 890348f

1 file changed

Lines changed: 38 additions & 9 deletions

File tree

streamingpro-mlsql/src/main/java/streaming/core/datasource/impl/MLSQLDelta.scala

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,15 +35,26 @@ class MLSQLDelta(override val uid: String) extends MLSQLBaseFileSource with WowP
3535
val reader = config.df.get.sparkSession.read
3636

3737
val dataLake = new DataLake(config.df.get.sparkSession)
38-
val finalPath = if (dataLake.isEnable) {
39-
dataLake.identifyToPath(config.path)
40-
} else {
41-
resourceRealPath(context.execListener, Option(owner), config.path)
38+
val loadPath = (dataLake.isEnable, config.config.get("mode").map(_.toLowerCase())) match {
39+
case (true, None) =>
40+
if (config.path.contains("/")) {
41+
throw new MLSQLException(
42+
"""Delta path mode is not enabled, you can use table mode like "load delta.`public.test` as t1" or add parameter like
43+
| load delta.`/mlsql/delta/public/test` where mode= "path" """.stripMargin)
44+
}
45+
dataLake.identifyToPath(config.path)
46+
case (true, Some("path")) | (false, _) =>
47+
resourceRealPath(context.execListener, Option(owner), config.path)
48+
case (_, _) =>
49+
throw new MLSQLException(
50+
""" Can not resolve mode argument, you can add parameter like
51+
| load delta.`/mlsql/delta/public/test` where mode= "path"
52+
""".stripMargin)
4253
}
4354

4455
reader.options(rewriteConfig(config.config) ++ newOpt).
4556
format(format).
46-
load(finalPath)
57+
load(loadPath)
4758
}
4859

4960
(parameters.get("startingVersion").map(_.toLong), parameters.get("endingVersion").map(_.toLong)) match {
@@ -67,11 +78,29 @@ class MLSQLDelta(override val uid: String) extends MLSQLBaseFileSource with WowP
6778
}
6879

6980
val dataLake = new DataLake(config.df.get.sparkSession)
70-
val finalPath = if (dataLake.isEnable) {
71-
dataLake.identifyToPath(config.path)
72-
} else {
73-
resourceRealPath(context.execListener, Option(context.owner), config.path)
81+
val finalPath = (dataLake.isEnable, config.config.get("mode").map(_.toLowerCase())) match {
82+
case (true, None) =>
83+
if (config.path.contains("/")) {
84+
throw new MLSQLException(
85+
"""Delta path mode is not enabled, you can use table mode like "save overwrite test as delta.`public.test`" or add parameter like
86+
| save overwrite test as delta.`/mlsql/delta/public/test` where mode= "path" """.stripMargin)
87+
}
88+
dataLake.identifyToPath(config.path)
89+
case (true, Some("path")) =>
90+
if (config.path.startsWith(dataLake.value)) {
91+
throw new MLSQLException(
92+
s""" Can not save delta in table mode directory ${dataLake.value}""")
93+
}
94+
resourceRealPath(context.execListener, Option(context.owner), config.path)
95+
case (false, _) =>
96+
resourceRealPath(context.execListener, Option(context.owner), config.path)
97+
case (_, _) =>
98+
throw new MLSQLException(
99+
""" Can not resolve mode argument, you can add parameter like
100+
| save test as delta.`/mlsql/delta/public/test` where mode= "path"
101+
""".stripMargin)
74102
}
103+
75104
writer.options(rewriteConfig(config.config)).mode(config.mode).format(format).save(finalPath)
76105
}
77106

0 commit comments

Comments
 (0)