You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
CREATE TABLE IF NOT EXISTS p001 (name STRING, age INT)
USING PARQUET
LOCATION "/Users/penghuo/release/deltalake/parquet/p001";
INSERT INTO default.p001 VALUES ('a', 1), ('b', 2);
INSERT INTO default.p001 VALUES ('c', 3), ('d', 4);
detect new parquet files in dir
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.streaming._
import org.apache.spark.sql.types._
import org.apache.spark.sql.functions.input_file_name
val schema = new StructType().add("name", StringType).add("age", IntegerType)
val streamDF = spark.readStream.schema(schema).format("parquet").option("path", "/Users/penghuo/release/deltalake/parquet/p001").load()
val query = streamDF.select(input_file_name()).writeStream.format("console").start()
create external table
CREATE EXTERNAL TABLE IF NOT EXISTS default.t001
(name STRING, age INT)
USING DELTA
LOCATION "/Users/penghuo/release/deltalake/parquet/p001"
TBLPROPERTIES ('auto_refresh'='true');
add more data to parquet table
INSERT INTO default.p001 VALUES ('e', 5), ('f', 6);
INSERT INTO default.p001 VALUES ('g', 7);
check the result
the result should be reflected in t001
select * from t001 order by age;
a 1
b 2
c 3
d 4
e 5
f 6
g 7
No description provided.
The text was updated successfully, but these errors were encountered: