Add CREATE EXTERNAL TABLE statement

[penghuo]

No description provided.

[penghuo]

• code base, https://github.com/penghuo/delta/tree/hp/externalTable
• create parquet file

CREATE TABLE IF NOT EXISTS p001 (name STRING, age INT) 
USING PARQUET
LOCATION "/Users/penghuo/release/deltalake/parquet/p001";

INSERT INTO default.p001 VALUES ('a', 1), ('b', 2);
INSERT INTO default.p001 VALUES ('c', 3), ('d', 4);

• detect new parquet files in dir

import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.streaming._
import org.apache.spark.sql.types._
import org.apache.spark.sql.functions.input_file_name

val schema = new StructType().add("name", StringType).add("age", IntegerType)
val streamDF = spark.readStream.schema(schema).format("parquet").option("path", "/Users/penghuo/release/deltalake/parquet/p001").load()
val query = streamDF.select(input_file_name()).writeStream.format("console").start()

• create external table

CREATE EXTERNAL TABLE IF NOT EXISTS default.t001
(name STRING, age INT)
USING DELTA
LOCATION "/Users/penghuo/release/deltalake/parquet/p001"
TBLPROPERTIES ('auto_refresh'='true');

• add more data to parquet table

INSERT INTO default.p001 VALUES ('e', 5), ('f', 6);
INSERT INTO default.p001 VALUES ('g', 7);

• check the result
  • the result should be reflected in t001

select * from t001 order by age;

a    1
b    2
c    3
d    4
e    5
f    6
g    7

* the delta_log should has new log