Skip to content

Commit 3d6b00f

Browse files
agilelab-tmnd1991agilelab-tmnd1991
committed
Update delta protocol written spec
Update openapi spec wip wip wip Add nix stuff wippp
1 parent 20a0765 commit 3d6b00f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+2242
-453
lines changed

‎docsite/docs/protocols/delta-sharing-protocol.md

Lines changed: 263 additions & 15 deletions
Large diffs are not rendered by default.

‎protocol/delta-sharing-protocol-api.yml

Lines changed: 262 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,12 @@ paths:
527527
description: 'If set to true, return the historical metadata if seen in the delta log. This is for the streaming client to check if the table schema is still read compatible.'
528528
schema:
529529
type: boolean
530+
- in: header
531+
name: delta-sharing-capabilities
532+
required: false
533+
description: 'Delta Sharing Capabilities'
534+
schema:
535+
type: string
530536
responses:
531537
'400':
532538
$ref: "#/components/responses/400"
@@ -581,11 +587,14 @@ components:
581587
not Unary Represents a logical not check. This op should have once child.
582588
The supported value types:
583589
ValueType Description
584-
"bool" Represents an Boolean type.
585-
"int" Represents an Integer type.
586-
"long" Represents a Long type.
587-
"string" Represents a String type.
588-
"date" Represents a Date type in "yyyy-mm-dd" format.
590+
"bool" Represents an Boolean type.
591+
"int" Represents an Integer type.
592+
"long" Represents a Long type.
593+
"string" Represents a String type.
594+
"date" Represents a Date type in "yyyy-mm-dd" format.
595+
"float" Represents a Float type.
596+
"double" Represents a Double type.
597+
"timestamp" Represents a timestamp in ISO8601 format, in the UTC timezone.
589598
590599
ListShareResponse:
591600
type: object
@@ -771,27 +780,31 @@ components:
771780
properties:
772781
protocol:
773782
# it refers to ./delta-sharing-protocol.md#protocol
774-
$ref: '#/components/schemas/ProtocolObject'
783+
$ref: '#/components/schemas/ParquetProtocolObject'
775784
metadata:
776785
# it refers to ./delta-sharing-protocol.md#metadata
777-
$ref: '#/components/schemas/MetadataObject'
786+
$ref: '#/components/schemas/ParquetMetadataObject'
778787

779788
# This is not used for the spec but comes handy for autogeneration
780789
TableQueryResponseObject:
790+
oneOf:
791+
- $ref: '#/components/schemas/ParquetTableQueryResponseObject'
792+
- $ref: '#/components/schemas/DeltaTableQueryResponseObject'
793+
ParquetTableQueryResponseObject:
781794
type: object
782795
properties:
783796
protocol:
784797
# it refers to ./delta-sharing-protocol.md#protocol
785-
$ref: '#/components/schemas/ProtocolObject'
798+
$ref: '#/components/schemas/ParquetProtocolObject'
786799
metadata:
787800
# it refers to ./delta-sharing-protocol.md#metadata
788-
$ref: '#/components/schemas/MetadataObject'
801+
$ref: '#/components/schemas/ParquetMetadataObject'
789802
files:
790803
type: array
791804
items:
792805
# it refers to ./delta-sharing-protocol.md#file
793-
$ref: '#/components/schemas/FileObject'
794-
FileObject:
806+
$ref: '#/components/schemas/ParquetFileObject'
807+
ParquetFileObject:
795808
type: object
796809
properties:
797810
file:
@@ -825,7 +838,7 @@ components:
825838
- id
826839
- partitionValues
827840
- size
828-
ProtocolObject:
841+
ParquetProtocolObject:
829842
type: object
830843
properties:
831844
protocol:
@@ -834,15 +847,14 @@ components:
834847
minReaderVersion:
835848
type: integer
836849
format: int32
837-
FormatObject:
850+
ParquetFormatObject:
838851
type: object
839852
properties:
840853
provider:
841854
type: string
842855
required:
843856
- provider
844-
845-
MetadataObject:
857+
ParquetMetadataObject:
846858
type: object
847859
properties:
848860
metaData:
@@ -855,7 +867,7 @@ components:
855867
description:
856868
type: string
857869
format:
858-
$ref: '#/components/schemas/FormatObject'
870+
$ref: '#/components/schemas/ParquetFormatObject'
859871
schemaString:
860872
type: string
861873
partitionColumns:
@@ -881,7 +893,240 @@ components:
881893
- format
882894
- schemaString
883895
- partitionColumns
884-
896+
DeltaTableQueryResponseObject:
897+
type: object
898+
properties:
899+
protocol:
900+
# it refers to ./delta-sharing-protocol.md#protocol
901+
$ref: '#/components/schemas/DeltaProtocolObject'
902+
metadata:
903+
# it refers to ./delta-sharing-protocol.md#metadata
904+
$ref: '#/components/schemas/DeltaMetadataObject'
905+
files:
906+
type: array
907+
items:
908+
# it refers to ./delta-sharing-protocol.md#file
909+
$ref: '#/components/schemas/DeltaFileObject'
910+
DeltaProtocolObject:
911+
type: object
912+
properties:
913+
protocol:
914+
type: object
915+
properties:
916+
deltaProtocol:
917+
type: object
918+
properties:
919+
minReaderVersion:
920+
type: integer
921+
format: int32
922+
minWriterVersion:
923+
type: integer
924+
format: int32
925+
DeltaFormatObject:
926+
type: object
927+
properties:
928+
provider:
929+
type: string
930+
options:
931+
type: object
932+
additionalProperties:
933+
type: string
934+
required:
935+
- provider
936+
DeltaMetadata:
937+
type: object
938+
description: see https://github.com/delta-io/delta/blob/master/PROTOCOL.md#change-metadata
939+
required:
940+
- id
941+
- format
942+
- schemaString
943+
- partitionColumns
944+
- configuration
945+
properties:
946+
id:
947+
type: string
948+
name:
949+
type: string
950+
description:
951+
type: string
952+
format:
953+
$ref: '#/components/schemas/DeltaFormatObject'
954+
schemaString:
955+
type: string
956+
partitionColumns:
957+
type: array
958+
items:
959+
type: string
960+
createdTime:
961+
type: integer
962+
format: int64
963+
configuration:
964+
type: object
965+
additionalProperties:
966+
type: string
967+
DeltaMetadataObject:
968+
type: object
969+
properties:
970+
metaData:
971+
type: object
972+
properties:
973+
version:
974+
type: integer
975+
format: int64
976+
size:
977+
type: integer
978+
format: int64
979+
numFiles:
980+
type: integer
981+
format: int64
982+
deltaMetadata:
983+
$ref: '#/components/schemas/DeltaMetadata'
984+
required: [ deltaMetadata ]
985+
DeltaFileObject:
986+
required:
987+
- id
988+
- deltaSingleAction
989+
properties:
990+
id:
991+
type: string
992+
deletionVectorFileId:
993+
type: string
994+
version:
995+
type: integer
996+
format: int64
997+
timestamp:
998+
type: integer
999+
format: int64
1000+
expirationTimestamp:
1001+
type: integer
1002+
format: int64
1003+
deltaSingleAction:
1004+
$ref: '#/components/schemas/DeltaSingleAction'
1005+
DeltaSingleAction:
1006+
type: object
1007+
description: only one field can be not null, container of delta actions such as file, add, cdf or remove see https://github.com/delta-io/delta/tree/master/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions
1008+
properties:
1009+
add:
1010+
$ref: '#/components/schemas/DeltaAddFileForCDFAction'
1011+
cdf:
1012+
$ref: '#/components/schemas/DeltaAddCDCFileAction'
1013+
file:
1014+
$ref: '#/components/schemas/DeltaAddFileAction'
1015+
remove:
1016+
$ref: '#/components/schemas/DeltaRemoveFileAction'
1017+
DeltaAddFileForCDFAction:
1018+
type: object
1019+
description: see io.delta.sharing.server.model.AddFileForCDF
1020+
properties:
1021+
url:
1022+
type: string
1023+
id:
1024+
type: string
1025+
partitionValues:
1026+
type: object
1027+
additionalProperties:
1028+
type: string
1029+
size:
1030+
type: integer
1031+
format: int64
1032+
expirationTimestamp:
1033+
type: integer
1034+
format: int64
1035+
version:
1036+
type: integer
1037+
format: int64
1038+
timestamp:
1039+
type: integer
1040+
format: int64
1041+
stats:
1042+
type: string
1043+
DeltaAddCDCFileAction:
1044+
type: object
1045+
description: see io.delta.sharing.server.model.AddCDCFile
1046+
properties:
1047+
url:
1048+
type: string
1049+
id:
1050+
type: string
1051+
partitionValues:
1052+
type: object
1053+
additionalProperties:
1054+
type: string
1055+
size:
1056+
type: integer
1057+
format: int64
1058+
expirationTimestamp:
1059+
type: integer
1060+
format: int64
1061+
timestamp:
1062+
type: integer
1063+
format: int64
1064+
version:
1065+
type: integer
1066+
format: int64
1067+
DeltaAddFileAction:
1068+
type: object
1069+
description: see io.delta.sharing.server.model.AddFile
1070+
properties:
1071+
url:
1072+
type: string
1073+
id:
1074+
type: string
1075+
partitionValues:
1076+
type: object
1077+
additionalProperties:
1078+
type: string
1079+
size:
1080+
type: integer
1081+
format: int64
1082+
stats:
1083+
type: string
1084+
expirationTimestamp:
1085+
type: integer
1086+
format: int64
1087+
timestamp:
1088+
type: integer
1089+
format: int64
1090+
version:
1091+
type: integer
1092+
format: int64
1093+
DeltaRemoveFileAction:
1094+
type: object
1095+
properties:
1096+
url:
1097+
type: string
1098+
id:
1099+
type: string
1100+
partitionValues:
1101+
type: object
1102+
additionalProperties:
1103+
type: string
1104+
size:
1105+
type: integer
1106+
format: int64
1107+
expirationTimestamp:
1108+
type: integer
1109+
format: int64
1110+
timestamp:
1111+
type: integer
1112+
format: int64
1113+
version:
1114+
type: integer
1115+
format: int64
1116+
DeltaEndStreamAction:
1117+
description: An action that is returned as the last line of the streaming response. It allows the server to include additional data that might be dynamically generated while the streaming message is sent
1118+
type: object
1119+
properties:
1120+
refreshToken:
1121+
type: string
1122+
description: a token used to refresh pre-signed urls for a long running query
1123+
nextPageToken:
1124+
type: string
1125+
description: a token used to retrieve the subsequent page of a query
1126+
minUrlExpirationTimestamp:
1127+
description: the minimum url expiration timestamp of the urls returned in current response
1128+
type: integer
1129+
format: int64
8851130
responses:
8861131
"400":
8871132
description: The request is malformed

‎sbin/buildNativeContainer.sh

100644100755
Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
#!/bin/bash
2-
./gradlew server:imageBuild -Dquarkus.package.type=native -Dquarkus.native.container-build=true -Dquarkus.container-image.name=server-native
2+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
3+
pushd $SCRIPT_DIR/..
4+
nix-shell --run './gradlew server:app:imageBuild -Dquarkus.package.type=native -Dquarkus.native.container-build=true -Dquarkus.container-image.name=server-native'
5+
popd

‎sbin/startDocusaurus.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
3+
pushd $SCRIPT_DIR/..
4+
nix-shell --run './gradlew docsite:npm_run_start'
5+
popd

‎server/app/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ val serverGeneratorProperties = mapOf(
4949
"dateLibrary" to "java8",
5050
"disallowAdditionalPropertiesIfNotPresent" to "false",
5151
"generateBuilders" to "false",
52+
"legacyDiscriminatorBehavior" to "false",
5253
"generatePom" to "false",
5354
"interfaceOnly" to "true",
5455
"library" to "quarkus",

0 commit comments

Comments
 (0)