Skip to content

Commit e4df37a

Browse files
authored
Ballista: Prep for fixing shuffle mechansim, part 1 (#738)
1 parent c51e9ec commit e4df37a

File tree

10 files changed

+316
-189
lines changed

10 files changed

+316
-189
lines changed

ballista/rust/core/proto/ballista.proto

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -721,6 +721,7 @@ message PartitionLocation {
721721
PartitionId partition_id = 1;
722722
ExecutorMetadata executor_meta = 2;
723723
PartitionStats partition_stats = 3;
724+
string path = 4;
724725
}
725726

726727
// Unique identifier for a materialized partition of data
@@ -776,6 +777,17 @@ message FailedTask {
776777

777778
message CompletedTask {
778779
string executor_id = 1;
780+
// TODO tasks are currently always shuffle writes but this will not always be the case
781+
// so we might want to think about some refactoring of the task definitions
782+
repeated ShuffleWritePartition partitions = 2;
783+
}
784+
785+
message ShuffleWritePartition {
786+
uint64 partition_id = 1;
787+
string path = 2;
788+
uint64 num_batches = 3;
789+
uint64 num_rows = 4;
790+
uint64 num_bytes = 5;
779791
}
780792

781793
message TaskStatus {

0 commit comments

Comments
 (0)