Skip to content

Commit ef33273

Browse files
committed
Add comment re splitting large record batches
1 parent 703f2a8 commit ef33273

File tree

1 file changed

+11
-0
lines changed

1 file changed

+11
-0
lines changed

http/examples/get/server_low_mem.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,17 @@ def do_GET(self):
7171
for buffer in generate_batches(schema, make_reader(schema, batches)):
7272
self.wfile.write(buffer)
7373
self.wfile.flush()
74+
75+
# if any record batch could be larger than 2 GB, split it
76+
# into chunks before passing to self.wfile.write():
77+
78+
#chunk_size = int(2e9)
79+
#chunk_splits = len(buffer) // chunk_size
80+
#for i in range(chunk_splits):
81+
# self.wfile.write(buffer[i * chunk_size:i * chunk_size + chunk_size])
82+
# self.wfile.flush()
83+
#self.wfile.write(buffer[chunk_splits * chunk_size:])
84+
#self.wfile.flush()
7485

7586
batches = GetPutData()
7687

0 commit comments

Comments
 (0)