From 5cbce10a4dcbcbedc46ccfa0be24c67787ba2fb3 Mon Sep 17 00:00:00 2001 From: Jove Zhong Date: Wed, 17 Jul 2024 16:57:07 +0800 Subject: [PATCH] customize the batch size for proton --- core/dbio/database/transaction.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/dbio/database/transaction.go b/core/dbio/database/transaction.go index ff443be7..ca6627ed 100644 --- a/core/dbio/database/transaction.go +++ b/core/dbio/database/transaction.go @@ -322,6 +322,8 @@ func InsertBatchStream(conn Connection, tx Transaction, tableFName string, ds *i row = processClickhouseInsertRow(bColumns, row) } else if conn.GetType() == dbio.TypeDbTrino { row = processTrinoInsertRow(bColumns, row) + } else if conn.GetType() == dbio.TypeDbProton { + row = processClickhouseInsertRow(bColumns, row) } vals = append(vals, row...) } @@ -391,6 +393,12 @@ func InsertBatchStream(conn Connection, tx Transaction, tableFName string, ds *i batchSize = cast.ToInt(conn.GetTemplateValue("variable.batch_values")) / len(columns) } + if conn.GetType() == dbio.TypeDbProton { + batchSize = 100 + } else { + batchSize = cast.ToInt(conn.GetTemplateValue("variable.batch_values")) / len(columns) + } + for row := range batch.Rows { batchRows = append(batchRows, row) count++