Skip to content

Commit 331984b

Browse files
bewithgauravCopilotjahnvi480
authored
PERF: route setinputsizes through the native C++ execute pipeline (#736)
### Work Item / Issue Reference > GitHub Issue: #500 ------------------------------------------------------------------- ### Summary move `setinputsizes` overrides into the native detect, bind, and execute path and remove `DDBCSQLExecuteLegacy`. ------------------------------------------------------------------- ### Performance Results #### macOS arm64, Python 3.13, SQL Server 2022, release build | Path | Median | Improvement | |------|--------|-------------| | `main` legacy path | 2.205s | | | this PR native path | 1.686s | **24% faster** | the issue #500 workload runs 100 executions with 2,000 declared parameters each (1,000 `INT`/`VARCHAR` rows), calling `setinputsizes()` before every execute. results are the median of three runs and were confirmed with the base/PR execution order reversed. #### Cross-platform CI | Platform | `main` | This PR | Raw improvement | |----------|--------|---------|-----------------| | Windows SQL2022 | 4.0770s | 2.8424s | **30.3% faster** | | Windows SQL2025 | 3.8320s | 3.1110s | **18.8% faster** | | macOS SQL2022 | 18.0371s | 9.0534s | **49.8% faster** | | macOS SQL2025 | 9.6811s | 7.1356s | **26.3% faster** | | Linux SQL2022 | 2.7965s | 2.3523s | **15.9% faster** | `main` results are from [build 171915](https://dev.azure.com/sqlclientdrivers/public/_build/results?buildId=171915). PR results are from [build 172065](https://dev.azure.com/sqlclientdrivers/public/_build/results?buildId=172065), except macOS SQL2022, which is from the last completed run in [build 171939](https://dev.azure.com/sqlclientdrivers/public/_build/results?buildId=171939); the rerun reached the 20-minute benchmark timeout before that scenario completed. these are raw medians. the existing pyodbc-normalized score is not used here because the Windows pyodbc main baseline did not reproduce across hosted runners. --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Jahnvi Thakkar <61936179+jahnvi480@users.noreply.github.com>
1 parent 8cf8759 commit 331984b

6 files changed

Lines changed: 232 additions & 431 deletions

File tree

‎mssql_python/cursor.py‎

Lines changed: 27 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,7 @@ def __init__(self, connection: "Connection", timeout: int = 0) -> None:
362362

363363
self._connection: "Connection" = connection # Store as private attribute
364364
self._timeout: int = timeout
365-
self._inputsizes: Optional[List[Union[int, Tuple[Any, ...]]]] = None
365+
self._inputsizes: Optional[List[Tuple[int, int, int, int]]] = None
366366
# self.connection.autocommit = False
367367
self._initialize_cursor()
368368
self.description: Optional[
@@ -1192,7 +1192,14 @@ def setinputsizes(self, sizes: List[Union[int, tuple]]) -> None:
11921192
f"Must be a non-negative integer."
11931193
)
11941194

1195-
self._inputsizes.append((sql_type, column_size, decimal_digits))
1195+
self._inputsizes.append(
1196+
(
1197+
sql_type,
1198+
self._get_c_type_for_sql_type(sql_type),
1199+
column_size,
1200+
decimal_digits,
1201+
)
1202+
)
11961203
else:
11971204
# Handle single value (just sql_type)
11981205
sql_type = size_info
@@ -1203,7 +1210,9 @@ def setinputsizes(self, sizes: List[Union[int, tuple]]) -> None:
12031210
f"Invalid SQL type: {sql_type}. Must be a valid SQL type constant."
12041211
)
12051212

1206-
self._inputsizes.append((sql_type, 0, 0))
1213+
self._inputsizes.append(
1214+
(sql_type, self._get_c_type_for_sql_type(sql_type), 0, 0)
1215+
)
12071216

12081217
def _reset_inputsizes(self) -> None:
12091218
"""Reset input sizes after execution"""
@@ -1262,12 +1271,9 @@ def _create_parameter_types_list( # pylint: disable=too-many-arguments,too-many
12621271
"""
12631272
Maps parameter types for the given parameter.
12641273
1265-
Python-side type detection. The standard execute() path no longer calls this —
1266-
DetectParamTypes does the same job in C++ without crossing the pybind11
1267-
boundary per parameter. Two callers remain: the legacy execute() branch used
1268-
when setinputsizes overrides are active, and executemany(). The former goes
1269-
away once setinputsizes is handled natively; the latter needs its own
1270-
native columnwise detection before this can be deleted outright.
1274+
Python-side type detection for executemany(). The standard execute() path
1275+
uses DetectParamTypes in C++ instead. This helper goes away once executemany
1276+
has native columnwise detection.
12711277
12721278
Args:
12731279
parameter: parameter to bind.
@@ -1276,61 +1282,14 @@ def _create_parameter_types_list( # pylint: disable=too-many-arguments,too-many
12761282
"""
12771283
paraminfo = param_info()
12781284

1279-
# Check if we have explicit type information from setinputsizes
1280-
if self._inputsizes and i < len(self._inputsizes):
1281-
# Use explicit type information
1282-
sql_type, column_size, decimal_digits = self._inputsizes[i]
1283-
1284-
# Default is_dae to False for explicit types, but set to True for large strings/binary
1285-
is_dae = False
1286-
1287-
if parameter is None:
1288-
# For NULL parameters, always use SQL_C_DEFAULT regardless of SQL type
1289-
c_type = ddbc_sql_const.SQL_C_DEFAULT.value
1290-
else:
1291-
# For non-NULL parameters, determine the appropriate C type based on SQL type
1292-
c_type = self._get_c_type_for_sql_type(sql_type)
1293-
1294-
# Override DECIMAL/NUMERIC to use SQL_C_CHAR string binding (GH-503).
1295-
# The generic mapping returns SQL_C_NUMERIC which requires NumericData
1296-
# structs, but setinputsizes declares fixed precision/scale that may
1297-
# differ from per-value precision, causing misinterpretation. String
1298-
# binding lets ODBC convert using the declared columnSize/decimalDigits.
1299-
if sql_type in (
1300-
ddbc_sql_const.SQL_DECIMAL.value,
1301-
ddbc_sql_const.SQL_NUMERIC.value,
1302-
):
1303-
c_type = ddbc_sql_const.SQL_C_CHAR.value
1304-
if isinstance(parameter, decimal.Decimal):
1305-
parameters_list[i] = format(parameter, "f")
1306-
parameter = parameters_list[i]
1307-
1308-
# Check if this should be a DAE (data at execution) parameter
1309-
# For string types with large column sizes
1310-
if isinstance(parameter, str) and column_size > MAX_INLINE_CHAR:
1311-
is_dae = True
1312-
# For binary types with large column sizes
1313-
elif isinstance(parameter, (bytes, bytearray)) and column_size > 8000:
1314-
is_dae = True
1315-
1316-
# Sanitize precision/scale for numeric types
1317-
if sql_type in (
1318-
ddbc_sql_const.SQL_DECIMAL.value,
1319-
ddbc_sql_const.SQL_NUMERIC.value,
1320-
):
1321-
column_size = max(1, min(int(column_size) if column_size > 0 else 18, 38))
1322-
decimal_digits = min(max(0, decimal_digits), column_size)
1323-
1324-
else:
1325-
# Fall back to automatic type inference
1326-
sql_type, c_type, column_size, decimal_digits, is_dae = self._map_sql_type(
1327-
parameter,
1328-
parameters_list,
1329-
i,
1330-
min_val=min_val,
1331-
max_val=max_val,
1332-
decimal_as_numeric=decimal_as_numeric,
1333-
)
1285+
sql_type, c_type, column_size, decimal_digits, is_dae = self._map_sql_type(
1286+
parameter,
1287+
parameters_list,
1288+
i,
1289+
min_val=min_val,
1290+
max_val=max_val,
1291+
decimal_as_numeric=decimal_as_numeric,
1292+
)
13341293

13351294
# If TIME values are being bound via text C-types, normalize them to a
13361295
# textual representation expected by SQL_C_CHAR/SQL_C_WCHAR binding.
@@ -1814,70 +1773,18 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state
18141773
self.is_stmt_prepared = [False]
18151774
effective_use_prepare = use_prepare and not same_sql
18161775

1817-
# Standard path: when no inputsizes override, type detection + bind + execute
1818-
# all happen in C++ via DDBCSQLExecute. ParamInfo never crosses the pybind11
1819-
# boundary. This is the path ~99% of calls take.
1820-
use_standard_execute = parameters and not (
1821-
self._inputsizes and any(s is not None for s in self._inputsizes)
1822-
)
1823-
1824-
if use_standard_execute:
1776+
if parameters:
18251777
ret = ddbc_bindings.DDBCSQLExecute(
18261778
self.hstmt,
18271779
operation,
18281780
parameters,
1781+
self._inputsizes,
18291782
self.is_stmt_prepared,
18301783
effective_use_prepare,
18311784
encoding_settings,
18321785
)
18331786
else:
1834-
# LEGACY PATH — slated for removal in a future optimization round.
1835-
#
1836-
# Kept only for setinputsizes() callers, where the user's explicit type
1837-
# overrides have to be honoured instead of C++ detecting types itself.
1838-
# Type detection happens in Python here, so every parameter round-trips
1839-
# through pybind11 as a ParamInfo object, which is what makes it slow.
1840-
# Once setinputsizes overrides are handled natively, this branch and
1841-
# DDBCSQLExecuteLegacy both go away.
1842-
parameters_type = []
1843-
if parameters:
1844-
param_info = ddbc_bindings.ParamInfo
1845-
for i, param in enumerate(parameters):
1846-
# decimal_as_numeric=True so an uncovered money-range Decimal here
1847-
# (setinputsizes shorter than the parameter list) binds as SQL_NUMERIC
1848-
# like the native path, not VARCHAR (GH-740). executemany keeps the
1849-
# VARCHAR shortcut for its batch string binding (GH-503).
1850-
paraminfo = self._create_parameter_types_list(
1851-
param, param_info, parameters, i, decimal_as_numeric=True
1852-
)
1853-
parameters_type.append(paraminfo)
1854-
1855-
if logger.isEnabledFor(logging.DEBUG):
1856-
for i, param in enumerate(parameters):
1857-
logger.debug(
1858-
"""Parameter number: %s, Parameter: %s,
1859-
Param Python Type: %s, ParamInfo: %s, %s, %s, %s, %s""",
1860-
i + 1,
1861-
param,
1862-
str(type(param)),
1863-
parameters_type[i].paramSQLType,
1864-
parameters_type[i].paramCType,
1865-
parameters_type[i].columnSize,
1866-
parameters_type[i].decimalDigits,
1867-
parameters_type[i].inputOutputType,
1868-
)
1869-
1870-
# Legacy binding: accepts the pre-built ParamInfo list from Python.
1871-
# Goes away with the branch above.
1872-
ret = ddbc_bindings.DDBCSQLExecuteLegacy(
1873-
self.hstmt,
1874-
operation,
1875-
parameters,
1876-
parameters_type,
1877-
self.is_stmt_prepared,
1878-
effective_use_prepare,
1879-
encoding_settings,
1880-
)
1787+
ret = ddbc_bindings.DDBCSQLExecDirect(self.hstmt, operation)
18811788
# Check return code
18821789
try:
18831790

@@ -2564,14 +2471,11 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s
25642471

25652472
if self._inputsizes and col_index < len(self._inputsizes):
25662473
# Use explicitly set input sizes
2567-
sql_type, column_size, decimal_digits = self._inputsizes[col_index]
2474+
sql_type, c_type, column_size, decimal_digits = self._inputsizes[col_index]
25682475

25692476
# Default is_dae to False
25702477
is_dae = False
25712478

2572-
# Determine appropriate C type based on SQL type
2573-
c_type = self._get_c_type_for_sql_type(sql_type)
2574-
25752479
# Override DECIMAL/NUMERIC to use SQL_C_CHAR string binding (GH-503)
25762480
if sql_type in (
25772481
ddbc_sql_const.SQL_DECIMAL.value,

0 commit comments

Comments
 (0)