Skip to content

Commit a7cc4c0

Browse files
gargsaumyagargsaumya
committed
Add SQL_VARIANT data type support with native Python type preservation
- Add SQL_VARIANT constant (-150) to constants.py - Implement preprocessing approach in ddbc_bindings.cpp: * MapVariantCTypeToSQLType helper maps C types to SQL types * SQLGetData_wrap detects sql_variant and maps to base type * Handles old-style date/time C codes (9, 10, 11) * Handles SQL Server TIME type (code 16384) * Routes to existing type conversion logic (no duplication) - Move LOB detection before calculateRowSize in FetchAll_wrap - Add comprehensive test suite (25 tests): * Tests all SQL base types: INT, BIGINT, SMALLINT, TINYINT, REAL, FLOAT, DECIMAL, NUMERIC, BIT, VARCHAR, NVARCHAR, DATE, TIME, DATETIME, DATETIME2, VARBINARY, UNIQUEIDENTIFIER, NULL * Tests all fetch methods: fetchone(), fetchmany(), fetchall() * Tests implicit vs explicit type casting * All tests passing Type mappings: - Integer types → Python int - Float types → Python float - Exact numeric → Python Decimal - Character types → Python str - Date/time types → Python date/time/datetime objects - Binary → Python bytes - GUID → Python str/UUID - NULL → Python None
1 parent f8a6ca1 commit a7cc4c0

File tree

4 files changed

+652
-40
lines changed

4 files changed

+652
-40
lines changed

‎mssql_python/constants.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ class ConstantsDDBC(Enum):
117117
SQL_DATETIMEOFFSET = -155
118118
SQL_SS_TIME2 = -154
119119
SQL_SS_XML = -152
120+
SQL_SS_VARIANT = -150
120121
SQL_C_SS_TIMESTAMPOFFSET = 0x4001
121122
SQL_SCOPE_CURROW = 0
122123
SQL_BEST_ROWID = 1
@@ -374,6 +375,7 @@ def get_valid_types(cls) -> set:
374375
ConstantsDDBC.SQL_DATETIMEOFFSET.value,
375376
ConstantsDDBC.SQL_SS_XML.value,
376377
ConstantsDDBC.SQL_GUID.value,
378+
ConstantsDDBC.SQL_SS_VARIANT.value,
377379
}
378380

379381
# Could also add category methods for convenience

‎mssql_python/cursor.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -878,6 +878,7 @@ def _get_c_type_for_sql_type(self, sql_type: int) -> int:
878878
# Other types
879879
ddbc_sql_const.SQL_GUID.value: ddbc_sql_const.SQL_C_GUID.value,
880880
ddbc_sql_const.SQL_SS_XML.value: ddbc_sql_const.SQL_C_WCHAR.value,
881+
ddbc_sql_const.SQL_SS_VARIANT.value: ddbc_sql_const.SQL_C_BINARY.value,
881882
}
882883
return sql_to_c_type.get(sql_type, ddbc_sql_const.SQL_C_DEFAULT.value)
883884

‎mssql_python/pybind/ddbc_bindings.cpp‎

Lines changed: 149 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@
2727
#define MAX_DIGITS_IN_NUMERIC 64
2828
#define SQL_MAX_NUMERIC_LEN 16
2929
#define SQL_SS_XML (-152)
30+
#define SQL_SS_VARIANT (-150)
31+
#define SQL_CA_SS_BASE 1200
32+
#define SQL_CA_SS_VARIANT_TYPE (SQL_CA_SS_BASE + 15)
33+
#define SQL_CA_SS_VARIANT_SQL_TYPE (SQL_CA_SS_BASE + 16)
34+
#define SQL_CA_SS_VARIANT_SERVER_TYPE (SQL_CA_SS_BASE + 17)
3035

3136
#define STRINGIFY_FOR_CASE(x) \
3237
case x: \
@@ -1153,7 +1158,8 @@ void SqlHandle::markImplicitlyFreed() {
11531158
// Log error but don't throw - we're likely in cleanup/destructor path
11541159
LOG_ERROR("SAFETY VIOLATION: Attempted to mark non-STMT handle as implicitly freed. "
11551160
"Handle type=%d. This will cause handle leak. Only STMT handles are "
1156-
"automatically freed by parent DBC handles.", _type);
1161+
"automatically freed by parent DBC handles.",
1162+
_type);
11571163
return; // Refuse to mark - let normal free() handle it
11581164
}
11591165
_implicitly_freed = true;
@@ -2891,6 +2897,55 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT
28912897
}
28922898
}
28932899

2900+
// Helper function to map sql_variant's underlying C type to SQL data type
2901+
// This allows sql_variant to reuse existing fetch logic for each data type
2902+
SQLSMALLINT MapVariantCTypeToSQLType(SQLLEN variantCType) {
2903+
switch (variantCType) {
2904+
case SQL_C_SLONG:
2905+
case SQL_C_LONG:
2906+
return SQL_INTEGER;
2907+
case SQL_C_SSHORT:
2908+
case SQL_C_SHORT:
2909+
return SQL_SMALLINT;
2910+
case SQL_C_SBIGINT:
2911+
return SQL_BIGINT;
2912+
case SQL_C_FLOAT:
2913+
return SQL_REAL;
2914+
case SQL_C_DOUBLE:
2915+
return SQL_DOUBLE;
2916+
case SQL_C_BIT:
2917+
return SQL_BIT;
2918+
case SQL_C_CHAR:
2919+
return SQL_VARCHAR;
2920+
case SQL_C_WCHAR:
2921+
return SQL_WVARCHAR;
2922+
// Date/time types - handle both old-style (9, 10, 11) and new-style (91, 92, 93) codes
2923+
case 9: // SQL_C_DATE (old style)
2924+
case SQL_C_TYPE_DATE: // 91 (new style)
2925+
return SQL_TYPE_DATE;
2926+
case 10: // SQL_C_TIME (old style)
2927+
case SQL_C_TYPE_TIME: // 92 (new style)
2928+
case 16384: // SQL Server variant TIME type (observed value)
2929+
return SQL_TYPE_TIME;
2930+
case 11: // SQL_C_TIMESTAMP (old style)
2931+
case SQL_C_TYPE_TIMESTAMP: // 93 (new style)
2932+
return SQL_TYPE_TIMESTAMP;
2933+
case SQL_C_BINARY:
2934+
return SQL_VARBINARY;
2935+
case SQL_C_GUID:
2936+
return SQL_GUID;
2937+
case SQL_C_NUMERIC:
2938+
return SQL_NUMERIC;
2939+
case SQL_C_TINYINT:
2940+
case SQL_C_UTINYINT:
2941+
case SQL_C_STINYINT:
2942+
return SQL_TINYINT;
2943+
default:
2944+
// Unknown type, fallback to WVARCHAR for string conversion
2945+
return SQL_WVARCHAR;
2946+
}
2947+
}
2948+
28942949
// Helper function to retrieve column data
28952950
SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, py::list& row,
28962951
const std::string& charEncoding = "utf-8",
@@ -2929,7 +2984,40 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p
29292984
continue;
29302985
}
29312986

2932-
switch (dataType) {
2987+
printf("[DEBUG] SQLGetData_wrap: Column %d - dataType=%d, columnSize=%lu\n", i, dataType,
2988+
(unsigned long)columnSize);
2989+
2990+
// Preprocess sql_variant: detect underlying type and handle NULL
2991+
// This allows reuse of existing fetch logic instead of duplicating code
2992+
SQLSMALLINT effectiveDataType = dataType;
2993+
if (dataType == SQL_SS_VARIANT) {
2994+
// Step 1: Check for NULL using header read
2995+
SQLLEN indicator;
2996+
ret = SQLGetData_ptr(hStmt, i, SQL_C_BINARY, NULL, 0, &indicator);
2997+
if (indicator == SQL_NULL_DATA) {
2998+
row.append(py::none());
2999+
continue; // Skip to next column
3000+
}
3001+
3002+
// Step 2: Get the variant's underlying C data type
3003+
SQLLEN variantCType = 0;
3004+
ret =
3005+
SQLColAttribute_ptr(hStmt, i, SQL_CA_SS_VARIANT_TYPE, NULL, 0, NULL, &variantCType);
3006+
if (!SQL_SUCCEEDED(ret)) {
3007+
LOG("SQLGetData: Failed to get sql_variant underlying type for column %d", i);
3008+
row.append(py::none());
3009+
continue; // Skip to next column
3010+
}
3011+
3012+
printf("[DEBUG] SQLGetData_wrap: sql_variant column %d has variantCType=%ld\n", i,
3013+
(long)variantCType);
3014+
3015+
// Step 3: Map C type to SQL type so existing code can handle it
3016+
effectiveDataType = MapVariantCTypeToSQLType(variantCType);
3017+
printf("[DEBUG] SQLGetData_wrap: Mapped to effectiveDataType=%d\n", effectiveDataType);
3018+
}
3019+
3020+
switch (effectiveDataType) {
29333021
case SQL_CHAR:
29343022
case SQL_VARCHAR:
29353023
case SQL_LONGVARCHAR: {
@@ -4041,10 +4129,19 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch
40414129
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
40424130
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
40434131

4044-
if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR ||
4045-
dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY ||
4046-
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML) &&
4047-
(columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE)) {
4132+
printf("[DEBUG] FetchMany_wrap: Column %d - dataType=%d, columnSize=%lu\n", i + 1, dataType,
4133+
(unsigned long)columnSize);
4134+
4135+
// Detect LOB columns that need SQLGetData streaming
4136+
// sql_variant always uses SQLGetData for native type preservation
4137+
if (dataType == SQL_SS_VARIANT) {
4138+
lobColumns.push_back(i + 1); // 1-based
4139+
} else if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR ||
4140+
dataType == SQL_VARCHAR || dataType == SQL_LONGVARCHAR ||
4141+
dataType == SQL_VARBINARY || dataType == SQL_LONGVARBINARY ||
4142+
dataType == SQL_SS_XML) &&
4143+
(columnSize == 0 || columnSize == SQL_NO_TOTAL ||
4144+
columnSize > SQL_MAX_LOB_SIZE)) {
40484145
lobColumns.push_back(i + 1); // 1-based
40494146
}
40504147
}
@@ -4129,6 +4226,52 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows,
41294226
return ret;
41304227
}
41314228

4229+
// Detect LOB columns FIRST (before calculateRowSize)
4230+
// This allows sql_variant to skip the binding path entirely
4231+
std::vector<SQLUSMALLINT> lobColumns;
4232+
for (SQLSMALLINT i = 0; i < numCols; i++) {
4233+
auto colMeta = columnNames[i].cast<py::dict>();
4234+
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
4235+
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
4236+
4237+
printf("[DEBUG] FetchAll_wrap: Column %d - dataType=%d, columnSize=%lu\n", i + 1, dataType,
4238+
(unsigned long)columnSize);
4239+
4240+
// Detect LOB columns that need SQLGetData streaming
4241+
// sql_variant always uses SQLGetData for native type preservation
4242+
if (dataType == SQL_SS_VARIANT) {
4243+
lobColumns.push_back(i + 1); // 1-based
4244+
} else if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR ||
4245+
dataType == SQL_VARCHAR || dataType == SQL_LONGVARCHAR ||
4246+
dataType == SQL_VARBINARY || dataType == SQL_LONGVARBINARY ||
4247+
dataType == SQL_SS_XML) &&
4248+
(columnSize == 0 || columnSize == SQL_NO_TOTAL ||
4249+
columnSize > SQL_MAX_LOB_SIZE)) {
4250+
lobColumns.push_back(i + 1); // 1-based
4251+
}
4252+
}
4253+
4254+
// If we have LOBs → fall back to row-by-row fetch + SQLGetData_wrap
4255+
if (!lobColumns.empty()) {
4256+
LOG("FetchAll_wrap: LOB columns detected (%zu columns), using per-row "
4257+
"SQLGetData path",
4258+
lobColumns.size());
4259+
while (true) {
4260+
ret = SQLFetch_ptr(hStmt);
4261+
if (ret == SQL_NO_DATA)
4262+
break;
4263+
if (!SQL_SUCCEEDED(ret))
4264+
return ret;
4265+
4266+
py::list row;
4267+
SQLGetData_wrap(StatementHandle, numCols, row, charEncoding,
4268+
wcharEncoding); // <-- streams LOBs correctly
4269+
rows.append(row);
4270+
}
4271+
return SQL_SUCCESS;
4272+
}
4273+
4274+
// No LOBs detected - use binding path with batch fetching
41324275
// Define a memory limit (1 GB)
41334276
const size_t memoryLimit = 1ULL * 1024 * 1024 * 1024;
41344277
size_t totalRowSize = calculateRowSize(columnNames, numCols);
@@ -4169,40 +4312,6 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows,
41694312
}
41704313
LOG("FetchAll_wrap: Fetching data in batch sizes of %d", fetchSize);
41714314

4172-
std::vector<SQLUSMALLINT> lobColumns;
4173-
for (SQLSMALLINT i = 0; i < numCols; i++) {
4174-
auto colMeta = columnNames[i].cast<py::dict>();
4175-
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
4176-
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
4177-
4178-
if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR ||
4179-
dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY ||
4180-
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML) &&
4181-
(columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE)) {
4182-
lobColumns.push_back(i + 1); // 1-based
4183-
}
4184-
}
4185-
4186-
// If we have LOBs → fall back to row-by-row fetch + SQLGetData_wrap
4187-
if (!lobColumns.empty()) {
4188-
LOG("FetchAll_wrap: LOB columns detected (%zu columns), using per-row "
4189-
"SQLGetData path",
4190-
lobColumns.size());
4191-
while (true) {
4192-
ret = SQLFetch_ptr(hStmt);
4193-
if (ret == SQL_NO_DATA)
4194-
break;
4195-
if (!SQL_SUCCEEDED(ret))
4196-
return ret;
4197-
4198-
py::list row;
4199-
SQLGetData_wrap(StatementHandle, numCols, row, charEncoding,
4200-
wcharEncoding); // <-- streams LOBs correctly
4201-
rows.append(row);
4202-
}
4203-
return SQL_SUCCESS;
4204-
}
4205-
42064315
ColumnBuffers buffers(numCols, fetchSize);
42074316

42084317
// Bind columns

0 commit comments

Comments
 (0)