Skip to content

Commit d3b2f01

Browse files
authored
FIX: Update time handling in Arrow integration to include fractional seconds (#499)
### Work Item / Issue Reference <!-- IMPORTANT: Please follow the PR template guidelines below. For mssql-python maintainers: Insert your ADO Work Item ID below For external contributors: Insert Github Issue number below Only one reference is required - either GitHub issue OR ADO Work Item. --> <!-- External contributors: GitHub Issue --> > GitHub Issue: #498 ------------------------------------------------------------------- ### Summary <!-- Insert your summary of changes below. Minimum 10 characters required. --> Map SQL time types to the arrow nanosecond type and include the fraction part of SQL_SS_TIME2_STRUCT in the value calculation. For consistency, also remove the unreachable SQL_TIME and SQL_TYPE_TIME branches. <!-- ### PR Title Guide > For feature requests FEAT: (short-description) > For non-feature requests like test case updates, config updates , dependency updates etc CHORE: (short-description) > For Fix requests FIX: (short-description) > For doc update requests DOC: (short-description) > For Formatting, indentation, or styling update STYLE: (short-description) > For Refactor, without any feature changes REFACTOR: (short-description) > For release related changes, without any feature changes RELEASE: #<RELEASE_VERSION> (short-description) ### Contribution Guidelines External contributors: - Create a GitHub issue first: https://github.com/microsoft/mssql-python/issues/new - Link the GitHub issue in the "GitHub Issue" section above - Follow the PR title format and provide a meaningful summary mssql-python maintainers: - Create an ADO Work Item following internal processes - Link the ADO Work Item in the "ADO Work Item" section above - Follow the PR title format and provide a meaningful summary -->
1 parent 9bc78ae commit d3b2f01

File tree

2 files changed

+16
-24
lines changed

2 files changed

+16
-24
lines changed

mssql_python/pybind/ddbc_bindings.cpp

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ struct ArrowArrayPrivateData {
247247
std::unique_ptr<uint64_t[]> varVal;
248248
std::unique_ptr<int32_t[]> dateVal;
249249
std::unique_ptr<int64_t[]> tsMicroVal;
250-
std::unique_ptr<int32_t[]> timeSecondVal;
250+
std::unique_ptr<int64_t[]> timeNanoVal;
251251
std::unique_ptr<Int128_t[]> decimalVal;
252252

253253
std::vector<uint8_t> varData;
@@ -4691,12 +4691,10 @@ SQLRETURN FetchArrowBatch_wrap(
46914691
arrowColumnProducer->dateVal = std::make_unique<int32_t[]>(arrowBatchSize);
46924692
arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->dateVal.get();
46934693
break;
4694-
case SQL_TIME:
4695-
case SQL_TYPE_TIME:
46964694
case SQL_SS_TIME2:
4697-
format = "tts";
4698-
arrowColumnProducer->timeSecondVal = std::make_unique<int32_t[]>(arrowBatchSize);
4699-
arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->timeSecondVal.get();
4695+
format = "ttn";
4696+
arrowColumnProducer->timeNanoVal = std::make_unique<int64_t[]>(arrowBatchSize);
4697+
arrowColumnProducer->ptrValueBuffer = arrowColumnProducer->timeNanoVal.get();
47004698
break;
47014699
case SQL_BIT:
47024700
format = "b";
@@ -4965,8 +4963,6 @@ SQLRETURN FetchArrowBatch_wrap(
49654963
}
49664964
break;
49674965
}
4968-
case SQL_TIME:
4969-
case SQL_TYPE_TIME:
49704966
case SQL_SS_TIME2: {
49714967
buffers.timeBuffers[idxCol].resize(1);
49724968
ret = SQLGetData_ptr(
@@ -5226,14 +5222,13 @@ SQLRETURN FetchArrowBatch_wrap(
52265222
buffers.dateBuffers[idxCol][idxRowSql].day
52275223
);
52285224
break;
5229-
case SQL_TIME:
5230-
case SQL_TYPE_TIME:
52315225
case SQL_SS_TIME2: {
52325226
const SQL_SS_TIME2_STRUCT& timeValue = buffers.timeBuffers[idxCol][idxRowSql];
5233-
arrowColumnProducer->timeSecondVal[idxRowArrow] =
5234-
static_cast<int32_t>(timeValue.hour) * 3600 +
5235-
static_cast<int32_t>(timeValue.minute) * 60 +
5236-
static_cast<int32_t>(timeValue.second);
5227+
arrowColumnProducer->timeNanoVal[idxRowArrow] =
5228+
static_cast<int64_t>(timeValue.hour) * 3600 * 1000000000 +
5229+
static_cast<int64_t>(timeValue.minute) * 60 * 1000000000 +
5230+
static_cast<int64_t>(timeValue.second) * 1000000000 +
5231+
static_cast<int64_t>(timeValue.fraction);
52375232
break;
52385233
}
52395234
case SQL_BIT: {

tests/test_004_cursor_arrow.py

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,15 +89,20 @@ def get_arrow_test_data(include_lobs: bool, batch_length: int):
8989
],
9090
),
9191
(
92-
pa.time32("s"),
92+
pa.time64("ns"),
9393
"time(0)",
9494
[time(12, 0, 5, 0), None, time(23, 59, 59, 0), time(0, 0, 0, 0)],
9595
),
9696
(
97-
pa.time32("s"),
97+
pa.time64("ns"),
9898
"time(7)",
9999
[time(12, 0, 5, 0), None, time(23, 59, 59, 0), time(0, 0, 0, 0)],
100100
),
101+
(
102+
pa.time64("ns"),
103+
"time(7)",
104+
[time(12, 0, 5, 123456), None, time(23, 59, 59, 123456), time(0, 0, 0, 0)],
105+
),
101106
(
102107
pa.timestamp("us"),
103108
"datetime2(0)",
@@ -187,14 +192,6 @@ def _test_arrow_test_data(cursor: mssql_python.Cursor, arrow_test_data, fetch_le
187192

188193
# Validate that Parquet serialization/deserialization does not detect any issues
189194
tbl = pa.Table.from_batches([ret])
190-
# for some reason parquet converts seconds to milliseconds in time32
191-
for i_col, col in enumerate(tbl.columns):
192-
if col.type == pa.time32("s"):
193-
tbl = tbl.set_column(
194-
i_col,
195-
tbl.schema.field(i_col).name,
196-
col.cast(pa.time32("ms")),
197-
)
198195
buffer = io.BytesIO()
199196
pq.write_table(tbl, buffer)
200197
buffer.seek(0)

0 commit comments

Comments
 (0)