Skip to content
This repository was archived by the owner on Jun 3, 2025. It is now read-only.

Commit 5d022a7

Browse files
authored
[TextGeneration][Bug] Fix timer (#1294)
* create new timer if none * move timer check/creation to timer.py
1 parent 93ba723 commit 5d022a7

2 files changed

Lines changed: 11 additions & 1 deletion

File tree

src/deepsparse/transformers/engines/nl_decoder_engine.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ def __call__(
185185
186186
:return: The generated token and corresponding logits
187187
"""
188-
timer = self.timer_manager.current
188+
timer = self.timer_manager.current_or_new()
189189
if kv_cache:
190190
# if model has kv cache enabled, we need
191191
# to add the kv cache state to the input

src/deepsparse/utils/timer.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,16 @@ def all_times(self) -> Dict[str, List[float]]:
338338

339339
return all_times
340340

341+
def current_or_new(self) -> StagedTimer:
342+
"""
343+
Return the current timer if there is one, otherwise return a new one.
344+
"""
345+
if self.current:
346+
return self.current
347+
else:
348+
with self.new_timer_context(total_inference=False) as timer:
349+
return timer
350+
341351
def clear(self):
342352
for t in self._timers:
343353
t.clear()

0 commit comments

Comments
 (0)