make sure this is actually doing something

This commit is contained in:
Alex Cheema
2024-12-17 16:22:22 +00:00
parent 0a07223074
commit 3a58576f8c

View File

@@ -122,6 +122,7 @@ class Node:
if request_id not in self.buffered_token_output:
self.buffered_token_output[request_id] = ([], False)
is_finished = len(self.buffered_token_output[request_id][0]) >= self.max_generate_tokens
await asyncio.sleep(0.1)
if shard.is_last_layer() and not is_finished:
self.token_count += 1