fixbug: dead loop

This commit is contained in:
莘权 马 2023-08-31 22:43:58 +08:00
parent 67d08cb054
commit 614bdf9e74

View file

@ -286,13 +286,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
windows = []
idx = 0
while idx < total_len:
data_len = window_size - padding_size
if data_len + idx > total_len:
if window_size + idx > total_len: # 不足一个滑窗
windows.append(text[idx:])
break
w = text[idx : idx + data_len]
# 第一个窗口少算自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....]
# window_size=3, padding_size=1
# [1, 2, 3], [3, 4, 5], [5, 6, 7], ....
# idx=2, | idx=5 | idx=8 | ...
w = text[idx : idx + window_size]
windows.append(w)
idx += data_len
idx += window_size - padding_size if idx == 0 else window_size
for i in range(len(windows)):
if i + 1 == len(windows):
break