ExternalVendorCode/lollms-webui
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms-webui.git synced 2024-12-18 20:17:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update lollms_client_js.js

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-12-17 15:54:40 +01:00 committed by GitHub
parent f7314d46f6
commit 0293355386
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
endpoints/libraries/lollms_client_js.js
View File

@ -1114,7 +1114,7 @@ class TextChunker {
while (start < tokens.length) { while (start < tokens.length) {
const end = Math.min(start + chunkSize, tokens.length); const end = Math.min(start + chunkSize, tokens.length);
const chunkTokens = tokens.slice(start, end); const chunkTokens = tokens.slice(start, end);
const chunkText = tokenizer.detokenize(chunkTokens); const chunkText = await tokenizer.detokenize(chunkTokens);
chunks.push(chunkText); chunks.push(chunkText);
start += chunkSize - overlap; // Move start forward with overlap start += chunkSize - overlap; // Move start forward with overlap