ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2025-02-01 16:58:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

bugfix

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-05-18 23:11:04 +02:00
parent 352cdb7691
commit 32005ca92b
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lollms/databases/discussions_database.py
View File

@ -1043,8 +1043,8 @@ class Discussion:
formatted_text = ""
for message in reversed(self.messages): # Start from the newest message
formatted_message = f"{splitter_text}{message.sender.replace(':','').replace('!@>','')}:\n{message.content}\n"
tokenized_message = self.lollms.tokenize(formatted_message)
if len(tokenized_message) + len(self.lollms.tokenize(formatted_text)) <= max_allowed_tokens:
tokenized_message = self.lollms.model.tokenize(formatted_message)
if len(tokenized_message) + len(self.lollms.model.tokenize(formatted_text)) <= max_allowed_tokens:
formatted_text = formatted_message + formatted_text
else:
break # Stop if adding the next message would exceed the limit