ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-21 13:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update personality.py

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-05-31 11:20:37 +02:00 committed by GitHub
parent 942fced394
commit fa98d94184
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lollms/personality.py
View File

@ -2221,12 +2221,11 @@ class APScript(StateMachine):
chunk_summary_post_processing=None, chunk_summary_post_processing=None,
summary_mode=SUMMARY_MODE.SUMMARY_MODE_SEQUENCIAL summary_mode=SUMMARY_MODE.SUMMARY_MODE_SEQUENCIAL
): ):
depth=0
tk = self.personality.model.tokenize(text) tk = self.personality.model.tokenize(text)
prev_len = len(tk) prev_len = len(tk)
document_chunks=None document_chunks=None
while len(tk)>max_summary_size and (document_chunks is None or len(document_chunks)>1): while len(tk)>max_summary_size and (document_chunks is None or len(document_chunks)>1):
self.step_start(f"Comprerssing {doc_name}... [depth {depth+1}]") self.step_start(f"Comprerssing {doc_name}...")
chunk_size = int(self.personality.config.ctx_size*0.6) chunk_size = int(self.personality.config.ctx_size*0.6)
document_chunks = DocumentDecomposer.decompose_document(text, chunk_size, 0, self.personality.model.tokenize, self.personality.model.detokenize, True) document_chunks = DocumentDecomposer.decompose_document(text, chunk_size, 0, self.personality.model.tokenize, self.personality.model.detokenize, True)
text = self.summerize_chunks( text = self.summerize_chunks(
@ -2243,9 +2242,8 @@ class APScript(StateMachine):
dtk_ln=prev_len-len(tk) dtk_ln=prev_len-len(tk)
prev_len = len(tk) prev_len = len(tk)
self.step(f"Current text size : {prev_len}, max summary size : {max_summary_size}") self.step(f"Current text size : {prev_len}, max summary size : {max_summary_size}")
self.step_end(f"Comprerssing {doc_name}... [depth {depth+1}]") self.step_end(f"Comprerssing {doc_name}...")
depth += 1 if dtk_ln<=10: # it is not summarizing
if dtk_ln<=10: # it is not sumlmarizing
break break
return text return text