ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2025-01-26 06:09:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update internet.py

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-09-10 10:21:33 +02:00 committed by GitHub
parent 082ffcb4ce
commit 39ca983490
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lollms/internet.py
View File

@ -213,11 +213,13 @@ def get_relevant_text_block(
vectorizer:VectorDatabase,
title=None,
):
sm = ScrapeMaster(url)
result = sm.scrape_all()
if len(result["texts"])>0:
vectorizer.add_document(title if title else url, "\n".join(result["texts"]), url)
try:
sm = ScrapeMaster(url)
result = sm.scrape_all()
if len(result["texts"])>0:
vectorizer.add_document(title if title else url, "\n".join(result["texts"]), url)
except Exception as ex:
trace_exception(ex)
def extract_results(url, max_num, driver=None, wait_step_delay=0.5):