ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-18 20:27:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fixed internet in advanced mode

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-07-27 02:09:35 +02:00
parent ae1cd019e1
commit f0eca91b30
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lollms/internet.py
View File

@ -11,7 +11,7 @@ def get_favicon_url(url):
import requests
from bs4 import BeautifulSoup
try:
response = requests.get(url)
response = requests.get(url,timeout=3)
soup = BeautifulSoup(response.text, 'html.parser')
favicon_link = soup.find('link', rel='icon') or soup.find('link', rel='shortcut icon')
@ -215,7 +215,8 @@ def get_relevant_text_block(
):
sm = ScrapeMaster(url)
result = sm.scrape_all()
vectorizer.add_document(title if title else url, result["content"], url)
if len(result["texts"])>0:
vectorizer.add_document(title if title else url, "\n".join(result["texts"]), url)