ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-18 20:27:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

upgraded

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-12-09 23:30:40 +01:00
parent 82e10f2eca
commit b515a38645
8 changed files with 38 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
configs/config.yaml
View File

@ -1,5 +1,5 @@
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
version: 144
version: 145
# video viewing and news recovering
last_viewed_video: null
@ -280,7 +280,8 @@ audio_silenceTimer: 5000
# Data vectorization
rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data
rag_vectorizer: semantic # possible values semantic, tfidf, openai
rag_vectorizer: semantic # possible values semantic, tfidf, openai, ollama
rag_service_url: "http://localhost:11434" # rag service url for ollama
rag_vectorizer_model: "BAAI/bge-m3" # The model name if applicable
rag_vectorizer_parameters: null # Parameters of the model in json format
rag_chunk_size: 512 # number of tokens per chunk

5
lollms/configs/config.yaml
View File

@ -1,5 +1,5 @@
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
version: 144
version: 145
# video viewing and news recovering
last_viewed_video: null
@ -280,7 +280,8 @@ audio_silenceTimer: 5000
# Data vectorization
rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data
rag_vectorizer: semantic # possible values semantic, tfidf, openai
rag_vectorizer: semantic # possible values semantic, tfidf, openai, ollama
rag_service_url: "http://localhost:11434" # rag service url for ollama
rag_vectorizer_model: "BAAI/bge-m3" # The model name if applicable
rag_vectorizer_parameters: null # Parameters of the model in json format
rag_chunk_size: 512 # number of tokens per chunk

3
lollms/databases/discussions_database.py
View File

@ -783,6 +783,9 @@ class Discussion:
elif self.lollms.config.rag_vectorizer=="openai":
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
vectorizer = OpenAIVectorizer(self.lollms.config.rag_vectorizer_model, self.lollms.config.rag_vectorizer_openai_key)
elif self.config.rag_vectorizer == "ollama":
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer(self.lollms.config.rag_vectorizer_model, self.lollms.config.rag_service_url)
self.vectorizer = VectorDatabase(
self.discussion_rag_folder/"db.sqli",

4
lollms/databases/skills_database.py
View File

@ -22,6 +22,10 @@ class SkillsLibrary:
elif vectorizer == "openai":
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
v = OpenAIVectorizer()
elif self.config.rag_vectorizer == "ollama":
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer()
else:
from lollmsvectordb.lollms_vectorizers.semantic_vectorizer import SemanticVectorizer
v = SemanticVectorizer("BAAI/bge-m3")

3
lollms/internet.py
View File

@ -347,6 +347,9 @@ def internet_search_with_vectorization(query, chromedriver_path=None, internet_n
elif vectorizer == "openai":
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
v = OpenAIVectorizer()
elif vectorizer == "ollama":
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer()
vectorizer = VectorDatabase("", v, TikTokenTokenizer(), internet_vectorization_chunk_size, internet_vectorization_overlap_size)

11
lollms/personality.py
View File

@ -1379,6 +1379,9 @@ Use this structure:
elif self.config.rag_vectorizer == "openai":
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
v = OpenAIVectorizer(api_key=self.config.rag_vectorizer_openai_key)
elif self.config.rag_vectorizer == "ollama":
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer(self.config.rag_vectorizer_model, self.config.rag_service_url)
self.persona_data_vectorizer = VectorDatabase(self.database_path, v, TikTokenTokenizer(), self.config.rag_chunk_size, self.config.rag_overlap)
@ -1543,7 +1546,10 @@ Use this structure:
self.ShowBlockingMessage("Processing file\nPlease wait ...\nUsing open ai vectorizer")
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
v = OpenAIVectorizer()
elif self.config.rag_vectorizer == "ollama":
self.ShowBlockingMessage("Processing file\nPlease wait ...\nUsing ollama vectorizer")
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer(self.config.rag_vectorizer_model, self.config.rag_service_url)
self.vectorizer = VectorDatabase(
client.discussion.discussion_rag_folder/"db.sqli",
v,
@ -3741,6 +3747,9 @@ transition-all duration-300 ease-in-out">
elif vectorizer == "openai":
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
v = OpenAIVectorizer()
elif self.config.rag_vectorizer == "ollama":
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer(self.config.rag_vectorizer_model, self.config.rag_service_url)
vectorizer = VectorDatabase("", v, TikTokenTokenizer(), self.config.rag_chunk_size, self.config.rag_overlap)
vectorizer.add_document(title, text, url)

9
lollms/server/endpoints/lollms_file_system.py
View File

@ -142,6 +142,9 @@ def select_rag_database(client) -> Optional[Dict[str, Path]]:
elif lollmsElfServer.config.rag_vectorizer == "openai":
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
v = OpenAIVectorizer(lollmsElfServer.config.rag_vectorizer_openai_key)
elif lollmsElfServer.config.rag_vectorizer == "ollama":
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer(lollmsElfServer.config.rag_vectorizer_model, lollmsElfServer.config.rag_service_url)
vdb = VectorDatabase(Path(folder_path)/f"{db_name}.sqlite", v, lollmsElfServer.model if lollmsElfServer.model else TikTokenTokenizer())
# Get all files in the folder
@ -277,6 +280,9 @@ def toggle_mount_rag_database(database_infos: MountDatabase):
elif lollmsElfServer.config.rag_vectorizer == "openai":
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
v = OpenAIVectorizer(lollmsElfServer.config.rag_vectorizer_openai_key)
elif lollmsElfServer.config.rag_vectorizer == "ollama":
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer(lollmsElfServer.config.rag_vectorizer_model, lollmsElfServer.config.rag_service_url)
vdb = VectorDatabase(Path(path)/f"{database_infos.database_name}.sqlite", v, lollmsElfServer.model if lollmsElfServer.model else TikTokenTokenizer(), chunk_size=lollmsElfServer.config.rag_chunk_size, clean_chunks=lollmsElfServer.config.rag_clean_chunks, n_neighbors=lollmsElfServer.config.rag_n_chunks)
@ -344,6 +350,9 @@ async def vectorize_folder(database_infos: FolderInfos):
elif lollmsElfServer.config.rag_vectorizer == "openai":
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
v = OpenAIVectorizer(lollmsElfServer.config.rag_vectorizer_openai_key)
elif lollmsElfServer.config.rag_vectorizer == "ollama":
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer(lollmsElfServer.config.rag_vectorizer_model, lollmsElfServer.config.rag_service_url)
vector_db_path = Path(folder_path)/f"{db_name}.sqlite"

3
lollms/server/endpoints/lollms_rag.py
View File

@ -75,6 +75,9 @@ def get_user_vectorizer(user_key: str):
elif lollmsElfServer.config.rag_vectorizer == "openai":
from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer
v = OpenAIVectorizer(lollmsElfServer.config.rag_vectorizer_openai_key)
elif lollmsElfServer.config.rag_vectorizer == "ollama":
from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer
v = OllamaVectorizer(lollmsElfServer.config.rag_vectorizer_model, lollmsElfServer.config.rag_service_url)
return VectorDatabase(
"",