From f1ef8962a55b6abcbd76fb41a6b31ef9366b6e73 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Sun, 8 Oct 2023 01:14:44 +0200 Subject: [PATCH] Moved to the new external modules foir data vectorization and terminal printing --- lollms/app.py | 2 +- lollms/apps/console/__init__.py | 3 +- lollms/apps/server/__init__.py | 4 +- lollms/apps/settings/__init__.py | 3 +- lollms/apps/train/__init__.py | 2 +- lollms/binding.py | 3 +- lollms/config.py | 3 +- lollms/extension.py | 3 +- lollms/helpers.py | 74 +------------------------------- lollms/main_config.py | 2 +- lollms/paths.py | 2 +- lollms/personality.py | 12 +++--- lollms/terminal.py | 3 +- lollms/utilities.py | 2 +- requirements.txt | 5 ++- requirements_dev.txt | 3 +- 16 files changed, 32 insertions(+), 94 deletions(-) diff --git a/lollms/app.py b/lollms/app.py index 7969647..94140e3 100644 --- a/lollms/app.py +++ b/lollms/app.py @@ -1,5 +1,4 @@ from lollms.main_config import LOLLMSConfig -from lollms.helpers import ASCIIColors from lollms.paths import LollmsPaths from lollms.personality import PersonalityBuilder from lollms.binding import LLMBinding, BindingBuilder, ModelBuilder @@ -8,6 +7,7 @@ from lollms.config import InstallOption from lollms.helpers import trace_exception from lollms.terminal import MainMenu from typing import Callable +from ascii_colors import ASCIIColors import subprocess import importlib diff --git a/lollms/apps/console/__init__.py b/lollms/apps/console/__init__.py index 7a12c58..de51e1e 100644 --- a/lollms/apps/console/__init__.py +++ b/lollms/apps/console/__init__.py @@ -2,7 +2,8 @@ from lollms.config import InstallOption from lollms.binding import BindingBuilder, ModelBuilder from lollms.personality import MSG_TYPE, PersonalityBuilder from lollms.main_config import LOLLMSConfig -from lollms.helpers import ASCIIColors +from ascii_colors import ASCIIColors + from lollms.paths import LollmsPaths from lollms.app import LollmsApplication from lollms.terminal import MainMenu diff --git a/lollms/apps/server/__init__.py b/lollms/apps/server/__init__.py index f2eacf0..a8dc539 100644 --- a/lollms/apps/server/__init__.py +++ b/lollms/apps/server/__init__.py @@ -7,12 +7,12 @@ from lollms.personality import AIPersonality from lollms.main_config import LOLLMSConfig from lollms.binding import LLMBinding, BindingBuilder, ModelBuilder from lollms.personality import PersonalityBuilder -from lollms.helpers import ASCIIColors from lollms.apps.console import MainMenu from lollms.paths import LollmsPaths from lollms.apps.console import MainMenu from lollms.app import LollmsApplication -from lollms.utilities import TextVectorizer, trace_exception +from safe_store import TextVectorizer +from ascii_colors import ASCIIColors, trace_exception from typing import List, Tuple from typing import Callable import importlib diff --git a/lollms/apps/settings/__init__.py b/lollms/apps/settings/__init__.py index 39a4b4e..763fb93 100644 --- a/lollms/apps/settings/__init__.py +++ b/lollms/apps/settings/__init__.py @@ -1,5 +1,6 @@ from lollms.main_config import LOLLMSConfig -from lollms.helpers import ASCIIColors +from ascii_colors import ASCIIColors + from lollms.paths import LollmsPaths from pathlib import Path import argparse diff --git a/lollms/apps/train/__init__.py b/lollms/apps/train/__init__.py index 7ee53cb..4dadaa1 100644 --- a/lollms/apps/train/__init__.py +++ b/lollms/apps/train/__init__.py @@ -2,10 +2,10 @@ from lollms.config import InstallOption from lollms.binding import BindingBuilder, ModelBuilder from lollms.personality import MSG_TYPE, PersonalityBuilder from lollms.main_config import LOLLMSConfig -from lollms.helpers import ASCIIColors from lollms.paths import LollmsPaths from lollms.app import LollmsApplication from lollms.terminal import MainMenu +from ascii_colors import ASCIIColors from typing import Callable from pathlib import Path diff --git a/lollms/binding.py b/lollms/binding.py index d0914a9..a2a6265 100644 --- a/lollms/binding.py +++ b/lollms/binding.py @@ -11,7 +11,8 @@ from typing import Dict, Any from pathlib import Path from typing import Callable from lollms.paths import LollmsPaths -from lollms.helpers import ASCIIColors +from ascii_colors import ASCIIColors + import tempfile import requests diff --git a/lollms/config.py b/lollms/config.py index 3e9eb28..56a2d94 100644 --- a/lollms/config.py +++ b/lollms/config.py @@ -1,5 +1,6 @@ from pathlib import Path -from lollms.helpers import ASCIIColors +from ascii_colors import ASCIIColors + import yaml from enum import Enum diff --git a/lollms/extension.py b/lollms/extension.py index bf9d376..156bfda 100644 --- a/lollms/extension.py +++ b/lollms/extension.py @@ -3,7 +3,8 @@ # Description : # This is the main class to be imported by the extension # it gives your code access to the model, the callback functions, the model conditionning etc -from lollms.helpers import ASCIIColors +from ascii_colors import ASCIIColors + from lollms.config import InstallOption, TypedConfig, BaseConfig, ConfigTemplate from lollms.paths import LollmsPaths from enum import Enum diff --git a/lollms/helpers.py b/lollms/helpers.py index b8181d4..71c1c08 100644 --- a/lollms/helpers.py +++ b/lollms/helpers.py @@ -1,4 +1,6 @@ import traceback +from ascii_colors import ASCIIColors + def get_trace_exception(ex): """ @@ -17,75 +19,3 @@ def trace_exception(ex): """ ASCIIColors.error(get_trace_exception(ex)) - -class ASCIIColors: - # Reset - color_reset = '\u001b[0m' - - # Regular colors - color_black = '\u001b[30m' - color_red = '\u001b[31m' - color_green = '\u001b[32m' - color_yellow = '\u001b[33m' - color_blue = '\u001b[34m' - color_magenta = '\u001b[35m' - color_cyan = '\u001b[36m' - color_white = '\u001b[37m' - color_orange = '\u001b[38;5;202m' - - # Bright colors - color_bright_black = '\u001b[30;1m' - color_bright_red = '\u001b[31;1m' - color_bright_green = '\u001b[32;1m' - color_bright_yellow = '\u001b[33;1m' - color_bright_blue = '\u001b[34;1m' - color_bright_magenta = '\u001b[35;1m' - color_bright_cyan = '\u001b[36;1m' - color_bright_white = '\u001b[37;1m' - color_bright_orange = '\u001b[38;5;208m' - - @staticmethod - def print(text, color=color_bright_red, end="\n", flush=False): - print(f"{color}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - @staticmethod - def warning(text, end="\n", flush=False): - print(f"{ASCIIColors.color_bright_orange}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - @staticmethod - def error(text, end="\n", flush=False): - print(f"{ASCIIColors.color_bright_red}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - @staticmethod - def success(text, end="\n", flush=False): - print(f"{ASCIIColors.color_green}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - @staticmethod - def info(text, end="\n", flush=False): - print(f"{ASCIIColors.color_bright_blue}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - @staticmethod - def red(text, end="\n", flush=False): - print(f"{ASCIIColors.color_red}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - @staticmethod - def green(text, end="\n", flush=False): - print(f"{ASCIIColors.color_green}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - @staticmethod - def blue(text, end="\n", flush=False): - print(f"{ASCIIColors.color_blue}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - @staticmethod - def yellow(text, end="\n", flush=False): - print(f"{ASCIIColors.color_yellow}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - @staticmethod - def magenta(text, end="\n", flush=False): - print(f"{ASCIIColors.color_magenta}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - @staticmethod - def cyan(text, end="\n", flush=False): - print(f"{ASCIIColors.color_cyan}{text}{ASCIIColors.color_reset}", end=end, flush=flush) - - - diff --git a/lollms/main_config.py b/lollms/main_config.py index 2d179d5..009237b 100644 --- a/lollms/main_config.py +++ b/lollms/main_config.py @@ -3,7 +3,7 @@ __github__ = "https://github.com/ParisNeo/lollms" __copyright__ = "Copyright 2023, " __license__ = "Apache 2.0" -from lollms.helpers import ASCIIColors +from ascii_colors import ASCIIColors from lollms.paths import LollmsPaths from lollms.config import BaseConfig #from lollms.binding import LLMBinding diff --git a/lollms/paths.py b/lollms/paths.py index e4a8d61..cd151e7 100644 --- a/lollms/paths.py +++ b/lollms/paths.py @@ -1,6 +1,6 @@ from pathlib import Path import shutil -from lollms.helpers import ASCIIColors +from ascii_colors import ASCIIColors from lollms.config import BaseConfig import subprocess diff --git a/lollms/personality.py b/lollms/personality.py index b402449..ef091f8 100644 --- a/lollms/personality.py +++ b/lollms/personality.py @@ -15,11 +15,12 @@ import importlib import shutil import subprocess import yaml -from lollms.helpers import ASCIIColors +from ascii_colors import ASCIIColors + from lollms.types import MSG_TYPE from typing import Callable import json -from lollms.utilities import TextVectorizer, GenericDataLoader +from safe_store import TextVectorizer, GenericDataLoader, VisualizationMethod, VectorizationMethod from functools import partial from typing import Dict, Any @@ -315,14 +316,11 @@ Date: {{date}} db_path = self.lollms_paths.personal_databases_path / "personalities" / self.name / "db.json" db_path.parent.mkdir(parents=True, exist_ok=True) if self.vectorizer is None: - self.vectorizer = TextVectorizer(self.config.data_vectorization_method, # supported "model_embedding" or "ftidf_vectorizer" + self.vectorizer = TextVectorizer(VectorizationMethod(self.config.data_vectorization_method), # supported "model_embedding" or "ftidf_vectorizer" model=self.model, #needed in case of using model_embedding database_path=db_path, save_db=self.config.data_vectorization_save_db, - visualize_data_at_startup=False, - visualize_data_at_add_file=False, - visualize_data_at_generate=False, - data_visualization_method="PCA", + data_visualization_method=VisualizationMethod.PCA, database_dict=None) try: data = GenericDataLoader.read_file(path) diff --git a/lollms/terminal.py b/lollms/terminal.py index ff78bd1..bdb0202 100644 --- a/lollms/terminal.py +++ b/lollms/terminal.py @@ -1,5 +1,6 @@ -from lollms.helpers import ASCIIColors +from ascii_colors import ASCIIColors + from typing import TYPE_CHECKING if TYPE_CHECKING: from lollms.app import LollmsApplication diff --git a/lollms/utilities.py b/lollms/utilities.py index ceeaf30..c0bce7e 100644 --- a/lollms/utilities.py +++ b/lollms/utilities.py @@ -1,4 +1,4 @@ -from lollms.helpers import ASCIIColors, trace_exception +from ascii_colors import ASCIIColors, trace_exception from sklearn.feature_extraction.text import TfidfVectorizer import numpy as np from pathlib import Path diff --git a/requirements.txt b/requirements.txt index 6811d00..7d65cb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,7 @@ requests matplotlib seaborn mplcursors -scikit-learn \ No newline at end of file +scikit-learn + +ascii_colors +safe_store \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 423b0d1..2809c04 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -7,4 +7,5 @@ flask-cors simple-websocket wget setuptools -requests \ No newline at end of file +requests +safe_store \ No newline at end of file