Moved to the new external modules foir data vectorization and terminal printing

lollms/app.py

@@ -1,5 +1,4 @@
 from lollms.main_config import LOLLMSConfig
-from lollms.helpers import ASCIIColors
 from lollms.paths import LollmsPaths
 from lollms.personality import PersonalityBuilder
 from lollms.binding import LLMBinding, BindingBuilder, ModelBuilder
@@ -8,6 +7,7 @@ from lollms.config import InstallOption
 from lollms.helpers import trace_exception
 from lollms.terminal import MainMenu
 from typing import Callable
+from ascii_colors import ASCIIColors
 
 import subprocess
 import importlib

lollms/apps/console/__init__.py

@@ -2,7 +2,8 @@ from lollms.config import InstallOption
 from lollms.binding import BindingBuilder, ModelBuilder
 from lollms.personality import MSG_TYPE, PersonalityBuilder
 from lollms.main_config import LOLLMSConfig
-from lollms.helpers import ASCIIColors
+from ascii_colors import ASCIIColors
+
 from lollms.paths import LollmsPaths
 from lollms.app import LollmsApplication
 from lollms.terminal import MainMenu

lollms/apps/server/__init__.py

@@ -7,12 +7,12 @@ from lollms.personality import AIPersonality
 from lollms.main_config import LOLLMSConfig
 from lollms.binding import LLMBinding, BindingBuilder, ModelBuilder
 from lollms.personality import PersonalityBuilder
-from lollms.helpers import ASCIIColors
 from lollms.apps.console import MainMenu
 from lollms.paths import LollmsPaths
 from lollms.apps.console import MainMenu
 from lollms.app import LollmsApplication
-from lollms.utilities import TextVectorizer, trace_exception
+from safe_store import TextVectorizer
+from ascii_colors import ASCIIColors, trace_exception
 from typing import List, Tuple
 from typing import Callable
 import importlib

lollms/apps/settings/__init__.py

@@ -1,5 +1,6 @@
 from lollms.main_config import LOLLMSConfig
-from lollms.helpers import ASCIIColors
+from ascii_colors import ASCIIColors
+
 from lollms.paths import LollmsPaths
 from pathlib import Path
 import argparse

lollms/apps/train/__init__.py

@@ -2,10 +2,10 @@ from lollms.config import InstallOption
 from lollms.binding import BindingBuilder, ModelBuilder
 from lollms.personality import MSG_TYPE, PersonalityBuilder
 from lollms.main_config import LOLLMSConfig
-from lollms.helpers import ASCIIColors
 from lollms.paths import LollmsPaths
 from lollms.app import LollmsApplication
 from lollms.terminal import MainMenu
+from ascii_colors import ASCIIColors
 
 from typing import Callable
 from pathlib import Path

lollms/binding.py

@@ -11,7 +11,8 @@ from typing import Dict, Any
 from pathlib import Path
 from typing import Callable
 from lollms.paths import LollmsPaths
-from lollms.helpers import ASCIIColors
+from ascii_colors import ASCIIColors
+
 
 import tempfile
 import requests

lollms/config.py

@@ -1,5 +1,6 @@
 from pathlib import Path
-from lollms.helpers import ASCIIColors
+from ascii_colors import ASCIIColors
+
 
 import yaml
 from enum import Enum

lollms/extension.py

@@ -3,7 +3,8 @@
 # Description :
 # This is the main class to be imported by the extension
 # it gives your code access to the model, the callback functions, the model conditionning etc
-from lollms.helpers import ASCIIColors
+from ascii_colors import ASCIIColors
+
 from lollms.config import InstallOption, TypedConfig, BaseConfig, ConfigTemplate
 from lollms.paths import LollmsPaths
 from enum import Enum

lollms/helpers.py

@@ -1,4 +1,6 @@
 import traceback
+from ascii_colors import ASCIIColors
+
 
 def get_trace_exception(ex):
     """
@@ -17,75 +19,3 @@ def trace_exception(ex):
     """
     ASCIIColors.error(get_trace_exception(ex))
 
-
-class ASCIIColors:
-    # Reset
-    color_reset = '\u001b[0m'
-
-    # Regular colors
-    color_black = '\u001b[30m'
-    color_red = '\u001b[31m'
-    color_green = '\u001b[32m'
-    color_yellow = '\u001b[33m'
-    color_blue = '\u001b[34m'
-    color_magenta = '\u001b[35m'
-    color_cyan = '\u001b[36m'
-    color_white = '\u001b[37m'
-    color_orange = '\u001b[38;5;202m'
-
-    # Bright colors
-    color_bright_black = '\u001b[30;1m'
-    color_bright_red = '\u001b[31;1m'
-    color_bright_green = '\u001b[32;1m'
-    color_bright_yellow = '\u001b[33;1m'
-    color_bright_blue = '\u001b[34;1m'
-    color_bright_magenta = '\u001b[35;1m'
-    color_bright_cyan = '\u001b[36;1m'
-    color_bright_white = '\u001b[37;1m'
-    color_bright_orange = '\u001b[38;5;208m'
-
-    @staticmethod
-    def print(text, color=color_bright_red, end="\n", flush=False):
-        print(f"{color}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    @staticmethod
-    def warning(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_bright_orange}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    @staticmethod
-    def error(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_bright_red}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    @staticmethod
-    def success(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_green}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    @staticmethod
-    def info(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_bright_blue}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    @staticmethod
-    def red(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_red}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-    @staticmethod
-    def green(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_green}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    @staticmethod
-    def blue(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_blue}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    @staticmethod
-    def yellow(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_yellow}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    @staticmethod
-    def magenta(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_magenta}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    @staticmethod
-    def cyan(text, end="\n", flush=False):
-        print(f"{ASCIIColors.color_cyan}{text}{ASCIIColors.color_reset}", end=end, flush=flush)
-
-    
-

lollms/main_config.py

@@ -3,7 +3,7 @@ __github__ = "https://github.com/ParisNeo/lollms"
 __copyright__ = "Copyright 2023, "
 __license__ = "Apache 2.0"
 
-from lollms.helpers     import ASCIIColors
+from ascii_colors       import ASCIIColors
 from lollms.paths       import LollmsPaths
 from lollms.config      import BaseConfig
 #from lollms.binding import  LLMBinding

lollms/paths.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 import shutil
-from lollms.helpers import ASCIIColors
+from ascii_colors import ASCIIColors
 from lollms.config import BaseConfig
 import subprocess
 

lollms/personality.py

@@ -15,11 +15,12 @@ import importlib
 import shutil
 import subprocess
 import yaml
-from lollms.helpers import ASCIIColors
+from ascii_colors import ASCIIColors
+
 from lollms.types import MSG_TYPE
 from typing import Callable
 import json
-from lollms.utilities import TextVectorizer, GenericDataLoader
+from safe_store import TextVectorizer, GenericDataLoader, VisualizationMethod, VectorizationMethod
 from functools import partial
 from typing import Dict, Any
 
@@ -315,14 +316,11 @@ Date: {{date}}
         db_path = self.lollms_paths.personal_databases_path / "personalities" / self.name / "db.json"
         db_path.parent.mkdir(parents=True, exist_ok=True)
         if self.vectorizer is None:
-            self.vectorizer = TextVectorizer(self.config.data_vectorization_method, # supported "model_embedding" or "ftidf_vectorizer"
+            self.vectorizer = TextVectorizer(VectorizationMethod(self.config.data_vectorization_method), # supported "model_embedding" or "ftidf_vectorizer"
                         model=self.model, #needed in case of using model_embedding
                         database_path=db_path,
                         save_db=self.config.data_vectorization_save_db,
-                        visualize_data_at_startup=False,
-                        visualize_data_at_add_file=False,
-                        visualize_data_at_generate=False,
-                        data_visualization_method="PCA",
+                        data_visualization_method=VisualizationMethod.PCA,
                         database_dict=None)
         try:
             data = GenericDataLoader.read_file(path)

lollms/terminal.py

@@ -1,5 +1,6 @@
 
-from lollms.helpers import ASCIIColors
+from ascii_colors import ASCIIColors
+
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from lollms.app import LollmsApplication

lollms/utilities.py

@@ -1,4 +1,4 @@
-from lollms.helpers import ASCIIColors, trace_exception
+from ascii_colors import ASCIIColors, trace_exception
 from sklearn.feature_extraction.text import TfidfVectorizer
 import numpy as np
 from pathlib import Path

requirements.txt

@@ -13,4 +13,7 @@ requests
 matplotlib
 seaborn
 mplcursors
-scikit-learn
+scikit-learn
+
+ascii_colors
+safe_store

requirements_dev.txt

@@ -7,4 +7,5 @@ flask-cors
 simple-websocket
 wget
 setuptools
-requests
+requests
+safe_store