diff --git a/MANIFEST.in b/MANIFEST.in index 561e452..a8f9716 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ recursive-include lollms/configs * global-exclude *.bin global-exclude *.pyc global-exclude local_config.yaml +global-exclude global_paths_cfg.yaml global-exclude .installed global-exclude .git global-exclude .gitignore \ No newline at end of file diff --git a/lollms/config.py b/lollms/config.py index eeebb39..6c91511 100644 --- a/lollms/config.py +++ b/lollms/config.py @@ -242,7 +242,15 @@ class BaseConfig: def from_template(template:ConfigTemplate, exceptional_keys: list = [], file_path: Path | str = None): config = {} for entry in template.template: - config[entry["name"]]=entry["value"] + if entry["type"]!="list": + config[entry["name"]]=entry["value"] + else: + try: + config[entry["name"]]=eval(entry["value"]) + except Exception as ex: + ASCIIColors.error(f'Could not set parameter {entry["name"]}. Exception occures : {ex}') + config[entry["name"]]=[] + return BaseConfig(exceptional_keys, config, file_path) def to_dict(self): diff --git a/lollms/console.py b/lollms/console.py index e02f906..ef96c68 100644 --- a/lollms/console.py +++ b/lollms/console.py @@ -23,12 +23,14 @@ class LollmsApplication: def load_personality(self): pass +def reset_paths(lollms_paths:LollmsPaths): + lollms_paths.resetPaths() def reset_all_installs(lollms_paths:LollmsPaths): ASCIIColors.info("Removeing all configuration files to force reinstall") ASCIIColors.info(f"Searching files from {lollms_paths.personal_configuration_path}") for file_path in lollms_paths.personal_configuration_path.iterdir(): - if file_path.name!="local_config.yaml" and file_path.suffix.lower()==".yaml": + if file_path.name!=f"{lollms_paths.tool_prefix}local_config.yaml" and file_path.suffix.lower()==".yaml": file_path.unlink() ASCIIColors.info(f"Deleted file: {file_path}") @@ -234,6 +236,7 @@ class MainMenu: print(f"{ASCIIColors.color_green}4 -{ASCIIColors.color_reset} Reinstall current Binding") print(f"{ASCIIColors.color_green}5 -{ASCIIColors.color_reset} Reinstall current Personality") print(f"{ASCIIColors.color_green}6 -{ASCIIColors.color_reset} Reset all installs") + print(f"{ASCIIColors.color_green}7 -{ASCIIColors.color_reset} Reset paths") print(f"{ASCIIColors.color_green}0 -{ASCIIColors.color_reset} Back to app") print(f"{ASCIIColors.color_green}-1 -{ASCIIColors.color_reset} Help") print(f"{ASCIIColors.color_green}-2 -{ASCIIColors.color_reset} Exit app") @@ -250,6 +253,8 @@ class MainMenu: self.reinstall_personality() elif choice == "6": reset_all_installs() + elif choice == "6": + reset_paths() elif choice == "0": print("Back to main app...") @@ -280,7 +285,7 @@ class Conversation(LollmsApplication): self.bot_says = "" # get paths - lollms_paths = LollmsPaths.find_paths(force_local=False) + lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix="lollms_server_") # Configuration loading part config = LOLLMSConfig.autoload(lollms_paths, configuration_path) @@ -606,7 +611,8 @@ def main(): LollmsPaths.reset_configs() if args.reset_config: - cfg_path = LollmsPaths.find_paths().personal_configuration_path / "local_config.yaml" + lollms_paths = LollmsPaths.find_paths(tool_prefix="lollms_server_") + cfg_path = lollms_paths.personal_configuration_path / f"{lollms_paths.tool_prefix}local_config.yaml" try: cfg_path.unlink() ASCIIColors.success("LOLLMS configuration reset successfully") diff --git a/lollms/main_config.py b/lollms/main_config.py index 9d2ff05..daddbd6 100644 --- a/lollms/main_config.py +++ b/lollms/main_config.py @@ -66,11 +66,11 @@ class LOLLMSConfig(BaseConfig): @staticmethod - def autoload(lollms_paths, config_path:str=None): + def autoload(lollms_paths:LollmsPaths, config_path:str=None): # Configuration loading part original_cfg_path = lollms_paths.default_cfg_path if config_path is None: - local = lollms_paths.personal_configuration_path / "local_config.yaml" + local = lollms_paths.personal_configuration_path / f"{lollms_paths.tool_prefix}local_config.yaml" if not local.exists(): shutil.copy(original_cfg_path, local) cfg_path = local diff --git a/lollms/paths.py b/lollms/paths.py index 9a03d6b..0ff4414 100644 --- a/lollms/paths.py +++ b/lollms/paths.py @@ -15,37 +15,53 @@ bindings_zoo_repo = "https://github.com/ParisNeo/lollms_bindings_zoo.git" # Now we speify the personal folders class LollmsPaths: - def __init__(self, lollms_path=None, personal_path=None, custom_default_cfg_path=None): + def __init__(self, global_paths_cfg_path=None, lollms_path=None, personal_path=None, custom_default_cfg_path=None, tool_prefix=""): + self.global_paths_cfg_path = global_paths_cfg_path + self.tool_prefix = tool_prefix if lollms_path is None: - lollms_path = Path(__file__).parent + lollms_path = Path(__file__).parent else: - lollms_path = Path(lollms_path) + lollms_path = Path(lollms_path) if personal_path is None: - personal_path = Path.home() / "Documents/lollms" + personal_path = Path.home() / "Documents/lollms" else: - personal_path = Path(personal_path) + personal_path = Path(personal_path) if custom_default_cfg_path is not None: - self.default_cfg_path = Path(custom_default_cfg_path) + self.default_cfg_path = Path(custom_default_cfg_path) else: - self.default_cfg_path = lollms_path / "configs/config.yaml" + self.default_cfg_path = lollms_path / "configs/config.yaml" - self.personal_path = personal_path - self.personal_configuration_path = personal_path / "configs" - self.personal_data_path = personal_path / "data" - self.personal_databases_path = personal_path / "databases" - self.personal_models_path = personal_path / "models" - self.personal_uploads_path = lollms_path / "uploads" - self.personal_log_path = lollms_path / "logs" + self.personal_path = personal_path + self.personal_configuration_path = personal_path / "configs" + self.personal_data_path = personal_path / "data" + self.personal_databases_path = personal_path / "databases" + self.personal_models_path = personal_path / "models" + self.personal_uploads_path = lollms_path / "uploads" + self.personal_log_path = lollms_path / "logs" - self.bindings_zoo_path = personal_path / "bindings_zoo" - self.personalities_zoo_path = personal_path / "personalities_zoo" + self.bindings_zoo_path = personal_path / "bindings_zoo" + self.personalities_zoo_path = personal_path / "personalities_zoo" self.create_directories() self.copy_default_config() + def __str__(self) -> str: + directories = { + "Global paths configuration Path": self.global_paths_cfg_path, + "Personal Configuration Path": self.personal_configuration_path, + "Personal Data Path": self.personal_data_path, + "Personal Databases Path": self.personal_databases_path, + "Personal Models Path": self.personal_models_path, + "Personal Uploads Path": self.personal_uploads_path, + "Personal Log Path": self.personal_log_path, + "Bindings Zoo Path": self.bindings_zoo_path, + "Personalities Zoo Path": self.personalities_zoo_path + } + return "\n".join([f"{category}: {path}" for category, path in directories.items()]) + def change_personal_path(self, path): self.personal_path = path @@ -78,53 +94,97 @@ class LollmsPaths: def copy_default_config(self): - local_config_path = self.personal_configuration_path / "local_config.yaml" + local_config_path = self.personal_configuration_path / f"{self.tool_prefix}local_config.yaml" if not local_config_path.exists(): shutil.copy(self.default_cfg_path, str(local_config_path)) + def resetPaths(self, force_local=None): + global_paths_cfg_path = Path(f"./{self.tool_prefix}global_paths_cfg.yaml") + + if force_local is None: + if global_paths_cfg_path.exists(): + force_local = True + else: + force_local = False + print(f"To make it clear where your data are stored, we now give the user the choice where to put its data.") + print(f"This allows you to mutualize models which are heavy, between multiple lollms compatible apps.") + print(f"You can change this at any tome using the lollms-settings script or by simply change the content of the global_paths_cfg.yaml file.") + found = False + while not found: + print(f"Please provide a folder to store your configurations files, your models and your personal data (database, custom personalities etc).") + cfg = BaseConfig(config={ + "lollms_path":str(Path(__file__).parent), + "lollms_personal_path":str(Path.home()/"Documents/lollms") + }) + + cfg.lollms_personal_path = input(f"Folder path: ({cfg.lollms_personal_path}):") + if cfg.lollms_personal_path=="": + cfg.lollms_personal_path = str(Path.home()/"Documents/lollms") + + print(f"Selected: {cfg.lollms_personal_path}") + pp= Path(cfg.lollms_personal_path) + if not pp.exists(): + try: + pp.mkdir(parents=True) + except: + print(f"{ASCIIColors.color_red}It seams there is an error in the path you rovided{ASCIIColors.color_reset}") + continue + if force_local: + global_paths_cfg_path = Path(f"./{self.tool_prefix}global_paths_cfg.yaml") + else: + global_paths_cfg_path = Path.home()/f"{self.tool_prefix}global_paths_cfg.yaml" + cfg.save_config(global_paths_cfg_path) + found = True + + return LollmsPaths(global_paths_cfg_path, cfg.lollms_path, cfg.lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path) + @staticmethod - def find_paths(force_local=False, custom_default_cfg_path=None): + def find_paths(force_local=False, custom_default_cfg_path=None, tool_prefix=""): lollms_path = Path(__file__).parent - global_paths_cfg = Path("./global_paths_cfg.yaml") - if global_paths_cfg.exists(): + global_paths_cfg_path = Path(f"./{tool_prefix}global_paths_cfg.yaml") + if global_paths_cfg_path.exists(): try: cfg = BaseConfig() - cfg.load_config(global_paths_cfg) + cfg.load_config(global_paths_cfg_path) lollms_path = cfg.lollms_path lollms_personal_path = cfg.lollms_personal_path - return LollmsPaths(lollms_path, lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path) + return LollmsPaths(global_paths_cfg_path, lollms_path, lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path, tool_prefix=tool_prefix) except Exception as ex: print(f"{ASCIIColors.color_red}Global paths configuration file found but seems to be corrupted{ASCIIColors.color_reset}") print("Couldn't find your personal data path!") - cfg.lollms_path = Path(__file__).parent - cfg.lollms_personal_path = input("Please specify the folder where your configuration files, your models and your custom personalities need to be stored:") - cfg.save_config(global_paths_cfg) + cfg.lollms_path = str(Path(__file__).parent) + cfg["lollms_personal_path"] = str(Path.home()/"Documents/lollms") + print("Please specify the folder where your configuration files, your models and your custom personalities need to be stored:") + lollms_personal_path = input(f"Folder path: ({cfg.lollms_personal_path}):") + if lollms_personal_path!="": + cfg.lollms_personal_path=lollms_personal_path + cfg.save_config(global_paths_cfg_path) lollms_path = cfg.lollms_path lollms_personal_path = cfg.lollms_personal_path - return LollmsPaths(lollms_path, lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path) + return LollmsPaths(global_paths_cfg_path, lollms_path, lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path, tool_prefix=tool_prefix) else: # if the app is not forcing a specific path, then try to find out if the default installed library has specified a default path - global_paths_cfg = lollms_path/"global_paths_cfg.yaml" - if global_paths_cfg.exists(): + global_paths_cfg_path = Path.home()/f"{tool_prefix}global_paths_cfg.yaml" + if global_paths_cfg_path.exists(): cfg = BaseConfig() - cfg.load_config(global_paths_cfg) + cfg.load_config(global_paths_cfg_path) try: lollms_path = cfg.lollms_path lollms_personal_path = cfg.lollms_personal_path - return LollmsPaths(lollms_path, lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path) + return LollmsPaths(global_paths_cfg_path, lollms_path, lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path, tool_prefix=tool_prefix) except Exception as ex: print(f"{ASCIIColors.color_red}Global paths configuration file found but seems to be corrupted{ASCIIColors.color_reset}") cfg.lollms_path = Path(__file__).parent cfg.lollms_personal_path = input("Please specify the folder where your configuration files, your models and your custom personalities need to be stored:") - cfg.save_config(global_paths_cfg) + cfg.save_config(global_paths_cfg_path) lollms_path = cfg.lollms_path lollms_personal_path = cfg.lollms_personal_path - return LollmsPaths(lollms_path, lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path) + return LollmsPaths(global_paths_cfg_path, lollms_path, lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path, tool_prefix=tool_prefix) else: # First time print(f"{ASCIIColors.color_green}Welcome! It seems this is your first use of the new lollms app.{ASCIIColors.color_reset}") print(f"To make it clear where your data are stored, we now give the user the choice where to put its data.") print(f"This allows you to mutualize models which are heavy, between multiple lollms compatible apps.") - print(f"You can change this at any tome using the lollms-update_path script or by simply change the content of the global_paths_cfg.yaml file.") + print(f"You can change this at any tome using the lollms-settings script or by simply change the content of the global_paths_cfg.yaml file.") found = False while not found: print(f"Please provide a folder to store your configurations files, your models and your personal data (database, custom personalities etc).") @@ -146,68 +206,26 @@ class LollmsPaths: print(f"{ASCIIColors.color_red}It seams there is an error in the path you rovided{ASCIIColors.color_reset}") continue if force_local: - global_paths_cfg = Path("./global_paths_cfg.yaml") + global_paths_cfg_path = Path(f"./{tool_prefix}global_paths_cfg.yaml") else: - global_paths_cfg = lollms_path/"global_paths_cfg.yaml" - cfg.save_config(global_paths_cfg) + global_paths_cfg_path = Path.home()/f"{tool_prefix}global_paths_cfg.yaml" + cfg.save_config(global_paths_cfg_path) found = True - return LollmsPaths(cfg.lollms_path, cfg.lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path) + return LollmsPaths(global_paths_cfg_path, cfg.lollms_path, cfg.lollms_personal_path, custom_default_cfg_path=custom_default_cfg_path, tool_prefix=tool_prefix) @staticmethod - def reset_configs(): + def reset_configs(tool_prefix=""): lollms_path = Path(__file__).parent - global_paths_cfg = Path("./global_paths_cfg.yaml") - if global_paths_cfg.exists(): + global_paths_cfg_path = Path(f"./{tool_prefix}global_paths_cfg.yaml") + if global_paths_cfg_path.exists(): ASCIIColors.error("Resetting local settings") - global_paths_cfg.unlink() + global_paths_cfg_path.unlink() return - global_paths_cfg = lollms_path/"global_paths_cfg.yaml" - if global_paths_cfg.exists(): + global_paths_cfg_path = Path.home()/f"{tool_prefix}global_paths_cfg.yaml" + if global_paths_cfg_path.exists(): ASCIIColors.error("Resetting global settings") - global_paths_cfg.unlink() + global_paths_cfg_path.unlink() -# Try to find out if the application has a global paths config -# If the application has a local configuration file that points us to the paths configuration then load it -""" -global_paths_cfg = Path("./global_paths_cfg.yaml") -if global_paths_cfg.exists(): - cfg = BaseConfig() - cfg.load_config(global_paths_cfg) - try: - lollms_personal_path = cfg.global_path - except Exception as ex: - print("Couldn't find your global path!") - cfg.global_path = input("Please specify the folder where your configuration files, your models and your custom personalities need to be stored:") - lollms_personal_path = cfg.global_path - cfg.save_config(global_paths_cfg) -else: - # if the app is not forcing a specific path, then try to find out if the default installed library has specified a default path - global_paths_cfg = lollms_path/"global_paths_cfg.yaml" - if global_paths_cfg.exists(): - cfg = BaseConfig() - cfg.load_config(global_paths_cfg) - try: - lollms_personal_path = cfg.global_path - except Exception as ex: - print("Couldn't find your global path!") - cfg.global_path = input("Please specify the folder where your configuration files, your models and your custom personalities need to be stored:") - lollms_personal_path = cfg.global_path - cfg.save_config(global_paths_cfg) - - lollms_personal_path = Path.home()/"Documents/lollms" - -lollms_personal_configuration_path = lollms_personal_path/"configs" -lollms_personal_models_path = lollms_personal_path/"models" - -lollms_personal_path.mkdir(parents=True, exist_ok=True) -lollms_personal_configuration_path.mkdir(parents=True, exist_ok=True) -lollms_personal_models_path.mkdir(parents=True, exist_ok=True) - -if not(lollms_personal_configuration_path/"local_config.yaml").exists(): - shutil.copy(lollms_path / "configs/config.yaml", str(lollms_personal_configuration_path/"local_config.yaml")) - - -""" diff --git a/lollms/server.py b/lollms/server.py index 88e7b59..18fd3b3 100644 --- a/lollms/server.py +++ b/lollms/server.py @@ -23,7 +23,7 @@ def reset_all_installs(lollms_paths:LollmsPaths): ASCIIColors.info("Removeing all configuration files to force reinstall") ASCIIColors.info(f"Searching files from {lollms_paths.personal_configuration_path}") for file_path in lollms_paths.personal_configuration_path.iterdir(): - if file_path.name!="local_config.yaml" and file_path.suffix.lower()==".yaml": + if file_path.name!=f"{lollms_paths.tool_prefix}local_config.yaml" and file_path.suffix.lower()==".yaml": file_path.unlink() ASCIIColors.info(f"Deleted file: {file_path}") @@ -40,7 +40,7 @@ class LoLLMsServer: self.is_ready = True - self.lollms_paths = LollmsPaths.find_paths(force_local=False) + self.lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix="lollms_server_") self.menu = MainMenu(self) parser = argparse.ArgumentParser() parser.add_argument('--host', '-hst', default=host, help='Host name') @@ -55,7 +55,7 @@ class LoLLMsServer: parser.add_argument('--models_path', '-mp', default=str(self.lollms_paths.personal_models_path), help='The path to the models folder') - parser.add_argument('--binding_name', '-b', default="llama_cpp_official", + parser.add_argument('--binding_name', '-b', default=None, help='Binding to be used by default') parser.add_argument('--model_name', '-m', default=None, help='Model name') @@ -76,7 +76,8 @@ class LoLLMsServer: LollmsPaths.reset_configs() if args.reset_config: - cfg_path = LollmsPaths.find_paths().personal_configuration_path / "local_config.yaml" + lollms_paths = LollmsPaths.find_paths(tool_prefix="lollms_server_") + cfg_path = lollms_paths.personal_configuration_path / f"{lollms_paths.tool_prefix}local_config.yaml" try: cfg_path.unlink() ASCIIColors.success("LOLLMS configuration reset successfully") diff --git a/lollms/settings.py b/lollms/settings.py index b001a4b..8258670 100644 --- a/lollms/settings.py +++ b/lollms/settings.py @@ -14,7 +14,7 @@ def reset_all_installs(lollms_paths:LollmsPaths): ASCIIColors.info("Removeing all configuration files to force reinstall") ASCIIColors.info(f"Searching files from {lollms_paths.personal_configuration_path}") for file_path in lollms_paths.personal_configuration_path.iterdir(): - if file_path.name!="local_config.yaml" and file_path.suffix.lower()==".yaml": + if file_path.name!=f"{lollms_paths.tool_prefix}local_config.yaml" and file_path.suffix.lower()==".yaml": file_path.unlink() ASCIIColors.info(f"Deleted file: {file_path}") @@ -36,7 +36,10 @@ class Settings: self.bot_says = "" # get paths - self.lollms_paths = LollmsPaths.find_paths(force_local=False) + self.lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix="lollms_server_") + ASCIIColors.yellow("------ Lollms Paths ------") + ASCIIColors.info(self.lollms_paths) + ASCIIColors.yellow("------ ------------ ------") # Build menu self.menu = MainMenu(self) @@ -44,7 +47,7 @@ class Settings: # Change configuration original = self.lollms_paths.default_cfg_path if configuration_path is None: - local = self.lollms_paths.personal_configuration_path / "local_config.yaml" + local = self.lollms_paths.personal_configuration_path / f"{self.lollms_paths.tool_prefix}local_config.yaml" else: local = Path(configuration_path) @@ -54,12 +57,11 @@ class Settings: self.config = LOLLMSConfig(self.cfg_path, self.lollms_paths) # load binding - self.load_binding() - - # Load model - self.load_model() - # cfg.binding_name = llm_binding.binding_folder_name - # cfg.model_name = model_name + if self.config.binding_name is not None: + self.load_binding() + # Load model + if self.config.model_name is not None: + self.load_model() # Load personality try: @@ -252,7 +254,8 @@ def main(): LollmsPaths.reset_configs() if args.reset_config: - cfg_path = LollmsPaths.find_paths().personal_configuration_path / "local_config.yaml" + lollms_paths = LollmsPaths.find_paths(tool_prefix="lollms_server_") + cfg_path = lollms_paths.personal_configuration_path / f"{lollms_paths.tool_prefix}local_config.yaml" try: cfg_path.unlink() ASCIIColors.success("LOLLMS configuration reset successfully") diff --git a/setup.py b/setup.py index 24b731c..b7b4c64 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_all_files(path): setuptools.setup( name="lollms", - version="2.0.32", + version="2.1.10", author="Saifeddine ALOUI", author_email="aloui.saifeddine@gmail.com", description="A python library for AI personality definition",