From 938281b5b91206ec90602ba60d6cb6c352d1b666 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Sun, 31 Dec 2023 12:43:49 +0100 Subject: [PATCH] New scripts --- app.py | 2 +- lollms_core | 2 +- scripts/python/lollms_install.py | 0 scripts/python/models_zoo | 1 + scripts/python/xtts_start.py | 10 ++ scripts/windows/win_install copy.bat | 225 +++++++++++++++++++++++++++ scripts/windows/win_install.bat | 65 +------- web/src/views/SettingsView.vue | 22 ++- zoos/personalities_zoo | 2 +- 9 files changed, 258 insertions(+), 71 deletions(-) create mode 100644 scripts/python/lollms_install.py create mode 160000 scripts/python/models_zoo create mode 100644 scripts/python/xtts_start.py create mode 100644 scripts/windows/win_install copy.bat diff --git a/app.py b/app.py index 25f5d524..22c26d27 100644 --- a/app.py +++ b/app.py @@ -13,7 +13,7 @@ __github__ = "https://github.com/ParisNeo/lollms-webui" __copyright__ = "Copyright 2023, " __license__ = "Apache 2.0" -__version__ ="8.0" +__version__ ="8.5 (alpha)" main_repo = "https://github.com/ParisNeo/lollms-webui.git" diff --git a/lollms_core b/lollms_core index ce44d0c8..fbac0b87 160000 --- a/lollms_core +++ b/lollms_core @@ -1 +1 @@ -Subproject commit ce44d0c87d3750bc61d6ada70491efb8e4c15bf6 +Subproject commit fbac0b87312d153f7bc535867979b940f57c37ec diff --git a/scripts/python/lollms_install.py b/scripts/python/lollms_install.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/python/models_zoo b/scripts/python/models_zoo new file mode 160000 index 00000000..8ca6f513 --- /dev/null +++ b/scripts/python/models_zoo @@ -0,0 +1 @@ +Subproject commit 8ca6f513073eaea1db4bcd09c8b41e9f8abe2cc2 diff --git a/scripts/python/xtts_start.py b/scripts/python/xtts_start.py new file mode 100644 index 00000000..776468e7 --- /dev/null +++ b/scripts/python/xtts_start.py @@ -0,0 +1,10 @@ +from lollms.paths import LollmsPaths +from pathlib import Path +from ascii_colors import ASCIIColors +import subprocess + +global_path = Path(__file__).parent.parent.parent/"global_paths_cfg.yaml" +ASCIIColors.yellow(f"global_path: {global_path}") +lollms_paths = LollmsPaths(global_path) +output_folder = lollms_paths.personal_outputs_path/"audio_out" +subprocess.Popen(["python", "-m", "xtts_api_server", "-o", f"{output_folder}", "-sf", f"{lollms_paths.custom_voices_path}"]) \ No newline at end of file diff --git a/scripts/windows/win_install copy.bat b/scripts/windows/win_install copy.bat new file mode 100644 index 00000000..391902e6 --- /dev/null +++ b/scripts/windows/win_install copy.bat @@ -0,0 +1,225 @@ +@echo off + +@rem This script will install miniconda and git with all dependencies for this project +@rem This enables a user to install this project without manually installing conda and git. + +@rem workaround for broken Windows installs +set PATH=%PATH%;%SystemRoot%\system32 + +cd /D "%~dp0" + +echo "%cd%"| findstr /C:" " >nul && call :PrintBigMessage "This script relies on Miniconda which can not be silently installed under a path with spaces. Please put it in a path without spaces and try again" && goto failed +call :PrintBigMessage "WARNING: This script relies on Miniconda which will fail to install if the path is too long." +set "SPCHARMESSAGE="WARNING: Special characters were detected in the installation path!" " This can cause the installation to fail!"" +echo "%CD%"| findstr /R /C:"[!#\$%&()\*+,;<=>?@\[\]\^`{|}~]" >nul && ( + call :PrintBigMessage %SPCHARMESSAGE% +) +set SPCHARMESSAGE= + +pause +cls + +md + +echo " ___ ___ ___ ___ ___ ___ " +echo " /\__\ /\ \ /\__\ /\__\ /\__\ /\ \ " +echo " /:/ / /::\ \ /:/ / /:/ / /::| | /::\ \ " +echo " /:/ / /:/\:\ \ /:/ / /:/ / /:|:| | /:/\ \ \ " +echo " /:/ / /:/ \:\ \ /:/ / /:/ / /:/|:|__|__ _\:\~\ \ \ " +echo " /:/__/ /:/__/ \:\__\ /:/__/ /:/__/ /:/ |::::\__\ /\ \:\ \ \__\ " +echo " \:\ \ \:\ \ /:/ / \:\ \ \:\ \ \/__/~~/:/ / \:\ \:\ \/__/ " +echo " \:\ \ \:\ /:/ / \:\ \ \:\ \ /:/ / \:\ \:\__\ " +echo " \:\ \ \:\/:/ / \:\ \ \:\ \ /:/ / \:\/:/ / " +echo " \:\__\ \::/ / \:\__\ \:\__\ /:/ / \::/ / " +echo " \/__/ \/__/ \/__/ \/__/ \/__/ \/__/ " +echo By ParisNeo + +:retry +echo Please specify if you want to use a GPU or CPU. +echo *Note* that only NVidea GPUs (cuda) or AMD GPUs (rocm) are supported. +echo A) Enable cuda GPU +echo B) Enable ROCm compatible GPU (AMD and other GPUs) (NOT SUPPORTED UNDER WINDOWS) +echo C) Run CPU mode +set /p "gpuchoice=Input> " +set gpuchoice=%gpuchoice:~0,1% + +if /I "%gpuchoice%" == "A" ( + set "PACKAGES_TO_INSTALL=python=3.10 cuda-toolkit ninja git cuda-compiler" + set "CHANNEL=-c nvidia/label/cuda-12.1.1 -c nvidia -c conda-forge" +) else if /I "%gpuchoice%" == "B" ( + set "PACKAGES_TO_INSTALL=python=3.10 rocm-comgr rocm-smi ninja git" + set "CHANNEL=-c conda-forge" +) else if /I "%gpuchoice%" == "C" ( + set "PACKAGES_TO_INSTALL=python=3.10 m2w64-toolchain ninja git" + set "CHANNEL=-c conda-forge" +) else ( + echo Invalid choice. Retry + goto retry +) + +@rem better isolation for virtual environment +SET "CONDA_SHLVL=" +SET PYTHONNOUSERSITE=1 +SET "PYTHONPATH=" +SET "PYTHONHOME=" +SET "TEMP=%cd%\installer_files\temp" +SET "TMP=%cd%\installer_files\temp" + +set MINICONDA_DIR=%cd%\installer_files\miniconda3 +set INSTALL_ENV_DIR=%cd%\installer_files\lollms_env +set MINICONDA_DOWNLOAD_URL=https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe +set REPO_URL=https://github.com/ParisNeo/lollms-webui.git + +if not exist "%MINICONDA_DIR%\Scripts\conda.exe" ( + @rem download miniconda + echo Downloading Miniconda installer from %MINICONDA_DOWNLOAD_URL% + call curl -LOk "%MINICONDA_DOWNLOAD_URL%" + + @rem install miniconda + echo. && echo Installing Miniconda To "%MINICONDA_DIR%" && echo Please Wait... && echo. + start "" /W /D "%cd%" "Miniconda3-latest-Windows-x86_64.exe" /InstallationType=JustMe /NoShortcuts=1 /AddToPath=0 /RegisterPython=0 /NoRegistry=1 /S /D=%MINICONDA_DIR% || ( echo. && echo Miniconda installer not found. && goto failed ) + del /q "Miniconda3-latest-Windows-x86_64.exe" + if not exist "%MINICONDA_DIR%\Scripts\activate.bat" ( echo. && echo Miniconda install failed. && goto end ) +) + +@rem activate miniconda +call "%MINICONDA_DIR%\Scripts\activate.bat" || ( echo Miniconda hook not found. && goto end ) + +if /I "%gpuchoice%" == "B" ( + echo Installing ROCM AMD tools... + rem Set variables for the installer URL and output file + set "installerUrl=https://www.amd.com/en/developer/rocm-hub/eula/licenses.html?filename=AMD-Software-PRO-Edition-23.Q3-Win10-Win11-For-HIP.exe" + set "outputFile=AMD-Software-PRO-Edition-23.Q3-Win10-Win11-For-HIP.exe" + + rem Download the installer using curl (make sure you have curl installed) + curl -o "%outputFile%" "%installerUrl%" + + rem Install RocM tools + "%outputFile%" + + rem Clean up the downloaded installer + del "%outputFile%" + + echo Installation complete. +) + +@rem create the installer env +if not exist "%INSTALL_ENV_DIR%" ( + echo Packages to install: %PACKAGES_TO_INSTALL% + call conda create --no-shortcuts -y -k -p "%INSTALL_ENV_DIR%" %CHANNEL% %PACKAGES_TO_INSTALL% || ( echo. && echo Conda environment creation failed. && goto end ) +) + +@rem check if conda environment was actually created +if not exist "%INSTALL_ENV_DIR%\python.exe" ( echo. && echo Conda environment is empty. && goto end ) + +@rem activate installer env +call conda activate "%INSTALL_ENV_DIR%" || ( echo. && echo Conda environment activation failed. && goto end ) + +@rem set default cuda toolkit to the one in the environment +set "CUDA_PATH=%INSTALL_ENV_DIR%" + +@rem clone the repository +if exist lollms-webui\ ( + cd lollms-webui + git pull +) else ( + git clone https://github.com/ParisNeo/lollms-webui.git + cd lollms-webui +) +git submodule update --init +cd zoos/bindings_zoo +git checkout main +cd ../personalities_zoo +git checkout main +cd ../extensions_zoo +git checkout main +cd ../models_zoo +git checkout main + +cd ../.. + +cd lollms_core +git checkout main + +cd ../utilities/safe_store +git checkout main + +cd ../.. + +@rem Loop through each "git+" requirement and uninstall it workaround for inconsistent git package updating +for /F "delims=" %%a in (requirements.txt) do echo "%%a"| findstr /C:"git+" >nul&& for /F "tokens=4 delims=/" %%b in ("%%a") do for /F "delims=@" %%c in ("%%b") do python -m pip uninstall -y %%c + +@rem install the pip requirements +call python -m pip install -r requirements.txt --upgrade + +@rem install the pip requirements +call python -m pip install -e lollms_core --upgrade + +@rem install the pip requirements +call python -m pip install -e utilities/safe_store --upgrade + + +@rem create launcher +if exist ..\win_run.bat ( + echo Win run found +) else ( + copy scripts\windows\win_run.bat ..\ +) + +if exist ..\win_update.bat ( + echo Win update found +) else ( + copy scripts\windows\win_update.bat ..\ +) + + +if exist ..\win_conda_session.bat ( + echo win conda session script found +) else ( + copy scripts\windows\win_conda_session.bat ..\ +) + +if exist ..\win_update_models.bat ( + echo Win update models found +) else ( + copy scripts\windows\win_update_models.bat ..\ +) + +setlocal enabledelayedexpansion + + +if /I "%gpuchoice%"=="C" ( + echo This is a .no_gpu file. > .no_gpu +) else ( + echo GPU is enabled, no .no_gpu file will be created. +) + +endlocal + +goto end + +:PrintBigMessage +echo. && echo. +echo ******************************************************************* +for %%M in (%*) do echo * %%~M +echo ******************************************************************* +echo. && echo. +exit /b +goto end +:failed +echo Install failed +goto endend +:end +cd .. +echo Creating bin folder (needed for ctransformers) +IF EXIST "installer_files\lollms_env\bin" ( + echo Folder already existing +) ELSE ( + MKDIR "installer_files\lollms_env\bin" + echo Folder created successfully! +) + + +echo Installation complete. +:endend +pause diff --git a/scripts/windows/win_install.bat b/scripts/windows/win_install.bat index 391902e6..32e51b51 100644 --- a/scripts/windows/win_install.bat +++ b/scripts/windows/win_install.bat @@ -43,20 +43,6 @@ echo C) Run CPU mode set /p "gpuchoice=Input> " set gpuchoice=%gpuchoice:~0,1% -if /I "%gpuchoice%" == "A" ( - set "PACKAGES_TO_INSTALL=python=3.10 cuda-toolkit ninja git cuda-compiler" - set "CHANNEL=-c nvidia/label/cuda-12.1.1 -c nvidia -c conda-forge" -) else if /I "%gpuchoice%" == "B" ( - set "PACKAGES_TO_INSTALL=python=3.10 rocm-comgr rocm-smi ninja git" - set "CHANNEL=-c conda-forge" -) else if /I "%gpuchoice%" == "C" ( - set "PACKAGES_TO_INSTALL=python=3.10 m2w64-toolchain ninja git" - set "CHANNEL=-c conda-forge" -) else ( - echo Invalid choice. Retry - goto retry -) - @rem better isolation for virtual environment SET "CONDA_SHLVL=" SET PYTHONNOUSERSITE=1 @@ -70,6 +56,8 @@ set INSTALL_ENV_DIR=%cd%\installer_files\lollms_env set MINICONDA_DOWNLOAD_URL=https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe set REPO_URL=https://github.com/ParisNeo/lollms-webui.git +set "PACKAGES_TO_INSTALL=python=3.11 git" + if not exist "%MINICONDA_DIR%\Scripts\conda.exe" ( @rem download miniconda echo Downloading Miniconda installer from %MINICONDA_DOWNLOAD_URL% @@ -85,23 +73,6 @@ if not exist "%MINICONDA_DIR%\Scripts\conda.exe" ( @rem activate miniconda call "%MINICONDA_DIR%\Scripts\activate.bat" || ( echo Miniconda hook not found. && goto end ) -if /I "%gpuchoice%" == "B" ( - echo Installing ROCM AMD tools... - rem Set variables for the installer URL and output file - set "installerUrl=https://www.amd.com/en/developer/rocm-hub/eula/licenses.html?filename=AMD-Software-PRO-Edition-23.Q3-Win10-Win11-For-HIP.exe" - set "outputFile=AMD-Software-PRO-Edition-23.Q3-Win10-Win11-For-HIP.exe" - - rem Download the installer using curl (make sure you have curl installed) - curl -o "%outputFile%" "%installerUrl%" - - rem Install RocM tools - "%outputFile%" - - rem Clean up the downloaded installer - del "%outputFile%" - - echo Installation complete. -) @rem create the installer env if not exist "%INSTALL_ENV_DIR%" ( @@ -115,9 +86,6 @@ if not exist "%INSTALL_ENV_DIR%\python.exe" ( echo. && echo Conda environment is @rem activate installer env call conda activate "%INSTALL_ENV_DIR%" || ( echo. && echo Conda environment activation failed. && goto end ) -@rem set default cuda toolkit to the one in the environment -set "CUDA_PATH=%INSTALL_ENV_DIR%" - @rem clone the repository if exist lollms-webui\ ( cd lollms-webui @@ -146,19 +114,6 @@ git checkout main cd ../.. -@rem Loop through each "git+" requirement and uninstall it workaround for inconsistent git package updating -for /F "delims=" %%a in (requirements.txt) do echo "%%a"| findstr /C:"git+" >nul&& for /F "tokens=4 delims=/" %%b in ("%%a") do for /F "delims=@" %%c in ("%%b") do python -m pip uninstall -y %%c - -@rem install the pip requirements -call python -m pip install -r requirements.txt --upgrade - -@rem install the pip requirements -call python -m pip install -e lollms_core --upgrade - -@rem install the pip requirements -call python -m pip install -e utilities/safe_store --upgrade - - @rem create launcher if exist ..\win_run.bat ( echo Win run found @@ -187,13 +142,6 @@ if exist ..\win_update_models.bat ( setlocal enabledelayedexpansion - -if /I "%gpuchoice%"=="C" ( - echo This is a .no_gpu file. > .no_gpu -) else ( - echo GPU is enabled, no .no_gpu file will be created. -) - endlocal goto end @@ -211,15 +159,6 @@ echo Install failed goto endend :end cd .. -echo Creating bin folder (needed for ctransformers) -IF EXIST "installer_files\lollms_env\bin" ( - echo Folder already existing -) ELSE ( - MKDIR "installer_files\lollms_env\bin" - echo Folder created successfully! -) - - echo Installation complete. :endend pause diff --git a/web/src/views/SettingsView.vue b/web/src/views/SettingsView.vue index 1d971698..8dc51f05 100644 --- a/web/src/views/SettingsView.vue +++ b/web/src/views/SettingsView.vue @@ -1921,11 +1921,23 @@ export default { return { voices: [], - voice_languages: [ - "en", - "fr", - "es" - ], + voice_languages:{"Arabic": "ar", + "Brazilian Portuguese": "pt", + "Chinese": "zh-cn", + "Czech": "cs", + "Dutch": "nl", + "English": "en", + "French": "fr", + "German": "de", + "Italian": "it", + "Polish": "pl", + "Russian": "ru", + "Spanish": "es", + "Turkish": "tr", + "Japanese": "ja", + "Korean": "ko", + "Hungarian": "hu", + "Hindi": "hi"}, binding_changed:false, SVGGPU:SVGGPU, models_zoo:[], diff --git a/zoos/personalities_zoo b/zoos/personalities_zoo index 44ce30dd..a728904e 160000 --- a/zoos/personalities_zoo +++ b/zoos/personalities_zoo @@ -1 +1 @@ -Subproject commit 44ce30dded87bf9993541393402841ea182dca7c +Subproject commit a728904ed1307199b9940000f26cf700f6d97e8e