UPgraded code. New backend working. And more

app.py

@@ -22,6 +22,8 @@ import re
 import traceback
 import sys
 from tqdm import tqdm
+import subprocess
+import signal
 from pyaipersonality import AIPersonality
 from gpt4all_api.db import DiscussionsDB, Discussion
 from flask import (
@@ -211,6 +213,16 @@ class Gpt4AllWebUI(GPT4AllAPI):
             "/get_all_personalities", "get_all_personalities", self.get_all_personalities, methods=["GET"]
         )
         
+        self.add_endpoint(
+            "/reset", "reset", self.reset, methods=["GET"]
+        )
+        
+        
+    def reset(self):
+        os.kill(os.getpid(), signal.SIGINT)  # Send the interrupt signal to the current process
+        subprocess.Popen(['python', 'your_app.py'])  # Restart the app using subprocess
+
+        return 'App is resetting...'
 
     def save_settings(self):
         save_config(self.config, self.config_file_path)
@@ -481,6 +493,7 @@ class Gpt4AllWebUI(GPT4AllAPI):
     
     def stop_gen(self):
         self.cancel_gen = True
+        self.process.cancel_generation()
         print("Stop generation received")
         return jsonify({"status": "ok"})         
 

backends/c_transformers/.installed

@@ -0,0 +1 @@
+ok

backends/c_transformers/__init__.py

@@ -44,6 +44,10 @@ class GPTJ(GPTBackend):
             model_type='dolly-v2'
         elif 'starcoder' in self.config['model']:
             model_type='starcoder'
+        elif 'llama' in self.config['model']:
+            model_type='llama'
+        elif 'mpt' in self.config['model']:
+            model_type='mpt'
         else:
             print("The model you are using is not supported by this backend")
             return
@@ -97,27 +101,33 @@ class GPTJ(GPTBackend):
             verbose (bool, optional): If true, the code will spit many informations about the generation process. Defaults to False.
         """
         try:
-            self.model.reset()
-            tokens = self.model.tokenize(prompt.encode())
             output = ""
+            self.model.reset()
+            tokens = self.model.tokenize(prompt)
+            count = 0
             for tok in self.model.generate(
-                                            tokens, 
+                                            tokens,
-                                            seed=self.config['seed'],
-                                            n_threads=self.config['n_threads'],
-                                            n_predict=n_predict,
                                             top_k=self.config['top_k'],
                                             top_p=self.config['top_p'],
-                                            temp=self.config['temperature'],
+                                            temperature=self.config['temperature'],
-                                            repeat_penalty=self.config['repeat_penalty'],
+                                            repetition_penalty=self.config['repeat_penalty'],
-                                            repeat_last_n=self.config['repeat_last_n'],
+                                            seed=self.config['seed'],
-                                            n_batch=8,
+                                            batch_size=1,
+                                            threads = self.config['n_threads'],
                                             reset=True,
                                            ):
+                
+
+                if count >= n_predict or self.model.is_eos_token(tok):
+                    break
                 word = self.model.detokenize(tok)
                 if new_text_callback is not None:
                     if not new_text_callback(word):
-                        return output
+                        break
                 output += word
+                count += 1
+                
+                
         except Exception as ex:
             print(ex)
         return output            

backends/c_transformers/install.py

@@ -0,0 +1,36 @@
+import subprocess
+from pathlib import Path
+import requests
+from tqdm import tqdm
+
+class Install:
+    def __init__(self, api):
+        # Get the current directory
+        current_dir = Path(__file__).resolve().parent
+        install_file = current_dir / ".installed"
+
+        if not install_file.exists():
+            print("-------------- cTransformers backend -------------------------------")
+            print("This is the first time you are using this backend.")
+            print("Installing ...")
+            try:
+                print("Checking pytorch")
+                import torch
+                import torchvision
+                if torch.cuda.is_available():
+                    print("CUDA is supported.")
+                else:
+                    print("CUDA is not supported. Reinstalling PyTorch with CUDA support.")
+                    self.reinstall_pytorch_with_cuda()
+            except Exception as ex:
+                self.reinstall_pytorch_with_cuda()
+            # Step 2: Install dependencies using pip from requirements.txt
+            requirements_file = current_dir / "requirements.txt"
+            subprocess.run(["pip", "install", "--upgrade", "--no-cache-dir", "-r", str(requirements_file)])
+            with open(install_file,"w") as f:
+                f.write("ok")
+            print("Installed successfully")
+            
+    def reinstall_pytorch_with_cuda(self):
+        subprocess.run(["pip", "install", "torch", "torchvision", "torchaudio", "--no-cache-dir", "--index-url", "https://download.pytorch.org/whl/cu117"])
+        

backends/c_transformers/models.yaml

@@ -1,7 +1,24 @@
-- bestGPTJ: 'true'
+- MPT-7B: 'true'
-  description: Current best commercially licensable model based on GPT-J and trained
+  description: MPT-7B
-    by Nomic AI on the latest curated GPT4All dataset.
+  filename: mpt-7b.ggmlv3.q5_1.bin
-  filename: ggml-gpt4all-j-v1.3-groovy.bin
+  license: Apache-2.0
-  filesize: '3785248281'
+  link: https://huggingface.co/TheBloke/MPT-7B-GGML/resolve/main/
-  isDefault: 'true'
+  owner: TheBloke
-  md5sum: 81a09a0ddf89690372fc296ff7f625af
+  server: https://huggingface.co/TheBloke/MPT-7B-GGML/resolve/main/
+  sha256: c947c38405921a199c603fed2ed63486811777ba370bb51c40c3132e5cfe84e5
+- MPT-7B-Instruct: 'true'
+  description: MPT-7B-Instruct
+  filename: mpt-7b-instruct.ggmlv3.q5_1.bin
+  license: Apache-2.0
+  link: https://huggingface.co/TheBloke/MPT-7B-Instruct-GGML/resolve/main/
+  owner: TheBloke
+  server: https://huggingface.co/TheBloke/MPT-7B-Instruct-GGML/resolve/main/
+  sha256: a4d17a39ac277d48a3d55aa74b36a4e6e1b891b58937a838243fad549d26c686
+- MPT-7B-Storywriter: 'true'
+  description: MPT-7B-Storywriter
+  filename: mpt-7b-storywriter.ggmlv3.q5_1.bin
+  license: Apache-2.0
+  link: https://huggingface.co/TheBloke/MPT-7B-Storywriter-GGML/resolve/main/
+  owner: TheBloke
+  server: https://huggingface.co/TheBloke/MPT-7B-Storywriter-GGML/resolve/main/
+  sha256: 3b7dd7aa7508cc8cb4e262fe4b93214826f38d18d04059075e05837457f5402

backends/c_transformers/requirements.txt

@@ -1 +1,3 @@
-ctransformers
+ctransformers
+transformers
+accelerate

backends/llama_cpp_official/.installed

@@ -0,0 +1 @@
+ok

backends/llama_cpp_official/__init__.py

@@ -39,7 +39,7 @@ class LLAMACPP(GPTBackend):
             seed = random.randint(1, 2**31)
             
         if not "n_gpu_layers" in self.config:
-            self.config["n_gpu_layers"] = 40
+            self.config["n_gpu_layers"] = 20
         self.model = Llama(model_path=f"./models/llama_cpp_official/{self.config['model']}", n_ctx=self.config["ctx_size"], n_gpu_layers=self.config["n_gpu_layers"], seed=seed)
 
 

backends/llama_cpp_official/install.py

@@ -0,0 +1,36 @@
+import subprocess
+from pathlib import Path
+import requests
+from tqdm import tqdm
+
+class Install:
+    def __init__(self, api):
+        # Get the current directory
+        current_dir = Path(__file__).resolve().parent
+        install_file = current_dir / ".installed"
+
+        if not install_file.exists():
+            print("-------------- llama_cpp_official backend -------------------------------")
+            print("This is the first time you are using this backend.")
+            print("Installing ...")
+            try:
+                print("Checking pytorch")
+                import torch
+                import torchvision
+                if torch.cuda.is_available():
+                    print("CUDA is supported.")
+                else:
+                    print("CUDA is not supported. Reinstalling PyTorch with CUDA support.")
+                    self.reinstall_pytorch_with_cuda()
+            except Exception as ex:
+                self.reinstall_pytorch_with_cuda()
+            # Step 2: Install dependencies using pip from requirements.txt
+            requirements_file = current_dir / "requirements.txt"
+            subprocess.run(["pip", "install", "--no-cache-dir", "-r", str(requirements_file)])
+            with open(install_file,"w") as f:
+                f.write("ok")
+            print("Installed successfully")
+            
+    def reinstall_pytorch_with_cuda(self):
+        subprocess.run(["pip", "install", "torch", "torchvision", "torchaudio", "--no-cache-dir", "--index-url", "https://download.pytorch.org/whl/cu117"])
+        

configs/default.yaml

@@ -1,7 +1,8 @@
 version: 4
+user_name: user
 config: default
 ctx_size: 2048
-n_gpu_layers: 40
+n_gpu_layers: 20 #Depends on your GPU size
 db_path: databases/database.db
 debug: false
 n_threads: 8

gpt4all_api/api.py

@@ -109,12 +109,21 @@ class ModelProcess:
             'errors':[]
             }
         
-    def load_backend(self, backend_name:str):
+    def load_backend(self, backend_name:str, install=False):
         backend_path = Path("backends")/backend_name
-        # first find out if there is a requirements.txt file
+        if install:
-        requirements_file = backend_path/"requirements.txt"
+            # first find out if there is a requirements.txt file
-        if requirements_file.exists():
+            install_file_name="install.py"
-            parse_requirements_file(requirements_file)        
+            install_script_path = backend_path / install_file_name        
+            if install_script_path.exists():
+                module_name = install_file_name[:-3]  # Remove the ".py" extension
+                module_spec = importlib.util.spec_from_file_location(module_name, str(install_script_path))
+                module = importlib.util.module_from_spec(module_spec)
+                module_spec.loader.exec_module(module)
+                if hasattr(module, "Install"):
+                    self._install = module.Install(self)
+                else:
+                    self._install = None
 
         # define the full absolute path to the module
         absolute_path = backend_path.resolve()
@@ -164,7 +173,7 @@ class ModelProcess:
     def rebuild_backend(self, config):
         try:
             print(" ******************* Building Backend from main Process *************************")
-            backend = self.load_backend(config["backend"])
+            backend = self.load_backend(config["backend"], install=False)
             print("Backend loaded successfully")
         except Exception as ex:
             print("Couldn't build backend")
@@ -177,7 +186,7 @@ class ModelProcess:
     def _rebuild_model(self):
         try:
             print(" ******************* Building Backend from generation Process *************************")
-            self.backend = self.load_backend(self.config["backend"])
+            self.backend = self.load_backend(self.config["backend"], install=True)
             print("Backend loaded successfully")
             try:
                 model_file = Path("models")/self.config["backend"]/self.config["model"]
@@ -554,24 +563,7 @@ class GPT4AllAPI():
             print("Couldn't download file:", str(e))
 
 
-
+   
-            
-    def load_backend(self, backend_name):
-        backend_path = Path("backends")/backend_name
-
-        # define the full absolute path to the module
-        absolute_path = backend_path.resolve()
-
-        # infer the module name from the file path
-        module_name = backend_path.stem
-
-        # use importlib to load the module from the file path
-        loader = importlib.machinery.SourceFileLoader(module_name, str(absolute_path/"__init__.py"))
-        backend_module = loader.load_module()
-        backend_class = getattr(backend_module, backend_module.backend_name)
-        return backend_class
-
-    
     def condition_chatbot(self):
         if self.current_discussion is None:
             self.current_discussion = self.db.load_last_discussion()

requirements.txt

@@ -3,12 +3,6 @@ flask_socketio
 pytest
 pyyaml
 markdown
---find-links https://download.pytorch.org/whl/cu117
-torch==2.0.0
-torchvision
-torchaudio
-transformers
-accelerate
 gevent
 gevent-websocket
 pyaipersonality>=0.0.14

web/dist/index.html

@@ -6,7 +6,7 @@
     
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>GPT4All - WEBUI</title>
-    <script type="module" crossorigin src="/assets/index-2fd62f58.js"></script>
+    <script type="module" crossorigin src="/assets/index-4aa29d83.js"></script>
     <link rel="stylesheet" href="/assets/index-31b0e39d.css">
   </head>
   <body>

web/src/components/ModelEntry.vue

@@ -8,16 +8,17 @@
         {{ title }}
       </h3>
       <div class="flex flex-shrink-0">
-
+        <a :href="path" @click.stop 
-     
+        class="flex hover:text-secondary duration-75 active:scale-90"
-      <a :href="path" @click.stop 
+          title="Download this manually (faster) and put it in the models/<your backend> folder then refresh">
-      class="flex hover:text-secondary duration-75 active:scale-90"
-        title="Download this manually (faster) and put it in the models/<your backend> folder then refresh">
-          
           <i data-feather="link" class="w-5 p-1"></i>
           {{ title }}
         </a>
       </div>
+      <div class="flex flex-shrink-0">
+        <i data-feather="link" class="w-5 p-1">Licence: </i>
+          {{ licence }}
+      </div>
       <p class="opacity-80">{{ description }}</p>
     </div>
     <div class="flex-shrink-0">
@@ -56,6 +57,8 @@ export default {
     title: String,
     icon: String,
     path: String,
+    owner: String,
+    licence: String,
     description: String,
     isInstalled: Boolean,
     onInstall: Function,

web/src/views/SettingsView.vue

@@ -196,6 +196,13 @@
 
                     </select>
                 </div>
+                <div class="m-2">
+                        <button @click="applyConfiguration" class="bg-blue-500 text-white py-2 px-4 rounded">
+                            Apply Configuration
+                        </button>
+                        <div v-if="isLoading" class="loader"></div>
+                </div>                      
+
             </div>
         </div>
         <div
@@ -623,7 +630,6 @@ export default {
                     this.$refs.toast.showToast("Configuration changed failed.\nPlease select model first", 4, false)
                     nextTick(() => {
                     feather.replace()
-
                 })
                 return
             }
@@ -637,7 +643,7 @@ export default {
                     this.settingsChanged = false
                 } else {
                     console.log("applying configuration failed")
-                    this.$refs.toast.showToast("Configuration changed failed.", 4, false)
+                    this.$refs.toast.showToast("Configuration change failed.", 4, false)
 
                 }
                 nextTick(() => {
@@ -651,8 +657,9 @@ export default {
             axios.post('/save_settings', {})
                 .then((res) => {
                     if (res) {
-                        if (res.status)
+                        if (res.status){
                             this.$refs.messageBox.showMessage("Settings saved!")
+                        }
                         else
                             this.$refs.messageBox.showMessage("Error: Couldn't save settings!")
                         return res.data;