bugfix in changing models

Fixed path in install

upgraded backends

app.py

@@ -147,7 +147,7 @@ class Gpt4AllWebUI(GPT4AllAPI):
 
     def list_backends(self):
         backends_dir = Path('./backends')  # replace with the actual path to the models folder
-        backends = [f.stem for f in backends_dir.iterdir() if f.is_dir()]
+        backends = [f.stem for f in backends_dir.iterdir() if f.is_dir() and f.stem!="__pycache__"]
         return jsonify(backends)
 
 
@@ -405,7 +405,8 @@ class Gpt4AllWebUI(GPT4AllAPI):
             if len(models)>0:            
                 self.config['model'] = models[0]
                 self.load_backend(self.BACKENDS_LIST[self.config["backend"]])
-                self.create_chatbot()
+                # Build chatbot
+                self.chatbot_bindings = self.create_chatbot()
                 return jsonify({"status": "ok"})
             else:
                 return jsonify({"status": "no_models_found"})
@@ -418,7 +419,8 @@ class Gpt4AllWebUI(GPT4AllAPI):
         if self.config['model']!= model:
             print("New model selected")            
             self.config['model'] = model
-            self.create_chatbot()
+            # Build chatbot
+            self.chatbot_bindings = self.create_chatbot()
             return jsonify({"status": "ok"})
 
         return jsonify({"status": "error"})    

backends/gpt_j/__init__.py

@@ -21,7 +21,7 @@ backend_name = "GPT_J"
 
 
 class GPT_J(GPTBackend):
-    file_extension='*.bin'
+    file_extension='*'
     def __init__(self, config:dict) -> None:
         """Builds a GPT-J backend
 
@@ -60,17 +60,20 @@ class GPT_J(GPTBackend):
         """
         num_tokens = self.get_num_tokens(prompt)
         print(f"Prompt has {num_tokens} tokens")
-        self.model.generate(
-            prompt,
-            callback=new_text_callback,
-            n_predict=num_tokens + n_predict,
-            seed=self.config['seed'] if self.config['seed']>0 else -1,
-            temp=self.config['temp'],
-            top_k=self.config['top_k'],
-            top_p=self.config['top_p'],
-            # repeat_penalty=self.config['repeat_penalty'],
-            # repeat_last_n = self.config['repeat_last_n'],
-            n_threads=self.config['n_threads'],
-            #verbose=verbose
-        )
+        try:
+            self.model.generate(
+                prompt,
+                callback=new_text_callback,
+                n_predict=num_tokens + n_predict,
+                seed=self.config['seed'] if self.config['seed']>0 else -1,
+                temp=self.config['temp'],
+                top_k=self.config['top_k'],
+                top_p=self.config['top_p'],
+                # repeat_penalty=self.config['repeat_penalty'],
+                # repeat_last_n = self.config['repeat_last_n'],
+                n_threads=self.config['n_threads'],
+                #verbose=verbose
+            )
+        except Exception as ex:
+            print(ex)
         #new_text_callback()

backends/gpt_q/__init__.py

@@ -12,16 +12,19 @@ from typing import Callable
 from transformers import AutoTokenizer
 from transformers import AutoModelForCausalLM
 from pyGpt4All.backend import GPTBackend
+from transformers import AutoTokenizer, pipeline
+from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
+from auto_gptq.eval_tasks import LanguageModelingTask
 
 __author__ = "parisneo"
 __github__ = "https://github.com/nomic-ai/gpt4all-ui"
 __copyright__ = "Copyright 2023, "
 __license__ = "Apache 2.0"
 
-backend_name = "Transformers"
+backend_name = "GPT-Q"
 
 
-class Transformers(GPTBackend):
+class GPT_Q(GPTBackend):
     file_extension='*'
     def __init__(self, config:dict) -> None:
         """Builds a GPT-J backend
@@ -31,8 +34,18 @@ class Transformers(GPTBackend):
         """
         super().__init__(config, True)
         self.config = config
-        self.tokenizer = tokenizer = AutoTokenizer.from_pretrained(f"./models/transformers/{self.config['model']}/tokenizer.json", local_files_only=True)
-        self.model = AutoModelForCausalLM.from_pretrained(f"./models/transformers/{self.config['model']}/model.bin", local_files_only=True)
+        # path = Path("models/hugging_face")/self.config['model']
+        path = "TheBloke/vicuna-13B-1.1-GPTQ-4bit-128g"
+        AutoGPTQForCausalLM.from_pretrained(path, BaseQuantizeConfig())
+        self.model = AutoModelForCausalLM.from_pretrained(path, low_cpu_mem_usage=True)
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+
+        self.generator = pipeline(
+            "text-generation",
+            model=self.model,
+            tokenizer=self.tokenizer,
+            device=0,  # Use GPU if available
+        )
 
 
     def generate(self, 

backends/hugging_face/__init__.py

@@ -0,0 +1,82 @@
+######
+# Project       : GPT4ALL-UI
+# File          : backend.py
+# Author        : ParisNeo with the help of the community
+# Supported by Nomic-AI
+# Licence       : Apache 2.0
+# Description   : 
+# This is an interface class for GPT4All-ui backends.
+######
+from pathlib import Path
+from typing import Callable
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from pyGpt4All.backend import GPTBackend
+import torch
+import time
+__author__ = "parisneo"
+__github__ = "https://github.com/nomic-ai/gpt4all-ui"
+__copyright__ = "Copyright 2023, "
+__license__ = "Apache 2.0"
+
+backend_name = "HuggingFace"
+
+
+class HuggingFace(GPTBackend):
+    file_extension='*'
+    def __init__(self, config:dict) -> None:
+        """Builds a Hugging face backend
+
+        Args:
+            config (dict): The configuration file
+        """
+        super().__init__(config, True)
+        self.config = config
+        path = self.config['model']
+
+        self.model = AutoModelForCausalLM.from_pretrained(Path("models/hugging_face")/path, low_cpu_mem_usage=True)
+        self.tokenizer = AutoTokenizer.from_pretrained(Path("models/hugging_face")/path)
+
+        self.generator = pipeline(
+            "text-generation",
+            model=self.model,
+            tokenizer=self.tokenizer,
+            device=0,  # Use GPU if available
+        )
+
+
+    def generate_callback(self, text, new_text_callback):
+        def callback(outputs):
+            generated_text = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+            new_text_callback(generated_text)
+            print(text + generated_text, end="\r")
+            time.sleep(0.5)
+        return callback
+    
+    def generate(self, 
+                 prompt:str,                  
+                 n_predict: int = 128,
+                 new_text_callback: Callable[[str], None] = bool,
+                 verbose: bool = False,
+                 **gpt_params ):
+        """Generates text out of a prompt
+
+        Args:
+            prompt (str): The prompt to use for generation
+            n_predict (int, optional): Number of tokens to prodict. Defaults to 128.
+            new_text_callback (Callable[[str], None], optional): A callback function that is called everytime a new text element is generated. Defaults to None.
+            verbose (bool, optional): If true, the code will spit many informations about the generation process. Defaults to False.
+        """
+        callback = self.generate_callback(prompt, new_text_callback)
+        outputs = self.generator(
+            prompt, 
+            max_length=100, 
+            do_sample=True, 
+            num_beams=5, 
+            temperature=self.config['temp'], 
+            top_k=self.config['top_k'],
+            top_p=self.config['top_p'],
+            repetition_penalty=self.config['repeat_penalty'],
+            repeat_last_n = self.config['repeat_last_n'],
+            callback=callback
+        )
+        print(outputs)

pyGpt4All/api.py

@@ -46,7 +46,7 @@ class GPT4AllAPI():
         self.full_message_list = []
 
         # Select backend
-        self.BACKENDS_LIST = {f.stem:f for f in Path("backends").iterdir() if f.is_dir()}
+        self.BACKENDS_LIST = {f.stem:f for f in Path("backends").iterdir() if f.is_dir()  and f.stem!="__pycache__"}
 
         self.load_backend(self.BACKENDS_LIST[self.config["backend"]])
 

requirements.txt

@@ -9,4 +9,5 @@ gpt4all-j==0.2.1
 torch==2.0.0
 torchvision
 torchaudio
-transformers
+transformers
+accelerate

webui.sh

@@ -142,7 +142,7 @@ else
     case $option in
       Download)
         if [ -x "$(command -v wget)" ]; then
-          wget "$MODEL_URL" -P ./models/
+          wget "$MODEL_URL" -P ./models/llama_cpp/
         elif [ -x "$(command -v curl)" ]; then
           curl -o "$MODEL" "$MODEL_URL"
         else