added new embedding model

endpoints/libraries/lollms_client_js.js

@@ -737,6 +737,7 @@ async generateCode(prompt, images = [], {
   }
 
   const codes = this.extractCodeBlocks(response);
+  console.log(codes)
   if (codes.length > 0) {
       let code = '';
       if (!codes[0].is_complete) {
@@ -852,94 +853,84 @@ async generateCodes(prompt, images = [], {
 
   return completeCodes;
 }
+extractCodeBlocks(text, return_remaining_text = false) {
+  const codes = [];
+  let remainingText = text;
+  let currentIndex = 0;
   
-extractCodeBlocks(text) {
+  while (true) {
-  const codeBlocks = [];
+      // Find next code block start
-  let remaining = text;
+      const startPos = remainingText.indexOf('```');
-  let blocIndex = 0;
+      if (startPos === -1) {
-  let firstIndex = 0;
-  const indices = [];
-
-  // Find all code block delimiters
-  while (remaining.length > 0) {
-    const index = remaining.indexOf("```");
-    if (index === -1) {
-      if (blocIndex % 2 === 1) {
-        indices.push(remaining.length + firstIndex);
-      }
           break;
       }
-    indices.push(index + firstIndex);
+          
-    remaining = remaining.slice(index + 3);
+      // Check for file name before code block
-    firstIndex += index + 3;
+      let fileName = '';
-    blocIndex++;
+      const fileNameMatch = remainingText.slice(0, startPos).lastIndexOf('<file_name>');
+      if (fileNameMatch !== -1) {
+          const fileNameEnd = remainingText.slice(0, startPos).lastIndexOf('</file_name>');
+          if (fileNameEnd !== -1 && fileNameMatch < fileNameEnd) {
+              fileName = remainingText.slice(fileNameMatch + 11, fileNameEnd).trim();
+          }
       }
       
-  let isStart = true;
+      // Get code type if specified
-  for (let i = 0; i < indices.length; i++) {
+      let codeType = '';
-    if (isStart) {
+      const nextNewline = remainingText.indexOf('\n', startPos + 3);
-      const blockInfo = {
+      let contentStart;
-        index: i,
+      
-        file_name: "",
+      if (nextNewline !== -1) {
-        section: "",
+          const potentialType = remainingText.slice(startPos + 3, nextNewline).trim();
-        content: "",
+          if (potentialType) {
-        type: "",
+              codeType = potentialType;
+              contentStart = nextNewline + 1;
+          } else {
+              contentStart = startPos + 3;
+          }
+      } else {
+          contentStart = startPos + 3;
+      }
+          
+      // Find matching end tag
+      let pos = contentStart;
+      let is_complete = false;
+      
+      // Find the closing backticks
+      const endPos = remainingText.indexOf('```', contentStart);
+      
+      if (endPos !== -1) {
+          // Found matching end tag
+          const content = remainingText.slice(contentStart, endPos).trim();
+          is_complete = true;
+          codes.push({
+              index: currentIndex,
+              fileName: fileName,
+              content: content,
+              type: codeType,
+              is_complete: true
+          });
+          remainingText = remainingText.slice(endPos + 3);
+      } else {
+          // Handle incomplete code block
+          const content = remainingText.slice(contentStart).trim();
+          codes.push({
+              index: currentIndex,
+              fileName: fileName,
+              content: content,
+              type: codeType,
               is_complete: false
-      };
+          });
-
+          remainingText = '';
-      // Check for file name in preceding line
-      const precedingText = text.slice(0, indices[i]).trim().split('\n');
-      if (precedingText.length > 0) {
-        const lastLine = precedingText[precedingText.length - 1].trim();
-        if (lastLine.startsWith("<file_name>") && lastLine.endsWith("</file_name>")) {
-          blockInfo.file_name = lastLine.slice("<file_name>".length, -"</file_name>".length).trim();
-        } else if (lastLine.startsWith("## filename:")) {
-          blockInfo.file_name = lastLine.slice("## filename:".length).trim();
-        }
-        if (lastLine.startsWith("<section>") && lastLine.endsWith("</section>")) {
-          blockInfo.section = lastLine.slice("<section>".length, -"</section>".length).trim();
-        }
       }
           
-      const subText = text.slice(indices[i] + 3);
+      currentIndex++;
-      if (subText.length > 0) {
-        const findSpace = subText.indexOf(" ");
-        const findReturn = subText.indexOf("\n");
-        let nextIndex = Math.min(findSpace === -1 ? Infinity : findSpace, findReturn === -1 ? Infinity : findReturn);
-        if (subText.slice(0, nextIndex).includes('{')) {
-          nextIndex = 0;
-        }
-        const startPos = nextIndex;
-
-        if (text[indices[i] + 3] === "\n" || text[indices[i] + 3] === " " || text[indices[i] + 3] === "\t") {
-          blockInfo.type = 'language-specific';
-        } else {
-          blockInfo.type = subText.slice(0, nextIndex);
   }
   
-        if (i + 1 < indices.length) {
+  if (return_remaining_text) {
-          const nextPos = indices[i + 1] - indices[i];
+      return { codes, remainingText };
-          if (nextPos - 3 < subText.length && subText[nextPos - 3] === "`") {
-            blockInfo.content = subText.slice(startPos, nextPos - 3).trim();
-            blockInfo.is_complete = true;
-          } else {
-            blockInfo.content = subText.slice(startPos, nextPos).trim();
-            blockInfo.is_complete = false;
   }
-        } else {
+  return codes;
-          blockInfo.content = subText.slice(startPos).trim();
-          blockInfo.is_complete = false;
-        }
-
-        codeBlocks.push(blockInfo);
-      }
-      isStart = false;
-    } else {
-      isStart = true;
-    }
-  }
-
-  return codeBlocks;
 }
 
 
@@ -1463,96 +1454,6 @@ buildPrompt(promptParts, sacrificeId = -1, contextSize = null, minimumSpareConte
   }
 }
 
-extractCodeBlocks(text) {
-  const codeBlocks = [];
-  let remaining = text;
-  let blocIndex = 0;
-  let firstIndex = 0;
-  const indices = [];
-
-  // Find all code block delimiters
-  while (remaining.length > 0) {
-    const index = remaining.indexOf("```");
-    if (index === -1) {
-      if (blocIndex % 2 === 1) {
-        indices.push(remaining.length + firstIndex);
-      }
-      break;
-    }
-    indices.push(index + firstIndex);
-    remaining = remaining.slice(index + 3);
-    firstIndex += index + 3;
-    blocIndex++;
-  }
-
-  let isStart = true;
-  for (let i = 0; i < indices.length; i++) {
-    if (isStart) {
-      const blockInfo = {
-        index: i,
-        file_name: "",
-        section: "",
-        content: "",
-        type: "",
-        is_complete: false
-      };
-
-      // Check for file name in preceding line
-      const precedingText = text.slice(0, indices[i]).trim().split('\n');
-      if (precedingText.length > 0) {
-        const lastLine = precedingText[precedingText.length - 1].trim();
-        if (lastLine.startsWith("<file_name>") && lastLine.endsWith("</file_name>")) {
-          blockInfo.file_name = lastLine.slice("<file_name>".length, -"</file_name>".length).trim();
-        } else if (lastLine.startsWith("## filename:")) {
-          blockInfo.file_name = lastLine.slice("## filename:".length).trim();
-        }
-        if (lastLine.startsWith("<section>") && lastLine.endsWith("</section>")) {
-          blockInfo.section = lastLine.slice("<section>".length, -"</section>".length).trim();
-        }
-      }
-
-      const subText = text.slice(indices[i] + 3);
-      if (subText.length > 0) {
-        const findSpace = subText.indexOf(" ");
-        const findReturn = subText.indexOf("\n");
-        let nextIndex = Math.min(findSpace === -1 ? Infinity : findSpace, findReturn === -1 ? Infinity : findReturn);
-        if (subText.slice(0, nextIndex).includes('{')) {
-          nextIndex = 0;
-        }
-        const startPos = nextIndex;
-
-        if (text[indices[i] + 3] === "\n" || text[indices[i] + 3] === " " || text[indices[i] + 3] === "\t") {
-          blockInfo.type = 'language-specific';
-        } else {
-          blockInfo.type = subText.slice(0, nextIndex);
-        }
-
-        if (i + 1 < indices.length) {
-          const nextPos = indices[i + 1] - indices[i];
-          if (nextPos - 3 < subText.length && subText[nextPos - 3] === "`") {
-            blockInfo.content = subText.slice(startPos, nextPos - 3).trim();
-            blockInfo.is_complete = true;
-          } else {
-            blockInfo.content = subText.slice(startPos, nextPos).trim();
-            blockInfo.is_complete = false;
-          }
-        } else {
-          blockInfo.content = subText.slice(startPos).trim();
-          blockInfo.is_complete = false;
-        }
-
-        codeBlocks.push(blockInfo);
-      }
-      isStart = false;
-    } else {
-      isStart = true;
-    }
-  }
-
-  return codeBlocks;
-}
-
-
 /**
  * Updates the given code based on the provided query string.
  * The query string can contain two types of modifications:

web/dist/index.html

@@ -6,8 +6,8 @@
     <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-svg.js"></script>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>LoLLMS WebUI</title>
-    <script type="module" crossorigin src="/assets/index-DWgRuRtV.js"></script>
+    <script type="module" crossorigin src="/assets/index-Jw3bEM7C.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-BvcBrpnc.css">
+    <link rel="stylesheet" crossorigin href="/assets/index-gFczogus.css">
   </head>
   <body>
     <div id="app"></div>

web/src/views/SettingsView.vue

@@ -1148,6 +1148,7 @@
                                 >
                                     <!-- Semantic Vectorizer Models -->
                                     <option v-if="configFile.rag_vectorizer === 'semantic'" value="BAAI/bge-m3">BAAI/bge-m3</option>
+                                    <option v-if="configFile.rag_vectorizer === 'semantic'" value="nvidia/NV-Embed-v2">nvidia/NV-Embed-v2</option>
                                     <option v-if="configFile.rag_vectorizer === 'semantic'" value="sentence-transformers/all-MiniLM-L6-v2">sentence-transformers/all-MiniLM-L6-v2</option>
                                     <option v-if="configFile.rag_vectorizer === 'semantic'" value="sentence-transformers/all-MiniLM-L12-v2">sentence-transformers/all-MiniLM-L12-v2</option>
                                     <option v-if="configFile.rag_vectorizer === 'semantic'" value="sentence-transformers/all-distilroberta-v1">sentence-transformers/all-distilroberta-v1</option>