enhanced languages management

events/lollms_discussion_events.py

@@ -29,6 +29,7 @@ import yaml
 from lollms.databases.discussions_database import Discussion
 from lollms.security import forbid_remote_access
 from datetime import datetime
+import shutil
 
 router = APIRouter()
 lollmsElfServer = LOLLMSWebUI.get_instance()
@@ -76,18 +77,26 @@ def add_events(sio:socketio):
             if lollmsElfServer.config.current_language and  current_language!= default_language:
                 language_path = lollmsElfServer.lollms_paths.personal_configuration_path/"personalities"/lollmsElfServer.personality.name/f"languages_{current_language}.yaml"
                 if not language_path.exists():
-                    lollmsElfServer.ShowBlockingMessage(f"This is the first time this personality speaks {current_language}\nLollms is reconditionning the persona in that language.\nThis will be done just once. Next time, the personality will speak {current_language} out of the box")
-                    language_path.parent.mkdir(exist_ok=True, parents=True)
-                    # Translating
-                    conditionning = lollmsElfServer.tasks_library.translate_conditionning(lollmsElfServer.personality._personality_conditioning, lollmsElfServer.personality.language, current_language)
-                    welcome_message = lollmsElfServer.tasks_library.translate_message(lollmsElfServer.personality.welcome_message, lollmsElfServer.personality.language, current_language)
-                    with open(language_path,"w",encoding="utf-8", errors="ignore") as f:
-                        yaml.safe_dump({"conditionning":conditionning,"welcome_message":welcome_message}, f)
-                    lollmsElfServer.HideBlockingMessage()
+                    #checking if there is already a translation in the personality folder
+                    persona_language_path = lollmsElfServer.lollms_paths.personalities_zoo_path/lollmsElfServer.personality.category/lollmsElfServer.personality.name.replace(" ","_")/"languages"/f"{current_language}.yaml"
+                    if persona_language_path.exists():
+                        shutil.copy(persona_language_path, language_path)
+                        with open(language_path,"r",encoding="utf-8", errors="ignore") as f:
+                            language_pack = yaml.safe_load(f)
+                            conditionning = language_pack["personality_conditioning"]
+                    else:
+                        lollmsElfServer.ShowBlockingMessage(f"This is the first time this personality speaks {current_language}\nLollms is reconditionning the persona in that language.\nThis will be done just once. Next time, the personality will speak {current_language} out of the box")
+                        language_path.parent.mkdir(exist_ok=True, parents=True)
+                        # Translating
+                        conditionning = lollmsElfServer.tasks_library.translate_conditionning(lollmsElfServer.personality._personality_conditioning, lollmsElfServer.personality.language, current_language)
+                        welcome_message = lollmsElfServer.tasks_library.translate_message(lollmsElfServer.personality.welcome_message, lollmsElfServer.personality.language, current_language)
+                        with open(language_path,"w",encoding="utf-8", errors="ignore") as f:
+                            yaml.safe_dump({"personality_conditioning":conditionning,"welcome_message":welcome_message}, f)
+                        lollmsElfServer.HideBlockingMessage()
                 else:
                     with open(language_path,"r",encoding="utf-8", errors="ignore") as f:
                         language_pack = yaml.safe_load(f)
-                        welcome_message = language_pack["welcome_message"]
+                        welcome_message = language_pack.get("welcome_message", lollmsElfServer.personality.welcome_message)
             else:
                 welcome_message = lollmsElfServer.personality.welcome_message
 

lollms_core

@@ -1 +1 @@
-Subproject commit 7ad456806e139700eb8b409a01fa951a99dd2638
+Subproject commit b2c9f73a3d51b4edc21040c2e478a67e003c9619

tools/LiteSQLViewer/LiteSQLViewer.py

@@ -0,0 +1,236 @@
+import sys
+from PyQt5.QtWidgets import (QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
+                             QPushButton, QLineEdit, QTableView, QHeaderView, QMessageBox,
+                             QFileDialog, QListWidget, QSplitter, QDialog, QLabel, QFormLayout)
+from PyQt5.QtSql import QSqlDatabase, QSqlTableModel
+from PyQt5.QtCore import Qt, QSortFilterProxyModel
+
+class AddRecordDialog(QDialog):
+    def __init__(self, columns, parent=None):
+        super().__init__(parent)
+        self.setWindowTitle("Add Record")
+        self.setGeometry(100, 100, 300, 200)
+        self.layout = QFormLayout(self)
+        self.line_edits = {}
+
+        for column in columns:
+            if column.lower() != "id":  # Exclude the ID field
+                line_edit = QLineEdit(self)
+                self.layout.addRow(QLabel(column), line_edit)
+                self.line_edits[column] = line_edit
+
+        self.submit_button = QPushButton("Submit", self)
+        self.submit_button.clicked.connect(self.accept)
+        self.layout.addRow(self.submit_button)
+
+    def get_values(self):
+        return {column: line_edit.text() for column, line_edit in self.line_edits.items()}
+
+class LiteSQLViewer(QMainWindow):
+    def __init__(self):
+        super().__init__()
+        self.setWindowTitle("LiteSQL Viewer")
+        self.setGeometry(100, 100, 800, 600)
+        self.setStyleSheet("""
+            QMainWindow {
+                background-color: #f0f0f0;
+            }
+            QPushButton {
+                background-color: #4CAF50;
+                color: white;
+                border: none;
+                padding: 8px 16px;
+                text-align: center;
+                text-decoration: none;
+                font-size: 14px;
+                margin: 4px 2px;
+                border-radius: 4px;
+            }
+            QPushButton:hover {
+                background-color: #45a049;
+            }
+            QLineEdit {
+                padding: 8px;
+                margin: 4px 2px;
+                border: 1px solid #ccc;
+                border-radius: 4px;
+            }
+            QTableView {
+                border: 1px solid #ddd;
+                gridline-color: #ddd;
+            }
+            QHeaderView::section {
+                background-color: #f2f2f2;
+                padding: 4px;
+                border: 1px solid #ddd;
+                font-weight: bold;
+            }
+            QListWidget {
+                width: 200px;
+                background-color: #ffffff;
+                border: 1px solid #ddd;
+            }
+        """)
+
+        self.db = QSqlDatabase.addDatabase("QSQLITE")
+        self.model = None
+        self.setup_ui()
+
+    def setup_ui(self):
+        central_widget = QWidget()
+        self.setCentralWidget(central_widget)
+        layout = QHBoxLayout()
+
+        # Splitter for left sidebar and main view
+        self.splitter = QSplitter(Qt.Horizontal)
+        layout.addWidget(self.splitter)
+
+        # Left sidebar for tables
+        self.table_list = QListWidget()
+        self.table_list.itemClicked.connect(self.load_table_from_list)
+        self.splitter.addWidget(self.table_list)
+
+        # Main layout
+        main_widget = QWidget()
+        main_layout = QVBoxLayout(main_widget)
+
+        # Buttons
+        button_layout = QHBoxLayout()
+        self.open_button = QPushButton("Open Database", self)
+        self.open_button.clicked.connect(self.open_database)
+        self.commit_button = QPushButton("Commit Changes", self)
+        self.commit_button.clicked.connect(self.commit_changes)
+        self.add_button = QPushButton("Add Record", self)
+        self.add_button.clicked.connect(self.add_record)
+        self.edit_button = QPushButton("Edit Record", self)
+        self.edit_button.clicked.connect(self.edit_record)
+        self.delete_button = QPushButton("Delete Record", self)
+        self.delete_button.clicked.connect(self.delete_record)
+        self.scroll_button = QPushButton("Scroll to Bottom", self)
+        self.scroll_button.clicked.connect(self.scroll_to_bottom)  # Connect to scroll method
+        button_layout.addWidget(self.open_button)
+        button_layout.addWidget(self.commit_button)
+        button_layout.addWidget(self.add_button)
+        button_layout.addWidget(self.edit_button)
+        button_layout.addWidget(self.delete_button)
+        button_layout.addWidget(self.scroll_button)  # Add scroll button to layout
+        main_layout.addLayout(button_layout)
+
+        # Search bar
+        self.search_bar = QLineEdit(self)
+        self.search_bar.setPlaceholderText("Search...")
+        self.search_bar.textChanged.connect(self.filter_table)
+        main_layout.addWidget(self.search_bar)
+
+        # Table view
+        self.table_view = QTableView()
+        self.table_view.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
+        self.table_view.verticalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
+        main_layout.addWidget(self.table_view)
+
+        self.splitter.addWidget(main_widget)
+        self.splitter.setSizes([150, 650])  # Set default sizes for the splitter
+
+        central_widget.setLayout(layout)
+
+    def open_database(self):
+        file_name, _ = QFileDialog.getOpenFileName(self, "Open Database", "", "SQLite Database Files (*.db *.sqlite);;All Files (*)")
+        if file_name:
+            self.db.setDatabaseName(file_name)
+            if not self.db.open():
+                QMessageBox.critical(self, "Error", "Could not open database")
+                return
+            self.load_tables()
+
+    def load_tables(self):
+        self.table_list.clear()
+        tables = self.db.tables()
+        if not tables:
+            QMessageBox.warning(self, "Warning", "No tables found in the database")
+            return
+        self.table_list.addItems(tables)
+
+    def load_table_from_list(self, item):
+        table_name = item.text()
+        self.model = QSqlTableModel(self, self.db)
+        self.model.setTable(table_name)
+        self.model.setEditStrategy(QSqlTableModel.OnFieldChange)
+        self.model.select()
+
+        self.proxy_model = QSortFilterProxyModel(self)
+        self.proxy_model.setSourceModel(self.model)
+        self.proxy_model.setFilterCaseSensitivity(Qt.CaseInsensitive)
+        self.proxy_model.setFilterKeyColumn(-1)  # Search all columns
+
+        self.table_view.setModel(self.proxy_model)
+
+    def filter_table(self, text):
+        self.proxy_model.setFilterFixedString(text)
+
+    def add_record(self):
+        if not self.model:
+            QMessageBox.warning(self, "Warning", "No table selected")
+            return
+        
+        # Get the column names, excluding the ID field
+        columns = [self.model.record().fieldName(i) for i in range(self.model.record().count()) if self.model.record().fieldName(i).lower() != "id"]
+        
+        # Create and show the dialog
+        dialog = AddRecordDialog(columns, self)
+        if dialog.exec_() == QDialog.Accepted:
+            values = dialog.get_values()
+            self.insert_record(values)
+
+    def insert_record(self, values):
+        if not self.model:
+            return
+        
+        row = self.model.rowCount()
+        self.model.insertRow(row)
+        
+        for column, value in values.items():
+            self.model.setData(self.model.index(row, self.model.fieldIndex(column)), value)
+
+    def edit_record(self):
+        if not self.model:
+            QMessageBox.warning(self, "Warning", "No table selected")
+            return
+        indexes = self.table_view.selectionModel().selectedRows()
+        if len(indexes) != 1:
+            QMessageBox.warning(self, "Warning", "Please select a single row to edit")
+            return
+        source_index = self.proxy_model.mapToSource(indexes[0])
+        self.table_view.edit(indexes[0])
+
+    def delete_record(self):
+        if not self.model:
+            QMessageBox.warning(self, "Warning", "No table selected")
+            return
+        indexes = self.table_view.selectionModel().selectedRows()
+        if not indexes:
+            QMessageBox.warning(self, "Warning", "Please select row(s) to delete")
+            return
+        confirm = QMessageBox.question(self, "Confirm Deletion", f"Are you sure you want to delete {len(indexes)} row(s)?",
+                                       QMessageBox.Yes | QMessageBox.No)
+        if confirm == QMessageBox.Yes:
+            for index in sorted(indexes, reverse=True):
+                source_index = self.proxy_model.mapToSource(index)
+                self.model.removeRow(source_index.row())
+            self.model.select()
+
+    def commit_changes(self):
+        if self.model:
+            if self.model.submitAll():
+                QMessageBox.information(self, "Success", "Changes committed successfully.")
+            else:
+                QMessageBox.critical(self, "Error", "Failed to commit changes: " + self.model.lastError().text())
+
+    def scroll_to_bottom(self):
+        if self.model and self.model.rowCount() > 0:
+            self.table_view.scrollToBottom()  # Scroll to the bottom of the table view
+
+if __name__ == '__main__':
+    app = QApplication(sys.argv)
+    window = LiteSQLViewer()
+    window.show()
+    sys.exit(app.exec_())

utilities/execution_engines/lilypond_execution_engine.py

@@ -49,7 +49,7 @@ def execute_lilypond(code, client:Client, message_id):
         
         # Create LilyPond file
         ly_file = root_folder/f"score_{message_id}.ly"
-        ly_file.write_text(code)
+        ly_file.write_text(code,encoding="utf8")
 
         # Get the PDF and MIDI outputs
         pdf_file = ly_file.with_suffix('.pdf')

web/dist/index.html

@@ -6,8 +6,8 @@
     <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-svg.js"></script>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>LoLLMS WebUI</title>
-    <script type="module" crossorigin src="/assets/index-CSHYN0YB.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-CZl152Xi.css">
+    <script type="module" crossorigin src="/assets/index-Fso6Zpfn.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-DBHwToRQ.css">
   </head>
   <body>
     <div id="app"></div>

web/package-lock.json

@@ -25,6 +25,7 @@
         "markdown-it-math": "^4.1.1",
         "markdown-it-mathjax": "^2.0.0",
         "markdown-it-multimd-table": "^4.2.3",
+        "markdown-it-texmath": "^1.0.0",
         "marked": "^14.1.2",
         "mathjax": "^3.2.2",
         "mermaid": "^11.2.1",
@@ -3558,6 +3559,12 @@
       "resolved": "https://registry.npmjs.org/markdown-it-multimd-table/-/markdown-it-multimd-table-4.2.3.tgz",
       "integrity": "sha512-KepCr2OMJqm7IT6sOIbuqHGe+NERhgy66XMrc5lo6dHW7oaPzMDtYwR1EGwK16/blb6mCSg4jqityOe0o/H7HA=="
     },
+    "node_modules/markdown-it-texmath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/markdown-it-texmath/-/markdown-it-texmath-1.0.0.tgz",
+      "integrity": "sha512-4hhkiX8/gus+6e53PLCUmUrsa6ZWGgJW2XCW6O0ASvZUiezIK900ZicinTDtG3kAO2kon7oUA/ReWmpW2FByxg==",
+      "license": "MIT"
+    },
     "node_modules/marked": {
       "version": "14.1.2",
       "resolved": "https://registry.npmjs.org/marked/-/marked-14.1.2.tgz",

web/package.json

@@ -28,6 +28,7 @@
     "markdown-it-math": "^4.1.1",
     "markdown-it-mathjax": "^2.0.0",
     "markdown-it-multimd-table": "^4.2.3",
+    "markdown-it-texmath": "^1.0.0",
     "marked": "^14.1.2",
     "mathjax": "^3.2.2",
     "mermaid": "^11.2.1",

web/src/components/MarkdownRenderer.vue

@@ -33,6 +33,10 @@ import CodeBlock from './CodeBlock.vue';
 import hljs from 'highlight.js';
 import mathjax from 'markdown-it-mathjax';
 
+
+import texmath from 'markdown-it-texmath';
+import katex from 'katex';
+
 function escapeHtml(unsafe) {
   return unsafe
     .replace(/&/g, "&")
@@ -80,7 +84,7 @@ export default {
         return hljs.highlight(validLanguage, code).value;
       },
       renderInline: true,
-      breaks: true,
+      breaks: false, // Prevent newlines from being converted to <br> tags
     })
       .use(emoji)
       .use(anchor)
@@ -100,10 +104,68 @@ export default {
         multilineCellEndMarker: '<|',
         multilineCellPadding: ' ',
         multilineCellJoiner: '\n',
-      }).use(mathjax({
-        inlineMath: [['$', '$'], ['\\(', '\\)']],
-        blockMath: [['$$', '$$'], ['\\[', '\\]']]
-      }))
+      });
+
+    // Add a custom rule to escape backslashes before LaTeX delimiters
+    md.core.ruler.before('normalize', 'escape_latex_delimiters', state => {
+      state.src = state.src.replace(/(?<!\\)(\\[\(\)\[\]])/g, '\\$1');
+    });
+
+    // Modify the inline LaTeX rule to ensure it only triggers once
+    md.inline.ruler.before('escape', 'inline_latex', function(state, silent) {
+      const start = state.pos;
+      const max = state.posMax;
+
+      if (state.src.slice(start, start + 2) !== '\\(') return false;
+
+      let end = start + 2;
+      while (end < max) {
+        if (state.src.slice(end, end + 2) === '\\)') {
+          end += 2;
+          break;
+        }
+        end++;
+      }
+
+      if (end === max) return false;
+
+      if (!silent) {
+        const token = state.push('latex_inline', 'latex', 0);
+        token.content = state.src.slice(start + 2, end - 2);
+        token.markup = '\\(\\)';
+      }
+
+      state.pos = end;
+      return true;
+    });
+
+    // Ensure the LaTeX is rendered only once
+    md.renderer.rules.latex_inline = function(tokens, idx) {
+      return '<span class="inline-latex">' + katex.renderToString(tokens[idx].content, {displayMode: true}) + '</span>';
+    };
+
+    // Enhance list rendering
+    md.renderer.rules.list_item_open = function (tokens, idx, options, env, self) {
+      const token = tokens[idx];
+      if (token.markup === '1.') {
+        // This is an ordered list item
+        const start = token.attrGet('start');
+        if (start) {
+          return `<li value="${start}">`;
+        }
+      }
+      return self.renderToken(tokens, idx, options);
+    };
+
+    md.use(texmath, {
+      engine: katex,
+      delimiters: [
+        {left: '$$', right: '$$', display: true},
+        {left: '$', right: '$', display: false},
+        {left: '\\[', right: '\\]', display: true}
+      ],
+      katexOptions: { macros: { "\\RR": "\\mathbb{R}" } }
+    });
 
     const markdownItems = ref([]);
     const updateMarkdown = () => {
@@ -164,4 +226,16 @@ export default {
 
 <style>
 /* Your existing styles */
+.katex-display {
+  display: inline-block;
+  margin: 0;
+}
+
+.katex {
+  display: inline-block;
+  white-space: nowrap;
+}
+.inline-latex {
+  display: inline !important;
+}
 </style>

web/src/components/TopBar.vue

@@ -354,6 +354,13 @@ export default {
 
   },
   methods: {
+    
+    addCustomLanguage() {
+        if (this.customLanguage.trim() !== '') {
+        this.selectLanguage(this.customLanguage);
+        this.customLanguage = ''; // Reset the input field after adding
+        }
+    },
     handleClickOutside(e) {
       const dropdown = this.$el
       if (!dropdown.contains(e.target)) {
@@ -460,6 +467,7 @@ export default {
         window.dispatchEvent(new Event('themeChanged'));
     },
     async selectLanguage(language) {
+        await this.$store.dispatch('changeLanguage', language);
         await this.$store.dispatch('changeLanguage', language);
         this.toggleLanguageMenu(); // Fermer le menu après le changement de langue
         this.language = language

web/src/main.js

@@ -357,9 +357,11 @@ export const store = createStore({
           const language = response.data;
           console.log("language", language)
           commit('setLanguage', language);
+          await this.dispatch('refreshMountedPersonalities');
 
-          console.log('Language changed successfully:', response.data.message);
+          console.log('Language changed successfully:', language);
       },
+      
       async deleteLanguage({ commit }, new_language) {
         console.log("Deleting ", new_language)
           let response = await axios.post('/del_personality_language', {

web/src/views/DiscussionsView.vue

@@ -1894,12 +1894,6 @@ export default {
                 }
             }
         },
-        addCustomLanguage() {
-            if (this.customLanguage.trim() !== '') {
-            this.selectLanguage(this.customLanguage);
-            this.customLanguage = ''; // Reset the input field after adding
-            }
-        },
         restartProgram(event) {
             event.preventDefault();
             this.$store.state.api_post_req('restart_program', this.$store.state.client_id)

zoos/bindings_zoo

@@ -1 +1 @@
-Subproject commit a7406c349335dfa982c1f3c751aca16852c06c77
+Subproject commit ea93d5d7deb174ec93975e5bb4cc09468ddbb8d9

zoos/models_zoo

@@ -1 +1 @@
-Subproject commit 6ad70aa8d7c9dffead7eed65bf6ccb4c655b1a25
+Subproject commit 5ee61221a32fb5dc4466838fe35cd83ea9a2403f

zoos/personalities_zoo

@@ -1 +1 @@
-Subproject commit 74f7af206a4f5f5c87973c8d2e3047938f004e4f
+Subproject commit 39ac19c3371fdb3f40a3145082d0e5ae92566006