From c1860cccfc14cf542631e7e0445cd5f0bb7d898a Mon Sep 17 00:00:00 2001 From: ParisNeo Date: Fri, 5 May 2023 01:05:02 +0200 Subject: [PATCH] Upgraded database, Added database documentation --- docs/dev/db_infos.md | 94 ++++++++++++++++++ docs/dev/scheme.png | Bin 0 -> 5129 bytes .../AdvancedInstallInstructions.md | 0 docs/{ => usage}/Build_extensions.md | 0 docs/{ => usage}/Linux_Osx_Install.md | 0 docs/{ => usage}/Linux_Osx_Usage.md | 0 .../english/general/gpt4all/config.yaml | 34 +++++-- pyGpt4All/api.py | 2 +- pyGpt4All/db.py | 30 ++++-- requirements.txt | 2 +- 10 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 docs/dev/db_infos.md create mode 100644 docs/dev/scheme.png rename docs/{ => usage}/AdvancedInstallInstructions.md (100%) rename docs/{ => usage}/Build_extensions.md (100%) rename docs/{ => usage}/Linux_Osx_Install.md (100%) rename docs/{ => usage}/Linux_Osx_Usage.md (100%) diff --git a/docs/dev/db_infos.md b/docs/dev/db_infos.md new file mode 100644 index 00000000..01feb0e1 --- /dev/null +++ b/docs/dev/db_infos.md @@ -0,0 +1,94 @@ +# Database Documentation + +## Introduction +This project implements a discussion forum and utilizes a SQLite database to store discussions and messages. + +## Database Schema +The following database schema has been implemented: + +- `discussion`: table to store discussion information including id and title. +- `message`: table to store message information including id, sender, content, type, rank, parent, and discussion_id. The type column is an integer representing the type of the message. The rank column is an integer representing the rank of the message. The parent column is an integer representing the id of the parent message. The discussion_id column is a foreign key referencing the id column in the discussion table. + +## Database Upgrades +The database schema is currently at version 2. The following upgrades have been made to the schema: + +In version 1, three columns have been added to the message table: type, rank, and parent. +In version 2, the parent column has been added to the message table (if it doesn't already exist). +Encoding +The encoding of the database is checked before creating/updating the schema. If the current encoding is not UTF-8, it is changed to UTF-8. + +## Implementation Details +The create database schema script is responsible for creating/updating the database schema. It first checks the encoding of the database and changes it to UTF-8 if necessary. Then, it checks if the required tables (discussion, message, and schema_version) exist. If any of these tables do not exist, they are created. The schema version is retrieved from the schema_version table. If the table is empty, version 0 is assumed. Otherwise, the version from the table is used. If the version is less than the current version, the schema is upgraded to the current version by adding the necessary columns to the message table. Finally, the schema version is updated or inserted into the schema_version table. + +## Documentation: + +The DiscussionDB class provides methods to manipulate the database and retrieve information about discussions and messages. Here are the methods available in this class: +``` +select(query, params=None, fetch_all=True) +``` + +This method executes an SQL select query on the database with optional parameters. It returns the cursor object for further processing. If fetch_all is True, it returns all the rows returned by the query, else it returns only the first row. + +``` +delete(query, params=None) +``` + +This method executes an SQL delete query on the database with optional parameters. It returns the cursor object for further processing. + +``` +insert(query, params=None) +``` + +This method executes an SQL insert query on the database with optional parameters. It returns the ID of the newly inserted row. + +``` +update(query, params=None) +``` + +This method executes an SQL update query on the database with optional parameters. + +``` +load_last_discussion() +``` + +This method retrieves the last discussion in the database or creates a new one if there are no discussions. It returns a Discussion instance for the retrieved or created discussion. + +``` +create_discussion(title="untitled") +``` + +This method creates a new discussion with the specified title. It returns a Discussion instance for the newly created discussion. + +``` +build_discussion(discussion_id=0) +``` + +This method retrieves the discussion with the specified ID or creates a new one if the ID is not found. It returns a Discussion instance for the retrieved or created discussion. + +``` +get_discussions() +``` + +This method retrieves all discussions in the database. It returns a list of dictionaries, where each dictionary represents a discussion and contains the discussion's ID and title. + +``` +does_last_discussion_have_messages() +``` + +This method checks if the last discussion in the database has any messages. It returns True if the last discussion has at least one message, else False. + +``` +remove_discussions() +``` + +This method removes all discussions and messages from the database. + +``` +export_to_json() +``` + +This method exports all discussions and messages in the database to a JSON file. It returns a list of dictionaries, where each dictionary represents a discussion and contains the discussion's ID, title, and messages. Each message is represented by a dictionary with the sender, content, type, rank, and parent fields. + + +## Database diagram +[](scheme.png) \ No newline at end of file diff --git a/docs/dev/scheme.png b/docs/dev/scheme.png new file mode 100644 index 0000000000000000000000000000000000000000..d0f76826ab80eb3b129bf2bb34543bf8fb44048f GIT binary patch literal 5129 zcmbVQi9b~D_n*?9<vNgXuKEHq9`*~h-?|q(g&U2pUIq!4NoO7Ss0cS;aN$rBcU?NuM zEbL*hO?2q8-XR20{H!eZLyt{p`?IHE1ymUhG}vB{+v16 zMS*>uLw7C1vLWIe9!cQ~?wM#LrTa5weSN)qbeTUoR^7FX|2)THv3S3iC%DsdKj-<} zI(uZMec2eFPkV~Ygr(Cu7`+WGv@ea+W-5a_HN)q&lZqOcd_MCFiPSblr7>aYq~S3R zrM;_#%Ha1ldqocdKyS~1@skJk|sG+&J`CCf|YhbvM(wSe@&gJue3=glZug}iTQaT_5 zw8xm7iJ|dnxHH-&XS8wcd~T69)Gk-zaf4n8uEjksB>4)Fqmk);M&ya`@t0k z+w+M^K@Et*#2(^8_kQF=-R;k6Pj4dD)N!?D)Le0 zvHVNDo~Lhk*?5SPdU;7<#CDx|)+CimooaL6`#Z!emfh!7#vZ4%QfJl!b+$n3{^#~N zjS@5Q(1N_>>FHh4dkjUQ_v_1}q@~S~e)1Gj8$2k{;&0tpCmor-c zbA^cL9r(yI|NDfzp8L$RL5h`TWTl?U*UnQ`@s+De@FgWrYy-Cq17e&wnyWf&G>u%X9r9MA_;w$#4Ja>{aejaAGwdYcAR@Hc7p73?3+x^)4!= zpMQ8~ZIE2;IS1S8G6E|&(uaxcY+3fZm>f_4r)qShHz*`)bMRl*H1R5P=yp!5l9)`U zc*UyodUW_Bfra$gK+sGUCyt%bk$w1}Fel5Nci_2L&6DR5|IaNi95ug-{bjY6Ky~{d zG={)$>f^e@ai!{M!F(q-p%N5U6(Q=d8xXAIV!jW@?P#fLS*@IJEk{?RS=9#{6}#aopkP9Gztq-5iQa~)eY z0|sKs*_TBmXMN}*!!pR2$)9N>JX#B(a*YY=}p;5klTIOr^0hH=;MW} zNxIP)PAr`DZqM79|2{Emmb9-+`Sr7khI6_@iYm>mw{L`)h(FQInk{WShf2BN0Olwf zEnQ&)%c4j8zmBVnOimPxd>=PVE})`ZzvPjqZiBk1WdKD;u$TKB3lx&@;rV`jL$Q!Q zWv!!|z`7GgA;04^=SE)Mz&cdRJZc1l!rc86bm_9ZsNz`*l3EG%@dIgfM)Gx(1pP8#Azib)s!NW zzIZ0k&kDVY)1cL1f`6AEH3EP|c|#FfH~%V?k{7SE>|=+OqvHY7`Cp1DvSvSjP_R4k zk?)QB@9EN&^*ZYstB)B2?Jg|o$3@K*tt2g@=*@VETVBOz8ShP?BBFqfRnI3z2m%T6 zk@W@l^HAt0?g2)Lg31bR6kRRmecx&>t|_E}SKJc9n~IE65J>&an((=W=`CV%npNEp%m_c=^F zc|=FTRY<loXp~U#CP$y!APZpcYjZ9iwZ{W3j2B*y zPSEeR9&2VjyzQcOWu;C`+9~M`&VBgoh(qFfaA?4VZoSMAT9{LBway*QYirFD%OfMl zqp)EsDzmeki`Y2)pPsy=8*y?sZ^0MO-Ml}VcmttdZydnIKH#A|0=!2T-Fy;t^6uG0 zb8urHTS)W(BtYd%*t?>>|M``b54A^s#fa3tx@8?-0T-TgH%*ZeX7 zRQpqW8HWB>QwIxi*aIX{CGDqPk0SBEoIGREz8b$=LjvC!UAPcXaW7e<7cLsVXS|7n z=$!l|LvU1$Mdbc%(AUKA%V7PIesdkT^a*>*uBhB;RoSNRVJc?RyNTT#{ z__SKdsj<858rO1wM0~n!!F49!5relkGqDC9VEgk28K$%~RovCke&y1&lwF>IEB)Nd z)nnaM?PS!!Ba84031;d5+?zzFYO5-RtxI>+aUlgR=~-y#Yv<`sFYVwYDA z`LCHQ;#)r>BSlvs-N};YGPX=f2@>74I>ZF$<_I9nT|s7nDu}s!;Y{dqADO=Up)30q zK!_8KzdK_%O;Jy6tac>@Uk#92T$(--?MIj$)zXNL_^PaaHN+>zzsHxpm%lv z7aV*UD?S*%+nv)W$id^L6BL+Rb1DL>QAjC$5c8fhz;_(f-F?5Wv&i_}lU&{$6LnV~ ziraB;y2mC$%pGkMQq~BF;-)t;a{D4q3H&`M7cmeXKmXTGz(#?|=@1!DY-|##7llIp z2sOYMQ(w;6Q{W^sNHmebJBDcNtKc!9_oe(vTD$|Vu5kX7Bw$njjW7wao|^xu5xi+L zcfpw`2cC;2J~1+to;w*eGGm?tXLA^GNlq#Bl{_{7v!G!y3ThD!e zH1#{rl#E!{OQEyi%MqzF*~&eSynOC~s%;HjW_tXOHDvA9Z46QTLaTbEaezsdU#q+p zW}3zyz&-Hk=^^xaSYHIOuYaq0Fp|9k zD**oeBu8I8gR(7-pyg7?q+!=V_hmqgxY%Fr76fWI$+FCFoIdkICF6y}qmr^VkyqA1@uffTPo-TPnjYON!C&V{pzU(?spyVl>3YfKaN6ux>u zq6e7HT)&(UXuaw?`7o}>XJ{C6SHAF|M39t_p`t(C3gv0uPsUW?$6~?<8Ie`qoZm)m z@Qj~jtli+dp>f^WhuZ+}roO}&kdB`}Xh=HZd)2rGm>$>pcizJ^Q8v+IeuqY#het3x zWT4w_XhTUboD@1G0_u-=+cXXk^0U4WL?GY$8RWSyN-JK_RUX8st=`&;?~R4>kIjI( zh&WKH+T#@BR~=Mo-zKy2g+o>>$&v{4Y(F8~aIRaYVTky$wf$mg4qX)Gc_c4OqD{e1 zc2zU)UNdWZX^I?`@@BivX{Y|V5{c;c15tFJ;?1l?I)1eZ8+U{U@>H1{hud-Ke&`H7~^5T z$0~w^`27jzg>UoQO7$9E*32mBY$VT8bjusO@WKtz2Fx5r?x>X@kKQJHD1Y^^l^}uE zyLmwI($5E~WP`3|jCnxJ4;z8NG2x_W{B|pW4#LiQ8htsLwb74Tu)HxYlq=BZ`L#pk zy4Z&ea`FP_)#O=dN#V_*WBQL<$#NWv!pVb>+!OngXCn|w!rKmoiD&P3)ZtMFBqfa0 z4ApIW+(8?cPF4K%Q|^6H3zh?~rhLupFSoJQh}azy*h%HS5nYGC*fjZ)Y4eIKWX2EL zsa<(48N&A&wmr@sp$Co^=i?!sB^E!qbU$^!W41O=hruJK_g8ja3sLRZLwY0q6mhmw zGK4+zG#*OOl@08Cz#kank!o=i*V33Xss9W2#q%p+MMPX3RFRf~87?ZZ)zUQJP#Qgz z;Cp$kG4ZaK@H>J^ z2Lb0uf1MA#O4_io65B4%U^*jvV_~D5uz_IecP9aCZFQ*ef7W9+rjNP&woDO!$=HA; zC5nR`j6P3TwA!elBD{(`%=m9x`^M%)GZ>{s?BLdtAs0o)X$Uk)CR6x&fdaQ=xzW%9oxl4N8G4CUh$ziRTjZ5d~EVZQVKNY^IQp9rU7n=avIIv zm`8yvKwOW;tD%QyF6P_5vh8kAQfYOKt~tLD6L&coJD7=;{jk@tMkNuwh3TcNJ z%G!MqMPt&rw--&o+3V#MRRw7Uy2}^)cF)F7z<{`|>HwRyx|F%n?|FS!-X4(ed0y#< z0|~KCv@?u;NfEK@N@ZEvxagthA71U-mYn*}``j($7!V~T9o4Zb|Er|PyH6?Q%V9T$ z!l%z?|IBD0N_>~%selY|I*t670DZ~&cf8SiqPHRUgtdJ#L~*UVzn4nZAMl`uBb=@k zSDTyPF6*yM3m#Z0B6JwcKz?L2SQ>2K=xt{S1xWumd`W=h zVs<+G_lBq`VdT1Pyue99n%`_7i+yuw%cYIV_m4yI<4j@gzP^2wZ`)nTP@t5Qi|y$r zf3U3;d8^&WJyNJ4_mKWVn`tu%G9P5D!XLK=-c+F`Ppe((Uoe`6f#IkYFSOmLw+E<$ zzjCqs)FJa*O#JAvVWNbYNrJo4*Zf-^_=&_Y^UxJ;7-D*V@^pux8~ILiq`!5;_mT4t zVvbckba#n!_S1hZ(UgOm=$^>vAB-rrsDq#s1$^97D{cHrbC4qpakt!Q2gTM8q+j;P zzr!xOsM7NH%tsDG5GuM*yn>7_xp+SpC9gXFS=?T6^;jP95z`3>$hEue_d@M%IdqcA zmQ?>-XZ1DPR8OPPNLB9zuD1H1n>#qN4=|9s?OqvTqNvb0c<%V_>{0*Uu5XC=W5ti0 zrUFv@Od{yTR(Hr&cDypD#gE(>3sc!K1wvJ)2r~M?CB!t^I`3^mb(2?;ZJ0ZJ7fLFz zMyvDY+>RG_Nls3Kx@t{c1EGgYLg4cM_0s}`B=q6!zcrSlu*kZc#kTlCF{GG|K!=qj LU{P@T=HveZC;UA- literal 0 HcmV?d00001 diff --git a/docs/AdvancedInstallInstructions.md b/docs/usage/AdvancedInstallInstructions.md similarity index 100% rename from docs/AdvancedInstallInstructions.md rename to docs/usage/AdvancedInstallInstructions.md diff --git a/docs/Build_extensions.md b/docs/usage/Build_extensions.md similarity index 100% rename from docs/Build_extensions.md rename to docs/usage/Build_extensions.md diff --git a/docs/Linux_Osx_Install.md b/docs/usage/Linux_Osx_Install.md similarity index 100% rename from docs/Linux_Osx_Install.md rename to docs/usage/Linux_Osx_Install.md diff --git a/docs/Linux_Osx_Usage.md b/docs/usage/Linux_Osx_Usage.md similarity index 100% rename from docs/Linux_Osx_Usage.md rename to docs/usage/Linux_Osx_Usage.md diff --git a/personalities/english/general/gpt4all/config.yaml b/personalities/english/general/gpt4all/config.yaml index 32d0cde9..fdd59def 100644 --- a/personalities/english/general/gpt4all/config.yaml +++ b/personalities/english/general/gpt4all/config.yaml @@ -7,7 +7,7 @@ # talking to. #The version of the PyAIPersonality used to build this file -pyaipersonality_version: 0.0.2 +pyaipersonality_version: 0.0.5 #The version of the personality version: 1.0.0 @@ -30,24 +30,44 @@ personality_description: | # The conditionning instructions sent to eh model at the start of the discussion personality_conditioning: | - ##Instructions:\nGPT4All is a smart and helpful Assistant built by Nomic-AI. It can discuss with humans and assist them. + ## Information: + Assistant's name is gpt4all + Today's date is {{date}} + ## Instructions: + Your mission is to assist user to perform various tasks and answer his questions #Welcome message to be sent to the user when a new discussion is started -welcome_message: "Welcome! I am GPT4All A free and open assistant. What can I do for you today?" +welcome_message: | + Welcome! My name is gpt4all. + How can I help you today? # This prefix is added at the beginning of any message input by the user -user_message_prefix: "##Human:\n" - +user_message_prefix: "###user: + + " # A text to put between user and chatbot messages link_text: "\n" # This prefix is added at the beginning of any message output by the ai -ai_message_prefix: "##Assistant:\n" +ai_message_prefix: "###gpt4all: + + " # Here is the list of extensions this personality requires dependencies: [] +# A list of texts to be used to detect that the model is hallucinating and stop the generation if any one of these is output by the model +anti_prompts: ["###user","### user","###gpt4all","### gpt4all"] + # Some personalities need a disclaimer to warn the user of potential harm that can be caused by the AI # for example, for medical assistants, it is important to tell the user to be careful and not use medication # without advise from a real docor. -disclaimer: "" \ No newline at end of file +disclaimer: "" + +# Here are default model parameters +model_temperature: 0.6 # higher: more creative, lower more deterministic +model_n_predicts: 1024 # higher: generates many words, lower generates +model_top_k: 50 +model_top_p: 0.90 +model_repeat_penalty: 1.0 +model_repeat_last_n: 40 \ No newline at end of file diff --git a/pyGpt4All/api.py b/pyGpt4All/api.py index dfd8da9e..6c2ec68d 100644 --- a/pyGpt4All/api.py +++ b/pyGpt4All/api.py @@ -177,7 +177,7 @@ class GPT4AllAPI(): sys.stdout.flush() self.bot_says += text - if not self.personality.user_message_prefix.strip().lower() in self.bot_says.lower(): + if not self.personality.detect_antiprompt(self.bot_says): self.socketio.emit('message', {'data': self.bot_says}); if self.cancel_gen: print("Generation canceled") diff --git a/pyGpt4All/db.py b/pyGpt4All/db.py index ae6db2a9..2c092900 100644 --- a/pyGpt4All/db.py +++ b/pyGpt4All/db.py @@ -19,7 +19,7 @@ class DiscussionsDB: """ create database schema """ - db_version = 2 + db_version = 3 # Verify encoding and change it if it is not complient with sqlite3.connect(self.db_path) as conn: # Execute a PRAGMA statement to get the current encoding of the database @@ -52,10 +52,11 @@ class DiscussionsDB: schema_table_exist = True try: cursor.execute(""" - CREATE TABLE discussion ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - title TEXT - ) + CREATE TABLE discussion ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) """) except Exception: discussion_table_exist=True @@ -68,8 +69,10 @@ class DiscussionsDB: type INT NOT NULL, rank INT NOT NULL, parent INT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP discussion_id INTEGER NOT NULL, - FOREIGN KEY (discussion_id) REFERENCES discussion(id) + FOREIGN KEY (discussion_id) REFERENCES discussion(id), + FOREIGN KEY (parent) REFERENCES message(id) ) """ ) @@ -94,6 +97,9 @@ class DiscussionsDB: cursor.execute("ALTER TABLE message ADD COLUMN type INT DEFAULT 0") # Added in V1 cursor.execute("ALTER TABLE message ADD COLUMN rank INT DEFAULT 0") # Added in V1 cursor.execute("ALTER TABLE message ADD COLUMN parent INT DEFAULT 0") # Added in V2 + cursor.execute("ALTER TABLE message ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP") # Added in V3 + cursor.execute("ALTER TABLE discussion ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP") # Added in V3 + # Upgrade the schema to version 1 elif version < 2: print(f"Upgrading schema to version {db_version}...") @@ -101,6 +107,18 @@ class DiscussionsDB: if message_table_exist: try: cursor.execute("ALTER TABLE message ADD COLUMN parent INT DEFAULT 0") # Added in V2 + cursor.execute("ALTER TABLE message ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP") # Added in V3 + cursor.execute("ALTER TABLE discussion ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP") # Added in V3 + except : + pass + # Upgrade the schema to version 1 + elif version < 3: + print(f"Upgrading schema to version {db_version}...") + # Add the 'created_at' column to the 'message' table + if message_table_exist: + try: + cursor.execute("ALTER TABLE message ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP") # Added in V3 + cursor.execute("ALTER TABLE discussion ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP") # Added in V3 except : pass # Update the schema version diff --git a/requirements.txt b/requirements.txt index 53ea6e62..2eabf83d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ transformers accelerate gevent gevent-websocket -pyaipersonality \ No newline at end of file +pyaipersonality==0.0.5 \ No newline at end of file