2024-07-29 22:02:24 +00:00
// This requires axios
// In the html don't forget to import axios.min.js cdn
// <script src="https://cdn.jsdelivr.net/npm/axios/dist/axios.min.js"></script>
// JavaScript equivalent of the ELF_GENERATION_FORMAT enum
const ELF _GENERATION _FORMAT = {
LOLLMS : 0 ,
OPENAI : 1 ,
OLLAMA : 2 ,
2024-12-05 15:39:26 +00:00
LITELLM : 3 ,
VLLM : 4
2024-07-29 22:02:24 +00:00
} ;
// JavaScript equivalent of the ELF_COMPLETION_FORMAT enum
const ELF _COMPLETION _FORMAT = {
Instruct : 0 ,
Chat : 1
} ;
// Ensuring the objects are immutable
Object . freeze ( ELF _GENERATION _FORMAT ) ;
Object . freeze ( ELF _COMPLETION _FORMAT ) ;
class LollmsClient {
constructor (
host _address = null ,
model _name = null ,
2024-11-24 17:27:28 +00:00
ctx _size = null ,
2024-07-29 22:02:24 +00:00
personality = - 1 ,
2024-11-24 17:27:28 +00:00
n _predict = null ,
2024-07-29 22:02:24 +00:00
temperature = 0.1 ,
top _k = 50 ,
top _p = 0.95 ,
repeat _penalty = 0.8 ,
repeat _last _n = 40 ,
seed = null ,
n _threads = 8 ,
service _key = "" ,
default _generation _mode = ELF _GENERATION _FORMAT . LOLLMS
) {
// Handle the import or initialization of tiktoken equivalent in JavaScript
// this.tokenizer = new TikTokenJS('gpt-3.5-turbo-1106'); // This is hypothetical
this . host _address = host _address ;
this . model _name = model _name ;
this . ctx _size = ctx _size ;
2024-11-24 17:27:28 +00:00
this . n _predict = n _predict ? n _predict : null ;
2024-07-29 22:02:24 +00:00
this . personality = personality ;
this . temperature = temperature ;
this . top _k = top _k ;
this . top _p = top _p ;
this . repeat _penalty = repeat _penalty ;
this . repeat _last _n = repeat _last _n ;
this . seed = seed ;
this . n _threads = n _threads ;
this . service _key = service _key ;
this . default _generation _mode = default _generation _mode ;
2024-08-29 22:57:14 +00:00
this . minNPredict = 10
2024-07-29 22:02:24 +00:00
this . template = {
start _header _id _template : "!@>" ,
end _header _id _template : ": " ,
separator _template : "\n" ,
start _user _header _id _template : "!@>" ,
end _user _header _id _template : ": " ,
end _user _message _id _template : "" ,
start _ai _header _id _template : "!@>" ,
end _ai _header _id _template : ": " ,
end _ai _message _id _template : "" ,
system _message _template : "system"
}
fetch ( '/template' )
. then ( ( response ) => {
if ( ! response . ok ) {
throw new Error ( 'Network response was not ok ' + response . statusText ) ;
}
return response . json ( ) ;
} )
. then ( ( data ) => {
console . log ( "data: " , data ) ;
this . template = data ;
} )
. catch ( ( error ) => {
console . error ( 'Error fetching template:' , error ) ;
} ) ;
}
2024-08-29 22:57:14 +00:00
updateSettings ( settings ) {
// Update each setting if it's provided in the settings object
if ( 'host_address' in settings ) this . host _address = settings . host _address ;
if ( 'model_name' in settings ) this . model _name = settings . model _name ;
if ( 'ctx_size' in settings ) this . ctx _size = settings . ctx _size ;
if ( 'n_predict' in settings ) this . n _predict = settings . n _predict ;
if ( 'personality' in settings ) this . personality = settings . personality ;
if ( 'temperature' in settings ) this . temperature = settings . temperature ;
if ( 'top_k' in settings ) this . top _k = settings . top _k ;
if ( 'top_p' in settings ) this . top _p = settings . top _p ;
if ( 'repeat_penalty' in settings ) this . repeat _penalty = settings . repeat _penalty ;
if ( 'repeat_last_n' in settings ) this . repeat _last _n = settings . repeat _last _n ;
if ( 'seed' in settings ) this . seed = settings . seed ;
if ( 'n_threads' in settings ) this . n _threads = settings . n _threads ;
if ( 'service_key' in settings ) this . service _key = settings . service _key ;
if ( 'default_generation_mode' in settings ) this . default _generation _mode = settings . default _generation _mode ;
// You might want to add some validation or type checking here
console . log ( 'Settings updated:' , settings ) ;
}
2024-09-15 14:30:06 +00:00
separatorTemplate ( ) {
return this . template . separator _template ;
}
2024-07-29 22:02:24 +00:00
system _message ( ) {
return this . template . start _header _id _template + this . template . system _message _template + this . template . end _header _id _template
}
2024-12-17 14:50:21 +00:00
custom _system _message ( msg ) {
return this . template . start _header _id _template + msg + this . template . end _header _id _template
}
2024-07-29 22:02:24 +00:00
ai _message ( ai _name = "assistant" ) {
return this . template . start _ai _header _id _template + ai _name + this . template . end _ai _header _id _template
}
user _message ( user _name = "user" ) {
return this . template . start _user _header _id _template + user _name + this . template . end _user _header _id _template
}
2024-08-29 22:57:14 +00:00
custom _message ( message _name = "message" ) {
return this . template . start _ai _header _id _template + message _name + this . template . end _ai _header _id _template
}
2024-07-29 22:02:24 +00:00
updateServerAddress ( newAddress ) {
this . serverAddress = newAddress ;
}
2024-11-25 21:48:47 +00:00
async tokenize ( prompt , return _named = false ) {
2024-07-29 22:02:24 +00:00
/ * *
* Tokenizes the given prompt using the model ' s tokenizer .
*
* @ param { string } prompt - The input prompt to be tokenized .
* @ returns { Array } A list of tokens representing the tokenized prompt .
* /
2024-11-25 21:48:47 +00:00
const output = await axios . post ( "/lollms_tokenize" , { "prompt" : prompt , "return_named" : return _named } ) ;
2024-11-26 00:36:37 +00:00
if ( return _named )
{
2024-11-26 00:40:47 +00:00
return output . data
2024-11-26 00:36:37 +00:00
}
else {
return output . data
}
2024-07-29 22:02:24 +00:00
}
2024-11-25 21:48:47 +00:00
async detokenize ( tokensList , return _named = false ) {
2024-07-29 22:02:24 +00:00
/ * *
* Detokenizes the given list of tokens using the model ' s tokenizer .
*
* @ param { Array } tokensList - A list of tokens to be detokenized .
* @ returns { string } The detokenized text as a string .
* /
2024-11-25 21:48:47 +00:00
const output = await axios . post ( "/lollms_detokenize" , { "tokens" : tokensList , "return_named" : return _named } ) ;
2024-11-26 00:36:37 +00:00
if ( return _named )
{
console . log ( output . data . text )
return output . data . text
}
else {
console . log ( output . data )
return output . data
}
2024-08-06 13:45:11 +00:00
}
2024-11-25 14:03:49 +00:00
cancel _generation ( ) {
// TODO: implement
}
2024-07-29 22:02:24 +00:00
generate ( prompt , {
n _predict = null ,
stream = false ,
temperature = 0.1 ,
top _k = 50 ,
top _p = 0.95 ,
repeat _penalty = 0.8 ,
repeat _last _n = 40 ,
seed = null ,
n _threads = 8 ,
service _key = "" ,
streamingCallback = null
} = { } ) {
switch ( this . default _generation _mode ) {
case ELF _GENERATION _FORMAT . LOLLMS :
return this . lollms _generate ( prompt , this . host _address , this . model _name , - 1 , n _predict , stream , temperature , top _k , top _p , repeat _penalty , repeat _last _n , seed , n _threads , service _key , streamingCallback ) ;
case ELF _GENERATION _FORMAT . OPENAI :
return this . openai _generate ( prompt , this . host _address , this . model _name , - 1 , n _predict , stream , temperature , top _k , top _p , repeat _penalty , repeat _last _n , seed , n _threads , ELF _COMPLETION _FORMAT . INSTRUCT , service _key , streamingCallback ) ;
case ELF _GENERATION _FORMAT . OLLAMA :
return this . ollama _generate ( prompt , this . host _address , this . model _name , - 1 , n _predict , stream , temperature , top _k , top _p , repeat _penalty , repeat _last _n , seed , n _threads , ELF _COMPLETION _FORMAT . INSTRUCT , service _key , streamingCallback ) ;
case ELF _GENERATION _FORMAT . LITELLM :
return this . litellm _generate ( prompt , this . host _address , this . model _name , - 1 , n _predict , stream , temperature , top _k , top _p , repeat _penalty , repeat _last _n , seed , n _threads , ELF _COMPLETION _FORMAT . INSTRUCT , service _key , streamingCallback ) ;
2024-12-05 15:39:26 +00:00
case ELF _GENERATION _FORMAT . VLLM :
return this . vllm _generate ( prompt , this . host _address , this . model _name , - 1 , n _predict , stream , temperature , top _k , top _p , repeat _penalty , repeat _last _n , seed , n _threads , ELF _COMPLETION _FORMAT . INSTRUCT , service _key , streamingCallback ) ;
2024-07-29 22:02:24 +00:00
default :
throw new Error ( 'Invalid generation mode' ) ;
}
}
2024-08-06 13:45:11 +00:00
generate _with _images ( prompt , images , {
n _predict = null ,
stream = false ,
temperature = 0.1 ,
top _k = 50 ,
top _p = 0.95 ,
repeat _penalty = 0.8 ,
repeat _last _n = 40 ,
seed = null ,
n _threads = 8 ,
service _key = "" ,
streamingCallback = null
} = { } ) {
switch ( this . default _generation _mode ) {
case ELF _GENERATION _FORMAT . LOLLMS :
return this . lollms _generate _with _images ( prompt , images , this . host _address , this . model _name , - 1 , n _predict , stream , temperature , top _k , top _p , repeat _penalty , repeat _last _n , seed , n _threads , service _key , streamingCallback ) ;
2024-09-04 23:54:52 +00:00
case ELF _GENERATION _FORMAT . OPENAI :
return this . openai _generate _with _images ( prompt , images , this . host _address , this . model _name , - 1 , n _predict , stream , temperature , top _k , top _p , repeat _penalty , repeat _last _n , seed , n _threads , ELF _COMPLETION _FORMAT . INSTRUCT , service _key , streamingCallback ) ;
2024-08-06 13:45:11 +00:00
default :
throw new Error ( 'Invalid generation mode' ) ;
}
}
2024-07-29 22:02:24 +00:00
async generateText ( prompt , options = { } ) {
// Destructure with default values from `this` if not provided in `options`
const {
host _address = this . host _address ,
model _name = this . model _name ,
personality = this . personality ,
n _predict = this . n _predict ,
stream = false ,
temperature = this . temperature ,
top _k = this . top _k ,
top _p = this . top _p ,
repeat _penalty = this . repeat _penalty ,
repeat _last _n = this . repeat _last _n ,
seed = this . seed ,
n _threads = this . n _threads ,
service _key = this . service _key ,
streamingCallback = null
} = options ;
try {
const result = await this . lollms _generate (
prompt ,
host _address ,
model _name ,
personality ,
n _predict ,
stream ,
temperature ,
top _k ,
top _p ,
repeat _penalty ,
repeat _last _n ,
seed ,
n _threads ,
service _key ,
streamingCallback
) ;
return result ;
} catch ( error ) {
// Handle any errors that occur during generation
console . error ( 'An error occurred during text generation:' , error ) ;
throw error ; // Re-throw the error if you want to allow the caller to handle it as well
}
}
async lollms _generate ( prompt , host _address = this . host _address , model _name = this . model _name , personality = this . personality , n _predict = this . n _predict , stream = false , temperature = this . temperature , top _k = this . top _k , top _p = this . top _p , repeat _penalty = this . repeat _penalty , repeat _last _n = this . repeat _last _n , seed = this . seed , n _threads = this . n _threads , service _key = this . service _key , streamingCallback = null ) {
let url ;
if ( host _address != null ) {
url = ` ${ host _address } /lollms_generate ` ;
}
else {
url = ` /lollms_generate ` ;
}
const headers = service _key !== "" ? {
'Content-Type' : 'application/json; charset=utf-8' ,
'Authorization' : ` Bearer ${ service _key } ` ,
} : {
'Content-Type' : 'application/json' ,
} ;
2024-08-06 13:45:11 +00:00
console . log ( "n_predict:" , n _predict )
console . log ( "self.n_predict:" , this . n _predict )
2024-07-29 22:02:24 +00:00
const data = JSON . stringify ( {
prompt : prompt ,
model _name : model _name ,
personality : personality ,
2024-08-06 13:45:11 +00:00
n _predict : n _predict ? n _predict : this . n _predict ,
2024-07-29 22:02:24 +00:00
stream : stream ,
temperature : temperature ,
top _k : top _k ,
top _p : top _p ,
repeat _penalty : repeat _penalty ,
repeat _last _n : repeat _last _n ,
seed : seed ,
n _threads : n _threads
} ) ;
try {
const response = await fetch ( url , {
method : 'POST' ,
headers : headers ,
body : data
} ) ;
// Check if the response is okay
if ( ! response . ok ) {
throw new Error ( 'Network response was not ok ' + response . statusText ) ;
}
// Read the response as plaintext
const responseBody = await response . text ( ) ;
console . log ( responseBody )
return responseBody ;
} catch ( error ) {
console . error ( error ) ;
return null ;
}
}
2024-08-06 13:45:11 +00:00
async lollms _generate _with _images ( prompt , images , host _address = this . host _address , model _name = this . model _name , personality = this . personality , n _predict = this . n _predict , stream = false , temperature = this . temperature , top _k = this . top _k , top _p = this . top _p , repeat _penalty = this . repeat _penalty , repeat _last _n = this . repeat _last _n , seed = this . seed , n _threads = this . n _threads , service _key = this . service _key , streamingCallback = null ) {
let url ;
if ( host _address != null ) {
url = ` ${ host _address } /lollms_generate_with_images ` ;
}
else {
url = ` /lollms_generate_with_images ` ;
}
const headers = service _key !== "" ? {
'Content-Type' : 'application/json; charset=utf-8' ,
'Authorization' : ` Bearer ${ service _key } ` ,
} : {
'Content-Type' : 'application/json' ,
} ;
console . log ( "n_predict:" , n _predict )
console . log ( "self.n_predict:" , this . n _predict )
const data = JSON . stringify ( {
prompt : prompt ,
images : images ,
model _name : model _name ,
personality : personality ,
n _predict : n _predict ? n _predict : this . n _predict ,
stream : stream ,
temperature : temperature ,
top _k : top _k ,
top _p : top _p ,
repeat _penalty : repeat _penalty ,
repeat _last _n : repeat _last _n ,
seed : seed ,
n _threads : n _threads
} ) ;
try {
const response = await fetch ( url , {
method : 'POST' ,
headers : headers ,
body : data
} ) ;
2024-07-29 22:02:24 +00:00
2024-08-06 13:45:11 +00:00
// Check if the response is okay
if ( ! response . ok ) {
throw new Error ( 'Network response was not ok ' + response . statusText ) ;
}
// Read the response as plaintext
const responseBody = await response . text ( ) ;
console . log ( responseBody )
return responseBody ;
} catch ( error ) {
console . error ( error ) ;
return null ;
}
}
2024-07-29 22:02:24 +00:00
async openai _generate ( prompt , host _address = this . host _address , model _name = this . model _name , personality = this . personality , n _predict = this . n _predict , stream = false , temperature = this . temperature , top _k = this . top _k , top _p = this . top _p , repeat _penalty = this . repeat _penalty , repeat _last _n = this . repeat _last _n , seed = this . seed , n _threads = this . n _threads , ELF _COMPLETION _FORMAT = "vllm instruct" , service _key = this . service _key , streamingCallback = null ) {
const url = ` ${ host _address } /generate_completion ` ;
const headers = service _key !== "" ? {
'Content-Type' : 'application/json; charset=utf-8' ,
'Authorization' : ` Bearer ${ service _key } ` ,
} : {
'Content-Type' : 'application/json' ,
} ;
const data = JSON . stringify ( {
prompt : prompt ,
model _name : model _name ,
personality : personality ,
n _predict : n _predict ,
stream : stream ,
temperature : temperature ,
top _k : top _k ,
top _p : top _p ,
repeat _penalty : repeat _penalty ,
repeat _last _n : repeat _last _n ,
seed : seed ,
n _threads : n _threads ,
completion _format : ELF _COMPLETION _FORMAT
} ) ;
try {
const response = await fetch ( url , {
method : 'POST' ,
headers : headers ,
body : data
} ) ;
if ( stream && streamingCallback ) {
// Note: Streaming with Fetch API in the browser might not work as expected because Fetch API does not support HTTP/2 server push.
// You would need a different approach for real-time streaming.
streamingCallback ( await response . json ( ) , 'MSG_TYPE_CHUNK' ) ;
} else {
return await response . json ( ) ;
}
} catch ( error ) {
console . error ( "Error generating completion:" , error ) ;
return null ;
}
}
2024-12-05 15:39:26 +00:00
2024-09-04 23:54:52 +00:00
async openai _generate _with _images ( prompt , images , options = { } ) {
const {
host _address = this . host _address ,
model _name = this . model _name ,
personality = this . personality ,
n _predict = this . n _predict ,
stream = false ,
temperature = this . temperature ,
top _k = this . top _k ,
top _p = this . top _p ,
repeat _penalty = this . repeat _penalty ,
repeat _last _n = this . repeat _last _n ,
seed = this . seed ,
n _threads = this . n _threads ,
max _image _width = - 1 ,
service _key = this . service _key ,
streamingCallback = null ,
} = options ;
const headers = {
'Content-Type' : 'application/json' ,
... ( service _key ? { 'Authorization' : ` Bearer ${ service _key } ` } : { } )
} ;
const data = {
model : model _name ,
messages : [
{
role : "user" ,
content : [
{
type : "text" ,
text : prompt
} ,
... images . map ( image _path => ( {
type : "image_url" ,
image _url : {
url : ` data:image/jpeg;base64, ${ this . encode _image ( image _path , max _image _width ) } `
}
} ) )
]
}
] ,
stream : true ,
temperature : parseFloat ( temperature ) ,
max _tokens : n _predict
} ;
const completion _format _path = "/v1/chat/completions" ;
const url = ` ${ host _address . endsWith ( "/" ) ? host _address . slice ( 0 , - 1 ) : host _address } ${ completion _format _path } ` ;
try {
const response = await fetch ( url , {
method : 'POST' ,
headers : headers ,
body : JSON . stringify ( data )
} ) ;
if ( ! response . ok ) {
const content = await response . json ( ) ;
if ( response . status === 400 ) {
this . error ( content . error ? . message || content . message ) ;
} else if ( response . status === 404 ) {
console . error ( await response . text ( ) ) ;
}
return ;
}
const reader = response . body . getReader ( ) ;
const decoder = new TextDecoder ( ) ;
let text = "" ;
while ( true ) {
const { done , value } = await reader . read ( ) ;
if ( done ) break ;
const chunk = decoder . decode ( value ) ;
const lines = chunk . split ( '\n' ) ;
for ( const line of lines ) {
if ( line . startsWith ( "data: " ) ) {
try {
const jsonData = JSON . parse ( line . slice ( 5 ) . trim ( ) ) ;
const content = jsonData . choices [ 0 ] ? . delta ? . content || "" ;
text += content ;
if ( streamingCallback ) {
if ( ! streamingCallback ( content , "MSG_TYPE_CHUNK" ) ) {
return text ;
}
}
} catch ( error ) {
// Handle JSON parsing error
}
} else if ( line . startsWith ( "{" ) ) {
try {
const jsonData = JSON . parse ( line ) ;
if ( jsonData . object === "error" ) {
this . error ( jsonData . message ) ;
return text ;
}
} catch ( error ) {
this . error ( "Couldn't generate text, verify your key or model name" ) ;
}
} else {
text += line ;
if ( streamingCallback ) {
if ( ! streamingCallback ( line , "MSG_TYPE_CHUNK" ) ) {
return text ;
}
}
}
}
}
return text ;
} catch ( error ) {
console . error ( "Error in openai_generate_with_images:" , error ) ;
throw error ;
}
}
2024-12-05 15:39:26 +00:00
async vllm _generate ( {
prompt ,
host _address = null ,
model _name = null ,
personality = null ,
n _predict = null ,
stream = false ,
temperature = null ,
top _k = null ,
top _p = null ,
repeat _penalty = null ,
repeat _last _n = null ,
seed = null ,
n _threads = null ,
completion _format = ELF _COMPLETION _FORMAT . Instruct , // Instruct or Chat
service _key = "" ,
streaming _callback = null
} ) {
// Set default values to instance variables if optional arguments are null
host _address = host _address || this . host _address ;
model _name = model _name || this . model _name ;
n _predict = n _predict || this . n _predict || this . minNPredict ;
personality = personality !== null ? personality : this . personality ;
temperature = temperature !== null ? temperature : this . temperature ;
top _k = top _k !== null ? top _k : this . top _k ;
top _p = top _p !== null ? top _p : this . top _p ;
repeat _penalty = repeat _penalty !== null ? repeat _penalty : this . repeat _penalty ;
repeat _last _n = repeat _last _n !== null ? repeat _last _n : this . repeat _last _n ;
seed = seed || this . seed ;
n _threads = n _threads || this . n _threads ;
const headers = {
"Content-Type" : "application/json" ,
... ( service _key && { Authorization : ` Bearer ${ service _key } ` } )
} ;
let data ;
let completionFormatPath ;
if ( completion _format === ELF _COMPLETION _FORMAT . Instruct ) {
data = {
model : model _name ,
prompt : prompt ,
stream : stream ,
temperature : parseFloat ( temperature ) ,
max _tokens : n _predict
} ;
completionFormatPath = "/v1/completions" ;
} else if ( completion _format === ELF _COMPLETION _FORMAT . Chat ) {
data = {
model : model _name ,
messages : [
{
role : "user" ,
content : prompt
}
] ,
stream : stream ,
temperature : parseFloat ( temperature ) ,
max _tokens : n _predict
} ;
completionFormatPath = "/v1/chat/completions" ;
}
if ( host _address . endsWith ( "/" ) ) {
host _address = host _address . slice ( 0 , - 1 ) ;
}
const url = ` ${ host _address } ${ completionFormatPath } ` ;
try {
const response = await axios . post ( url , data , {
headers : headers ,
responseType : stream ? "stream" : "json" ,
httpsAgent : this . verifySslCertificate
? undefined
: new ( require ( "https" ) . Agent ) ( { rejectUnauthorized : false } )
} ) ;
if ( stream ) {
let text = "" ;
response . data . on ( "data" , ( chunk ) => {
const decoded = chunk . toString ( "utf-8" ) ;
if ( decoded . startsWith ( "data: " ) ) {
try {
const jsonData = JSON . parse ( decoded . slice ( 5 ) . trim ( ) ) ;
let chunkContent = "" ;
if ( completion _format === ELF _COMPLETION _FORMAT . Chat ) {
chunkContent = jsonData . choices [ 0 ] ? . delta ? . content || "" ;
} else {
chunkContent = jsonData . choices [ 0 ] ? . text || "" ;
}
text += chunkContent ;
if ( streaming _callback ) {
if ( ! streaming _callback ( chunkContent , "MSG_TYPE_CHUNK" ) ) {
response . data . destroy ( ) ;
}
}
} catch ( error ) {
response . data . destroy ( ) ;
}
}
} ) ;
return new Promise ( ( resolve , reject ) => {
response . data . on ( "end" , ( ) => resolve ( text ) ) ;
response . data . on ( "error" , ( err ) => reject ( err ) ) ;
} ) ;
} else {
return response . data ;
}
} catch ( error ) {
if ( error . response ) {
const errorMessage =
error . response . data ? . error ? . message ||
error . response . data ? . message ||
"Unknown error occurred" ;
console . error ( errorMessage ) ;
throw new Error ( errorMessage ) ;
} else {
console . error ( error . message ) ;
throw error ;
}
}
}
2024-09-04 23:54:52 +00:00
async encode _image ( image _path , max _image _width = - 1 ) {
// In a browser environment, we'll use the File API and canvas
// For Node.js, you'd need to use libraries like 'sharp' or 'jimp'
return new Promise ( ( resolve , reject ) => {
const img = new Image ( ) ;
img . onload = ( ) => {
let width = img . width ;
let height = img . height ;
// Resize if necessary
if ( max _image _width !== - 1 && width > max _image _width ) {
const ratio = max _image _width / width ;
width = max _image _width ;
height = Math . round ( height * ratio ) ;
}
// Create a canvas to draw the image
const canvas = document . createElement ( 'canvas' ) ;
canvas . width = width ;
canvas . height = height ;
const ctx = canvas . getContext ( '2d' ) ;
// Draw the image on the canvas
ctx . drawImage ( img , 0 , 0 , width , height ) ;
// Convert to base64
const base64Image = canvas . toDataURL ( 'image/jpeg' ) . split ( ',' ) [ 1 ] ;
resolve ( base64Image ) ;
} ;
img . onerror = ( error ) => reject ( error ) ;
// Load the image from the provided path
img . src = image _path ;
} ) ;
}
async generateCode ( prompt , images = [ ] , {
n _predict = null ,
stream = false ,
temperature = 0.1 ,
top _k = 50 ,
top _p = 0.95 ,
repeat _penalty = 0.8 ,
repeat _last _n = 40 ,
seed = null ,
n _threads = 8 ,
service _key = "" ,
streamingCallback = null
} = { } ) {
let response ;
const systemHeader = this . custom _message ( "Generation infos" ) ;
2024-09-15 14:30:06 +00:00
const codeInstructions = "Generated code must be put inside the adequate markdown code tag. Use this template:\n```language name\nCode\n```\nMake sure only a single code tag is generated at each dialogue turn.\n" ;
const fullPrompt = systemHeader + codeInstructions + this . separatorTemplate ( ) + prompt ;
2024-09-04 23:54:52 +00:00
if ( images . length > 0 ) {
response = await this . generate _with _images ( fullPrompt , images , {
n _predict : n _predict ,
temperature : temperature ,
top _k : top _k ,
top _p : top _p ,
repeat _penalty : repeat _penalty ,
repeat _last _n : repeat _last _n ,
callback : streamingCallback
} ) ;
} else {
response = await this . generate ( fullPrompt , {
n _predict : n _predict ,
temperature : temperature ,
top _k : top _k ,
top _p : top _p ,
repeat _penalty : repeat _penalty ,
repeat _last _n : repeat _last _n ,
callback : streamingCallback
} ) ;
}
const codes = this . extractCodeBlocks ( response ) ;
2024-12-15 23:04:58 +00:00
console . log ( codes )
2024-09-04 23:54:52 +00:00
if ( codes . length > 0 ) {
let code = '' ;
if ( ! codes [ 0 ] . is _complete ) {
code = codes [ 0 ] . content . split ( '\n' ) . slice ( 0 , - 1 ) . join ( '\n' ) ;
while ( ! codes [ 0 ] . is _complete ) {
console . warn ( "The AI did not finish the code, let's ask it to continue" )
2024-11-02 21:16:22 +00:00
const continuePrompt = prompt + code + this . userFullHeader + "continue the code. Rewrite last line and continue the code. Don't forget to put the code inside a markdown code tag." + this . separatorTemplate ( ) + this . aiFullHeader ;
2024-11-24 17:27:28 +00:00
response = await this . generate ( continuePrompt , {
2024-09-04 23:54:52 +00:00
n _predict : n _predict ,
temperature : temperature ,
top _k : top _k ,
top _p : top _p ,
repeat _penalty : repeat _penalty ,
repeat _last _n : repeat _last _n ,
callback : streamingCallback
} ) ;
2024-11-24 22:38:28 +00:00
const codes = this . extractCodeBlocks ( response ) ;
if ( codes . length === 0 ) break ;
2024-09-04 23:54:52 +00:00
2024-11-24 22:38:28 +00:00
if ( ! codes [ 0 ] . is _complete ) {
code += '\n' + codes [ 0 ] . content . split ( '\n' ) . slice ( 0 , - 1 ) . join ( '\n' ) ;
2024-09-04 23:54:52 +00:00
} else {
2024-11-24 22:38:28 +00:00
code += '\n' + codes [ 0 ] . content ;
2024-09-04 23:54:52 +00:00
}
}
} else {
code = codes [ 0 ] . content ;
}
return code ;
} else {
return null ;
}
}
2024-09-08 00:58:54 +00:00
async generateCodes ( prompt , images = [ ] , {
n _predict = null ,
stream = false ,
temperature = 0.1 ,
top _k = 50 ,
top _p = 0.95 ,
repeat _penalty = 0.8 ,
repeat _last _n = 40 ,
seed = null ,
n _threads = 8 ,
service _key = "" ,
streamingCallback = null
} = { } ) {
let response ;
const systemHeader = this . custom _message ( "Generation infos" ) ;
const codeInstructions = "Generated code must be put inside the adequate markdown code tag. Use this template:\n```language name\nCode\n```\n" ;
2024-09-15 14:30:06 +00:00
const fullPrompt = systemHeader + codeInstructions + this . separatorTemplate ( ) + prompt ;
2024-09-08 00:58:54 +00:00
if ( images . length > 0 ) {
response = await this . generate _with _images ( fullPrompt , images , {
n _predict ,
temperature ,
top _k ,
top _p ,
repeat _penalty ,
repeat _last _n ,
callback : streamingCallback
} ) ;
} else {
response = await this . generate ( fullPrompt , {
n _predict ,
temperature ,
top _k ,
top _p ,
repeat _penalty ,
repeat _last _n ,
callback : streamingCallback
} ) ;
}
let codes = this . extractCodeBlocks ( response ) ;
let completeCodes = [ ] ;
while ( codes . length > 0 ) {
let currentCode = codes . shift ( ) ;
let codeContent = currentCode . content ;
while ( ! currentCode . is _complete ) {
console . warn ( "The AI did not finish the code, let's ask it to continue" ) ;
2024-09-15 14:30:06 +00:00
const continuePrompt = prompt + codeContent + this . userFullHeader + "continue the code. Rewrite last line and continue the code." + this . separatorTemplate ( ) + this . aiFullHeader ;
2024-09-08 00:58:54 +00:00
response = await this . generate ( continuePrompt , {
n _predict ,
temperature ,
top _k ,
top _p ,
repeat _penalty ,
repeat _last _n ,
callback : streamingCallback
} ) ;
const newCodes = this . extractCodeBlocks ( response ) ;
if ( newCodes . length === 0 ) break ;
// Append the content of the first new code block
codeContent += '\n' + newCodes [ 0 ] . content ;
currentCode = newCodes [ 0 ] ;
// If there are more code blocks, add them to the codes array
if ( newCodes . length > 1 ) {
codes = [ ... newCodes . slice ( 1 ) , ... codes ] ;
}
}
completeCodes . push ( {
language : currentCode . language ,
content : codeContent
} ) ;
}
return completeCodes ;
}
2024-12-15 23:04:58 +00:00
extractCodeBlocks ( text , return _remaining _text = false ) {
const codes = [ ] ;
let remainingText = text ;
let currentIndex = 0 ;
while ( true ) {
// Find next code block start
const startPos = remainingText . indexOf ( '```' ) ;
if ( startPos === - 1 ) {
break ;
2024-09-04 23:54:52 +00:00
}
2024-12-15 23:04:58 +00:00
// Check for file name before code block
let fileName = '' ;
const fileNameMatch = remainingText . slice ( 0 , startPos ) . lastIndexOf ( '<file_name>' ) ;
if ( fileNameMatch !== - 1 ) {
const fileNameEnd = remainingText . slice ( 0 , startPos ) . lastIndexOf ( '</file_name>' ) ;
if ( fileNameEnd !== - 1 && fileNameMatch < fileNameEnd ) {
fileName = remainingText . slice ( fileNameMatch + 11 , fileNameEnd ) . trim ( ) ;
}
2024-09-04 23:54:52 +00:00
}
2024-12-15 23:04:58 +00:00
// Get code type if specified
let codeType = '' ;
const nextNewline = remainingText . indexOf ( '\n' , startPos + 3 ) ;
let contentStart ;
if ( nextNewline !== - 1 ) {
const potentialType = remainingText . slice ( startPos + 3 , nextNewline ) . trim ( ) ;
if ( potentialType ) {
codeType = potentialType ;
contentStart = nextNewline + 1 ;
2024-09-04 23:54:52 +00:00
} else {
2024-12-15 23:04:58 +00:00
contentStart = startPos + 3 ;
2024-09-04 23:54:52 +00:00
}
2024-12-15 23:04:58 +00:00
} else {
contentStart = startPos + 3 ;
2024-09-04 23:54:52 +00:00
}
2024-12-15 23:04:58 +00:00
// Find matching end tag
let pos = contentStart ;
let is _complete = false ;
// Find the closing backticks
const endPos = remainingText . indexOf ( '```' , contentStart ) ;
if ( endPos !== - 1 ) {
// Found matching end tag
const content = remainingText . slice ( contentStart , endPos ) . trim ( ) ;
is _complete = true ;
codes . push ( {
index : currentIndex ,
fileName : fileName ,
content : content ,
type : codeType ,
is _complete : true
} ) ;
remainingText = remainingText . slice ( endPos + 3 ) ;
} else {
// Handle incomplete code block
const content = remainingText . slice ( contentStart ) . trim ( ) ;
codes . push ( {
index : currentIndex ,
fileName : fileName ,
content : content ,
type : codeType ,
is _complete : false
} ) ;
remainingText = '' ;
}
currentIndex ++ ;
2024-09-04 23:54:52 +00:00
}
2024-12-15 23:04:58 +00:00
if ( return _remaining _text ) {
return { codes , remainingText } ;
}
return codes ;
2024-09-04 23:54:52 +00:00
}
2024-07-29 22:02:24 +00:00
async listMountedPersonalities ( host _address = this . host _address ) {
const url = ` ${ host _address } /list_mounted_personalities ` ;
try {
const response = await fetch ( url ) ;
return await response . json ( ) ;
} catch ( error ) {
console . error ( error ) ;
return null ;
}
}
async listModels ( host _address = this . host _address ) {
const url = ` ${ host _address } /list_models ` ;
try {
const response = await fetch ( url ) ;
return await response . json ( ) ;
} catch ( error ) {
console . error ( error ) ;
return null ;
}
}
}
2024-09-12 08:57:10 +00:00
class TextChunker {
2024-12-17 13:50:58 +00:00
constructor ( chunkSize = 1024 , overlap = 0 , tokenizer = null , model = null ) {
2024-09-12 08:57:10 +00:00
this . chunkSize = chunkSize ;
this . overlap = overlap ;
this . tokenizer = tokenizer || new TikTokenTokenizer ( ) ;
this . model = model ;
}
2024-09-18 11:43:49 +00:00
async getTextChunks ( text , doc , cleanChunk = true , minNbTokensInChunk = 10 ) {
2024-09-12 08:57:10 +00:00
const paragraphs = text . split ( '\n\n' ) ;
const chunks = [ ] ;
let currentChunk = [ ] ;
let currentTokens = 0 ;
let chunkId = 0 ;
for ( const paragraph of paragraphs ) {
const cleanedParagraph = cleanChunk ? paragraph . trim ( ) : paragraph ;
2024-09-18 11:43:49 +00:00
const paragraphTokens = ( await this . tokenizer . tokenize ( cleanedParagraph ) ) . length ;
2024-09-12 08:57:10 +00:00
if ( currentTokens + paragraphTokens > this . chunkSize ) {
if ( currentTokens > minNbTokensInChunk ) {
let chunkText = currentChunk . join ( '\n\n' ) ;
if ( cleanChunk ) {
chunkText = TextChunker . removeUnnecessaryReturns ( chunkText ) ;
}
const chunk = new Chunk ( doc , '' , chunkText , currentTokens , chunkId ) ;
chunkId ++ ;
chunks . push ( chunk ) ;
}
if ( this . overlap > 0 ) {
currentChunk = [ ... currentChunk . slice ( - this . overlap ) , cleanedParagraph ] ;
} else {
currentChunk = [ cleanedParagraph ] ;
}
2024-09-18 11:43:49 +00:00
currentTokens = currentChunk . reduce ( async ( sum , p ) => sum + await this . tokenizer . tokenize ( p ) . length , 0 ) ;
2024-09-12 08:57:10 +00:00
} else {
currentChunk . push ( cleanedParagraph ) ;
currentTokens += paragraphTokens ;
}
}
if ( currentChunk . length > 0 && currentTokens > minNbTokensInChunk ) {
let chunkText = currentChunk . join ( '\n\n' ) ;
if ( cleanChunk ) {
chunkText = TextChunker . removeUnnecessaryReturns ( chunkText ) ;
}
const chunk = new Chunk ( doc , '' , chunkText , currentTokens , chunkId ) ;
chunks . push ( chunk ) ;
}
return chunks ;
}
static removeUnnecessaryReturns ( paragraph ) {
const lines = paragraph . split ( '\n' ) ;
return lines . filter ( line => line . trim ( ) ) . join ( '\n' ) ;
}
2024-12-17 13:50:58 +00:00
static async chunkText ( text , tokenizer , chunkSize = 1024 , overlap = 0 , cleanChunk = true , minNbTokensInChunk = 10 ) {
// Validate chunkSize
if ( isNaN ( chunkSize ) || chunkSize <= 0 ) {
console . warn ( ` Invalid chunkSize: ${ chunkSize } . Resetting to default value of 1024. ` ) ;
chunkSize = 1024 ;
}
2024-09-12 08:57:10 +00:00
const paragraphs = text . split ( '\n\n' ) ;
const chunks = [ ] ;
let currentChunk = [ ] ;
let currentTokens = 0 ;
2024-12-17 13:50:58 +00:00
console . log ( "Starting text chunking..." ) ;
console . log ( ` Using chunkSize: ${ chunkSize } , overlap: ${ overlap } , minNbTokensInChunk: ${ minNbTokensInChunk } ` ) ;
for ( const paragraph of paragraphs ) {
const cleanedParagraph = cleanChunk ? paragraph . trim ( ) : paragraph ;
const paragraphTokens = ( await tokenizer . tokenize ( cleanedParagraph ) ) . length ;
console . log ( ` Processing paragraph: " ${ cleanedParagraph } " ` ) ;
console . log ( ` Paragraph tokens: ${ paragraphTokens } ` ) ;
console . log ( ` Current tokens before adding: ${ currentTokens } ` ) ;
// Handle case where a single paragraph exceeds chunkSize
if ( paragraphTokens > chunkSize ) {
console . log ( ` Paragraph exceeds chunk size. Splitting... ` ) ;
const splitParagraphs = await this . splitLargeParagraph ( cleanedParagraph , tokenizer , chunkSize , overlap ) ;
for ( const subChunk of splitParagraphs ) {
chunks . push ( subChunk ) ;
console . log ( ` Chunk created from large paragraph: " ${ subChunk } " ` ) ;
}
continue ;
2024-09-12 08:57:10 +00:00
}
2024-12-17 13:50:58 +00:00
// If adding this paragraph exceeds the chunk size
if ( currentTokens + paragraphTokens > chunkSize ) {
if ( currentTokens >= minNbTokensInChunk ) {
let chunkText = currentChunk . join ( '\n\n' ) ;
if ( cleanChunk ) {
chunkText = TextChunker . removeUnnecessaryReturns ( chunkText ) ;
}
chunks . push ( chunkText ) ;
console . log ( ` Intermediate chunk created and added: " ${ chunkText } " ` ) ;
} else {
console . log ( "Skipping chunk creation due to insufficient tokens." ) ;
}
// Handle overlap
if ( overlap > 0 ) {
currentChunk = currentChunk . slice ( - overlap ) ; // Keep only the overlapping part
} else {
currentChunk = [ ] ;
}
currentChunk . push ( cleanedParagraph ) ;
currentTokens = paragraphTokens ; // Reset token count to the new paragraph's tokens
2024-09-12 08:57:10 +00:00
} else {
2024-12-17 13:50:58 +00:00
// Add paragraph to the current chunk
currentChunk . push ( cleanedParagraph ) ;
currentTokens += paragraphTokens ;
2024-09-12 08:57:10 +00:00
}
2024-12-17 13:50:58 +00:00
console . log ( ` Current chunk after processing: ${ currentChunk . join ( ' | ' ) } ` ) ;
console . log ( ` Current tokens after processing: ${ currentTokens } ` ) ;
}
// Add the last chunk if it meets the minimum token requirement
if ( currentChunk . length > 0 && currentTokens >= minNbTokensInChunk ) {
let chunkText = currentChunk . join ( '\n\n' ) ;
if ( cleanChunk ) {
chunkText = TextChunker . removeUnnecessaryReturns ( chunkText ) ;
}
chunks . push ( chunkText ) ;
console . log ( ` Final chunk created and added: " ${ chunkText } " ` ) ;
} else {
console . log ( "No final chunk created due to insufficient tokens." ) ;
}
console . log ( "Final Chunks:" ) ;
console . log ( chunks ) ;
return chunks ;
}
// Helper function to split a large paragraph into smaller chunks
static async splitLargeParagraph ( paragraph , tokenizer , chunkSize , overlap ) {
const tokens = await tokenizer . tokenize ( paragraph ) ;
const chunks = [ ] ;
let start = 0 ;
while ( start < tokens . length ) {
const end = Math . min ( start + chunkSize , tokens . length ) ;
const chunkTokens = tokens . slice ( start , end ) ;
2024-12-17 14:54:40 +00:00
const chunkText = await tokenizer . detokenize ( chunkTokens ) ;
2024-12-17 13:50:58 +00:00
chunks . push ( chunkText ) ;
start += chunkSize - overlap ; // Move start forward with overlap
2024-09-12 08:57:10 +00:00
}
return chunks ;
}
2024-12-17 13:50:58 +00:00
2024-09-12 08:57:10 +00:00
}
2024-07-29 22:02:24 +00:00
class TasksLibrary {
constructor ( lollms ) {
this . lollms = lollms ;
}
async translateTextChunk ( textChunk , outputLanguage = "french" , host _address = null , model _name = null , temperature = 0.1 , maxGenerationSize = 3000 ) {
const translationPrompt = [
2024-10-17 23:08:27 +00:00
` ${ this . lollms . system _message ( ) } ` ,
2024-07-29 22:02:24 +00:00
` Translate the following text to ${ outputLanguage } . ` ,
` Be faithful to the original text and do not add or remove any information. ` ,
` Respond only with the translated text. ` ,
` Do not add comments or explanations. ` ,
2024-10-17 23:08:27 +00:00
` ${ this . lollms . user _message ( "text to translate" ) } ` ,
2024-07-29 22:02:24 +00:00
` ${ textChunk } ` ,
2024-10-17 23:08:27 +00:00
` ${ this . lollms . ai _message ( "translation" ) } ` ,
2024-07-29 22:02:24 +00:00
] . join ( "\n" ) ;
const translated = await this . lollms . generateText (
translationPrompt ,
host _address ,
model _name ,
- 1 , // personality
maxGenerationSize , // n_predict
false , // stream
temperature , // temperature
undefined , // top_k, using undefined to fallback on LollmsClient's default
undefined , // top_p, using undefined to fallback on LollmsClient's default
undefined , // repeat_penalty, using undefined to fallback on LollmsClient's default
undefined , // repeat_last_n, using undefined to fallback on LollmsClient's default
undefined , // seed, using undefined to fallback on LollmsClient's default
undefined , // n_threads, using undefined to fallback on LollmsClient's default
undefined // service_key, using undefined to fallback on LollmsClient's default
) ;
return translated ;
}
2024-08-29 22:57:14 +00:00
async tokenize ( text ) {
// Assuming the LollmsClient has a method to tokenize text
return await this . lollms . tokenize ( text ) ;
}
2024-09-04 12:16:41 +00:00
async summarizeText (
2024-08-29 22:57:14 +00:00
text ,
summaryInstruction = "summarize" ,
docName = "chunk" ,
answerStart = "" ,
maxGenerationSize = 3000 ,
2024-12-17 13:50:58 +00:00
maxSummarySize = 1024 ,
2024-08-29 22:57:14 +00:00
callback = null ,
chunkSummaryPostProcessing = null ,
2024-09-17 09:00:46 +00:00
summaryMode = "SEQUENTIAL" ,
reformat = false
2024-09-04 12:16:41 +00:00
) {
2024-12-17 13:50:58 +00:00
console . log ( "Tokenizing incoming text:" )
2024-08-29 22:57:14 +00:00
let tk = await this . tokenize ( text ) ;
let prevLen = tk . length ;
let documentChunks = null ;
2024-12-05 15:39:26 +00:00
console . log ( ` Text size: ${ prevLen } / ${ maxSummarySize } ` )
2024-08-29 22:57:14 +00:00
while ( tk . length > maxSummarySize && ( documentChunks === null || documentChunks . length > 1 ) ) {
this . stepStart ( ` Compressing ${ docName } ... ` ) ;
2024-12-17 13:50:58 +00:00
let chunkSize = 1024 ;
if ( this . lollms . ctxSize ) {
chunkSize = Math . floor ( this . lollms . ctxSize * 0.6 ) ;
}
console . log ( "Chunk size:" , chunkSize )
2024-09-18 11:43:49 +00:00
documentChunks = await TextChunker . chunkText ( text , this . lollms , chunkSize , 0 , true ) ;
2024-12-05 15:39:26 +00:00
text = await this . summarizeChunks (
documentChunks ,
2024-08-29 22:57:14 +00:00
summaryInstruction ,
docName ,
answerStart ,
maxGenerationSize ,
callback ,
chunkSummaryPostProcessing ,
summaryMode
2024-12-05 15:39:26 +00:00
) ;
2024-08-29 22:57:14 +00:00
tk = await this . tokenize ( text ) ;
let dtkLn = prevLen - tk . length ;
prevLen = tk . length ;
this . step ( ` Current text size: ${ prevLen } , max summary size: ${ maxSummarySize } ` ) ;
this . stepEnd ( ` Compressing ${ docName } ... ` ) ;
if ( dtkLn <= 10 ) break ; // it is not summarizing
}
2024-09-17 09:00:46 +00:00
if ( reformat ) {
text = await this . lollms . generate (
[
this . lollms . system _message ( ) ,
` ${ text } ` ,
this . lollms . system _message ( ) ,
summaryInstruction ,
"Do not add any extra comments." ,
this . lollms . system _message ( ) + answerStart
] . join ( "\n" ) ,
) ;
}
2024-08-29 22:57:14 +00:00
return text ;
}
2024-12-02 07:55:13 +00:00
async smartDataExtraction (
2024-08-29 22:57:14 +00:00
text ,
dataExtractionInstruction = "summarize the current chunk." ,
finalTaskInstruction = "reformulate with better wording" ,
docName = "chunk" ,
answerStart = "" ,
maxGenerationSize = 3000 ,
2024-12-17 13:50:58 +00:00
maxSummarySize = 1024 ,
2024-08-29 22:57:14 +00:00
callback = null ,
chunkSummaryPostProcessing = null ,
summaryMode = "SEQUENTIAL"
2024-12-02 07:55:13 +00:00
) {
2024-08-29 22:57:14 +00:00
let tk = await this . tokenize ( text ) ;
let prevLen = tk . length ;
while ( tk . length > maxSummarySize ) {
2024-12-17 13:50:58 +00:00
let chunkSize = 1024 ;
if ( this . lollms . ctxSize ) {
chunkSize = Math . floor ( this . lollms . ctxSize * 0.6 ) ;
}
let documentChunks = await TextChunker . chunkText ( text , this . lollms , chunkSize , 0 , true ) ;
2024-12-05 15:39:26 +00:00
text = await this . summarizeChunks (
documentChunks ,
dataExtractionInstruction ,
2024-08-29 22:57:14 +00:00
docName ,
answerStart ,
maxGenerationSize ,
callback ,
chunkSummaryPostProcessing ,
summaryMode
2024-12-05 15:39:26 +00:00
) ;
2024-08-29 22:57:14 +00:00
tk = await this . tokenize ( text ) ;
let dtkLn = prevLen - tk . length ;
prevLen = tk . length ;
this . step ( ` Current text size: ${ prevLen } , max summary size: ${ maxSummarySize } ` ) ;
if ( dtkLn <= 10 ) break ; // it is not summarizing
}
this . stepStart ( "Rewriting ..." ) ;
2024-12-05 15:39:26 +00:00
text = await this . summarizeChunks (
[ text ] ,
finalTaskInstruction ,
2024-08-29 22:57:14 +00:00
docName ,
answerStart ,
maxGenerationSize ,
callback ,
2024-12-05 15:39:26 +00:00
chunkSummaryPostProcessing ,
summaryMode
) ;
2024-08-29 22:57:14 +00:00
this . stepEnd ( "Rewriting ..." ) ;
return text ;
}
2024-12-05 15:39:26 +00:00
async summarizeChunks (
2024-08-29 22:57:14 +00:00
chunks ,
summaryInstruction = "summarize the current chunk." ,
docName = "chunk" ,
answerStart = "" ,
maxGenerationSize = 3000 ,
callback = null ,
chunkSummaryPostProcessing = null ,
summaryMode = "SEQUENTIAL"
2024-12-05 15:39:26 +00:00
) {
2024-08-29 22:57:14 +00:00
if ( summaryMode === "SEQUENTIAL" ) {
let summary = "" ;
for ( let i = 0 ; i < chunks . length ; i ++ ) {
this . stepStart ( ` Summary of ${ docName } - Processing chunk: ${ i + 1 } / ${ chunks . length } ` ) ;
2024-12-17 14:50:21 +00:00
console . log ( ` chunk: ${ i } ` ) ;
console . log ( chunks [ i ] ) ;
2024-08-29 22:57:14 +00:00
summary = ` ${ answerStart } ` + await this . fastGen (
[
2024-12-17 15:04:11 +00:00
this . lollms . custom _system _message ( "Previous chunks summary" ) ,
2024-08-29 22:57:14 +00:00
` ${ summary } ` ,
2024-12-17 14:50:21 +00:00
this . lollms . custom _system _message ( "Current text chunk" ) ,
2024-08-29 22:57:14 +00:00
` ${ chunks [ i ] } ` ,
2024-12-17 15:04:11 +00:00
this . lollms . user _message ( ) ,
2024-08-29 22:57:14 +00:00
summaryInstruction ,
2024-12-17 15:04:11 +00:00
this . lollms . custom _system _message ( "important" ) ,
"Keep only information relevant to the context provided by the user" ,
"The output must keep information from the previous chunks summary and add the current chunk extracted information." ,
"Be precise and do not invent information that does not exist in the previous chunks summary or the current chunk." ,
"Do not add any extra comments and start your message directly by the summary." ,
this . lollms . ai _message ( ) + answerStart
2024-08-29 22:57:14 +00:00
] . join ( "\n" ) ,
maxGenerationSize ,
callback
) ;
if ( chunkSummaryPostProcessing ) {
summary = chunkSummaryPostProcessing ( summary ) ;
}
this . stepEnd ( ` Summary of ${ docName } - Processing chunk: ${ i + 1 } / ${ chunks . length } ` ) ;
}
return summary ;
} else {
let summaries = [ ] ;
for ( let i = 0 ; i < chunks . length ; i ++ ) {
this . stepStart ( ` Summary of ${ docName } - Processing chunk: ${ i + 1 } / ${ chunks . length } ` ) ;
let summary = ` ${ answerStart } ` + await this . fastGen (
[
` ${ this . lollms . system _message ( ) } Document_chunk [ ${ docName } ] ${ this . lollms . system _message ( ) } ` ,
` ${ chunks [ i ] } ` ,
` ${ this . lollms . system _message ( ) } ${ summaryInstruction } ` ,
"Answer directly with the summary with no extra comments." ,
` ${ this . lollms . system _message ( ) } summary ${ this . lollms . system _message ( ) } ` ,
` ${ answerStart } `
] . join ( "\n" ) ,
maxGenerationSize ,
callback
) ;
if ( chunkSummaryPostProcessing ) {
summary = chunkSummaryPostProcessing ( summary ) ;
}
summaries . push ( summary ) ;
this . stepEnd ( ` Summary of ${ docName } - Processing chunk: ${ i + 1 } / ${ chunks . length } ` ) ;
}
return summaries . join ( "\n" ) ;
}
}
async sequentialChunksSummary ( {
chunks ,
summaryInstruction = "summarize" ,
docName = "chunk" ,
answerStart = "" ,
maxGenerationSize = 3000 ,
callback = null ,
chunkSummaryPostProcessing = null
} ) {
let summaries = [ ] ;
for ( let i = 0 ; i < chunks . length ; i ++ ) {
let chunk1 = i < chunks . length - 1 ? chunks [ i + 1 ] : "" ;
let chunk = i > 0 ? summaries [ summaries . length - 1 ] : chunks [ i ] ;
this . stepStart ( ` Summary of ${ docName } - Processing chunk: ${ i + 1 } / ${ chunks . length } ` ) ;
let summary = ` ${ answerStart } ` + await this . fastGen (
[
` ${ this . lollms . system _message ( ) } Document_chunk: ${ docName } ${ this . lollms . system _message ( ) } ` ,
"Block1:" ,
` ${ chunk } ` ,
"Block2:" ,
` ${ chunk1 } ` ,
` ${ this . lollms . system _message ( ) } ${ summaryInstruction } ` ,
"Answer directly with the summary with no extra comments." ,
` ${ this . lollms . system _message ( ) } summary ${ this . lollms . system _message ( ) } ` ,
` ${ answerStart } `
] . join ( "\n" ) ,
maxGenerationSize ,
callback
) ;
if ( chunkSummaryPostProcessing ) {
summary = chunkSummaryPostProcessing ( summary ) ;
}
summaries . push ( summary ) ;
this . stepEnd ( ` Summary of ${ docName } - Processing chunk: ${ i + 1 } / ${ chunks . length } ` ) ;
}
return summaries . join ( "\n" ) ;
}
2024-09-12 09:47:39 +00:00
// Placeholder methods for step stepStart, stepEnd, fastGen
step ( message ) {
console . log ( message ) ;
}
2024-08-29 22:57:14 +00:00
stepStart ( message ) {
console . log ( message ) ;
}
stepEnd ( message ) {
console . log ( message ) ;
}
2024-07-29 22:02:24 +00:00
2024-08-29 22:57:14 +00:00
async fastGen ( prompt , maxGenerationSize , callback ) {
// Use the LollmsClient to generate text
const response = await this . lollms . generateText ( prompt ) ;
if ( callback ) callback ( response ) ;
return response ;
2024-08-03 22:07:41 +00:00
}
2024-08-29 22:57:14 +00:00
async yesNo ( question , context = "" , maxAnswerLength = 50 , conditioning = "" ) {
2024-08-03 22:07:41 +00:00
/ * *
* Analyzes the user prompt and answers whether it is asking to generate an image .
*
* @ param { string } question - The user ' s message .
* @ param { string } context - The context for the question .
* @ param { number } maxAnswerLength - The maximum length of the generated answer .
* @ param { string } conditioning - An optional system message to put at the beginning of the prompt .
* @ returns { boolean } True if the user prompt is asking to generate an image , False otherwise .
* /
return this . multichoiceQuestion ( question , [ "no" , "yes" ] , context , maxAnswerLength , conditioning ) > 0 ;
}
2024-08-29 22:57:14 +00:00
printPrompt ( prompt ) {
console . log ( prompt ) ;
}
async multichoiceQuestion ( question , possibleAnswers , context = "" , maxAnswerLength = 50 , conditioning = "" ) {
2024-08-03 22:07:41 +00:00
/ * *
* Interprets a multi - choice question from a user ' s response . This function expects only one choice as true .
* All other choices are considered false . If none are correct , returns - 1.
*
* @ param { string } question - The multi - choice question posed by the user .
* @ param { Array } possibleAnswers - A list containing all valid options for the chosen value .
* @ param { string } context - The context for the question .
* @ param { number } maxAnswerLength - Maximum string length allowed while interpreting the users ' responses .
* @ param { string } conditioning - An optional system message to put at the beginning of the prompt .
* @ returns { number } Index of the selected option within the possibleAnswers list . Or - 1 if there was no match found among any of them .
* /
const choices = possibleAnswers . map ( ( answer , index ) => ` ${ index } . ${ answer } ` ) . join ( "\n" ) ;
const elements = conditioning ? [ conditioning ] : [ ] ;
elements . push (
2024-08-29 22:57:14 +00:00
this . lollms . system _message ( ) ,
2024-08-03 22:07:41 +00:00
"Answer this multi choices question." ,
"Answer with an id from the possible answers." ,
"Do not answer with an id outside this possible answers."
) ;
if ( context ) {
elements . push (
2024-08-29 22:57:14 +00:00
this . lollms . custom _message ( "context" ) ,
2024-08-03 22:07:41 +00:00
context
) ;
}
elements . push (
2024-08-29 22:57:14 +00:00
this . lollms . custom _message ( ` question ` ) + ` $ ${ question } ` ,
this . lollms . custom _message ( ` possible answers ` ) ,
2024-08-03 22:07:41 +00:00
choices ,
2024-08-29 22:57:14 +00:00
this . lollms . custom _message ( "answer" )
2024-08-03 22:07:41 +00:00
) ;
const prompt = this . buildPrompt ( elements ) ;
2024-08-29 22:57:14 +00:00
let gen = await this . lollms . generate ( prompt , {
2024-08-03 22:07:41 +00:00
n _predict : maxAnswerLength ,
temperature : 0.1 ,
top _k : 50 ,
top _p : 0.9 ,
repeat _penalty : 1.0 ,
repeat _last _n : 50 ,
callback : this . sink
2024-08-29 22:57:14 +00:00
} )
gen = gen . trim ( ) . replace ( "" , "" ) . replace ( "" , "" ) ;
2024-08-03 22:07:41 +00:00
const selection = gen . trim ( ) . split ( " " ) [ 0 ] . replace ( "," , "" ) . replace ( "." , "" ) ;
this . printPrompt ( "Multi choice selection" , prompt + gen ) ;
try {
return parseInt ( selection , 10 ) ;
} catch ( error ) {
console . log ( "Model failed to answer the question" ) ;
return - 1 ;
}
}
buildPrompt ( promptParts , sacrificeId = - 1 , contextSize = null , minimumSpareContextSize = null ) {
/ * *
* Builds the prompt for code generation .
*
* @ param { Array < string > } promptParts - A list of strings representing the parts of the prompt .
* @ param { number } sacrificeId - The ID of the part to sacrifice .
* @ param { number } contextSize - The size of the context .
* @ param { number } minimumSpareContextSize - The minimum spare context size .
* @ returns { string } - The built prompt .
* /
if ( contextSize === null ) {
2024-08-29 22:57:14 +00:00
contextSize = this . lollms . ctxSize ;
2024-08-03 22:07:41 +00:00
}
if ( minimumSpareContextSize === null ) {
2024-08-29 22:57:14 +00:00
minimumSpareContextSize = this . lollms . minNPredict ;
2024-08-03 22:07:41 +00:00
}
if ( sacrificeId === - 1 || promptParts [ sacrificeId ] . length < 50 ) {
return promptParts . filter ( s => s !== "" ) . join ( "\n" ) ;
} else {
const partTokens = [ ] ;
let nbTokens = 0 ;
for ( let i = 0 ; i < promptParts . length ; i ++ ) {
const part = promptParts [ i ] ;
const tk = this . lollms . tokenize ( part ) ;
partTokens . push ( tk ) ;
if ( i !== sacrificeId ) {
nbTokens += tk . length ;
}
}
let sacrificeText = "" ;
if ( partTokens [ sacrificeId ] . length > 0 ) {
const sacrificeTk = partTokens [ sacrificeId ] ;
const tokensToKeep = sacrificeTk . slice ( - ( contextSize - nbTokens - minimumSpareContextSize ) ) ;
sacrificeText = this . lollms . detokenize ( tokensToKeep ) ;
}
promptParts [ sacrificeId ] = sacrificeText ;
return promptParts . filter ( s => s !== "" ) . join ( "\n" ) ;
}
}
2024-08-08 19:17:24 +00:00
/ * *
* Updates the given code based on the provided query string .
* The query string can contain two types of modifications :
* 1. FULL _REWRITE : Completely replaces the original code with the new code .
* 2. REPLACE : Replaces specific code snippets within the original code .
*
* @ param { string } originalCode - The original code to be updated .
* @ param { string } queryString - The string containing the update instructions .
* @ returns { object } - An object with the following properties :
* - updatedCode : The updated code .
* - modifications : An array of objects representing the changes made , each with properties 'oldCode' and 'newCode' .
* - hasQuery : A boolean indicating whether the queryString contained any valid queries .
* /
updateCode ( originalCode , queryString ) {
const queries = queryString . split ( '# REPLACE\n' ) ;
let updatedCode = originalCode ;
const modifications = [ ] ;
// Check if there's a FULL_REWRITE first
const fullRewriteStart = queryString . indexOf ( '# FULL_REWRITE' ) ;
if ( fullRewriteStart !== - 1 ) {
const newCode = queryString . slice ( fullRewriteStart + 14 ) . trim ( ) ;
updatedCode = newCode ;
modifications . push ( {
oldCode : originalCode ,
newCode
} ) ;
return {
updatedCode ,
modifications ,
hasQuery : true
} ;
}
2024-08-03 22:07:41 +00:00
2024-08-08 19:17:24 +00:00
if ( queries . length === 1 && queries [ 0 ] . trim ( ) === '' ) {
console . log ( "No queries detected" ) ;
return {
updatedCode ,
modifications : [ ] ,
hasQuery : false
} ;
}
for ( const query of queries ) {
if ( query . trim ( ) === '' ) continue ;
const originalCodeStart = query . indexOf ( '# ORIGINAL\n' ) + 11 ;
const originalCodeEnd = query . indexOf ( '\n# SET\n' ) ;
2024-08-09 12:21:42 +00:00
let oldCode = query . slice ( originalCodeStart , originalCodeEnd ) ;
2024-08-06 09:47:49 +00:00
2024-08-08 19:17:24 +00:00
const newCodeStart = query . indexOf ( '# SET\n' ) + 6 ;
const newCode = query . slice ( newCodeStart ) ;
const modification = {
oldCode : oldCode . trim ( ) ,
newCode : newCode . trim ( )
} ;
2024-08-09 12:21:42 +00:00
if ( oldCode == "<old_code>" ) {
oldCode = originalCode
}
console . log ( "oldCode:" )
console . log ( oldCode )
console . log ( "newCode:" )
console . log ( newCode )
console . log ( "Before update" , updatedCode ) ;
if ( oldCode === updatedCode ) {
console . log ( "Changing the whole content" )
updatedCode = newCode
}
else {
updatedCode = updatedCode . replace ( oldCode , newCode . trim ( ) ) ;
}
console . log ( "After update" , updatedCode ) ;
2024-08-08 19:17:24 +00:00
modifications . push ( modification ) ;
}
return {
updatedCode ,
modifications ,
hasQuery : true
} ;
}
}
2024-08-06 09:47:49 +00:00
class LOLLMSRAGClient {
2024-08-29 22:57:14 +00:00
constructor ( lc ) {
this . lc = lc ;
this . key = lc . service _key || this . generateRandomKey ( ) ;
console . log ( "Connecting to server with key:" , this . key ) ;
2024-08-06 09:47:49 +00:00
}
2024-08-29 22:57:14 +00:00
generateRandomKey ( ) {
// Generate a random key (UUID v4)
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' . replace ( /[xy]/g , function ( c ) {
const r = Math . random ( ) * 16 | 0 , v = c === 'x' ? r : ( r & 0x3 | 0x8 ) ;
return v . toString ( 16 ) ;
} ) ;
}
2024-08-06 09:47:49 +00:00
2024-08-29 22:57:14 +00:00
async request ( endpoint , method = 'GET' , body = null ) {
try {
2024-08-06 09:47:49 +00:00
const options = {
2024-08-29 22:57:14 +00:00
method ,
headers : {
'Content-Type' : 'application/json' ,
} ,
body : body ? JSON . stringify ( body ) : null ,
2024-08-06 09:47:49 +00:00
} ;
2024-12-08 23:29:36 +00:00
let response = "" ;
if ( this . lc . host _address != null ) {
response = await fetch ( ` ${ this . lc . host _address } ${ endpoint } ` , options ) ;
}
else {
response = await fetch ( ` ${ endpoint } ` , options ) ;
}
2024-08-29 22:57:14 +00:00
const data = await response . json ( ) ;
2024-08-06 09:47:49 +00:00
if ( ! response . ok ) {
2024-08-29 22:57:14 +00:00
throw new Error ( ` Error: ${ data . detail || response . statusText } ` ) ;
2024-08-06 09:47:49 +00:00
}
2024-08-29 22:57:14 +00:00
return data ;
} catch ( error ) {
console . error ( "Request failed:" , error ) ;
throw new Error ( ` Error: ${ error . message } ` ) ;
}
2024-08-06 09:47:49 +00:00
}
async addDocument ( title , content , path = "unknown" ) {
2024-08-29 22:57:14 +00:00
const document = { title , content , path , key : this . key } ;
return this . request ( '/add_document' , 'POST' , document ) ;
2024-08-06 09:47:49 +00:00
}
async removeDocument ( documentId ) {
2024-08-29 22:57:14 +00:00
const body = { key : this . key } ;
return this . request ( ` /remove_document/ ${ documentId } ` , 'POST' , body ) ;
2024-08-06 09:47:49 +00:00
}
async indexDatabase ( ) {
2024-08-29 22:57:14 +00:00
const body = { key : this . key } ;
console . log ( "Sending request to index database with body:" , body ) ;
2024-08-06 09:47:49 +00:00
2024-08-29 22:57:14 +00:00
try {
const response = await this . request ( '/index_database' , 'POST' , body , {
'Content-Type' : 'application/json'
} ) ;
console . log ( "Index database response:" , response ) ;
return response ;
} catch ( error ) {
console . error ( "Error indexing database:" , error ) ;
throw error ;
}
}
2024-08-06 09:47:49 +00:00
async search ( query ) {
2024-08-29 22:57:14 +00:00
const searchQuery = { query , key : this . key } ;
return this . request ( '/search' , 'POST' , searchQuery ) ;
2024-08-06 09:47:49 +00:00
}
async wipeDatabase ( ) {
2024-08-29 22:57:14 +00:00
const body = { key : this . key } ;
return this . request ( '/wipe_database' , 'DELETE' , body ) ;
2024-08-06 09:47:49 +00:00
}
}
2024-08-29 22:57:14 +00:00
2024-08-06 09:47:49 +00:00
// Example usage:
// const ragClient = new RAGClient('http://localhost:8000', 'your_bearer_token');
// ragClient.addDocument('My Title', 'This is the content of the document.')
// .then(response => console.log(response))
// .catch(error => console.error(error));