2023-07-14 23:19:43 +00:00
|
|
|
syntax = "proto3";
|
|
|
|
|
|
|
|
option go_package = "github.com/go-skynet/LocalAI/pkg/grpc/proto";
|
|
|
|
option java_multiple_files = true;
|
2023-07-14 23:19:43 +00:00
|
|
|
option java_package = "io.skynet.localai.backend";
|
|
|
|
option java_outer_classname = "LocalAIBackend";
|
2023-07-14 23:19:43 +00:00
|
|
|
|
2023-07-14 23:19:43 +00:00
|
|
|
package backend;
|
2023-07-14 23:19:43 +00:00
|
|
|
|
2023-07-14 23:19:43 +00:00
|
|
|
service Backend {
|
2023-07-14 23:19:43 +00:00
|
|
|
rpc Health(HealthMessage) returns (Reply) {}
|
|
|
|
rpc Predict(PredictOptions) returns (Reply) {}
|
|
|
|
rpc LoadModel(ModelOptions) returns (Result) {}
|
|
|
|
rpc PredictStream(PredictOptions) returns (stream Reply) {}
|
2023-07-14 23:19:43 +00:00
|
|
|
rpc Embedding(PredictOptions) returns (EmbeddingResult) {}
|
2023-07-14 23:19:43 +00:00
|
|
|
rpc GenerateImage(GenerateImageRequest) returns (Result) {}
|
|
|
|
rpc AudioTranscription(TranscriptRequest) returns (TranscriptResult) {}
|
|
|
|
rpc TTS(TTSRequest) returns (Result) {}
|
2024-08-24 00:20:28 +00:00
|
|
|
rpc SoundGeneration(SoundGenerationRequest) returns (Result) {}
|
2023-08-18 19:23:14 +00:00
|
|
|
rpc TokenizeString(PredictOptions) returns (TokenizationResponse) {}
|
|
|
|
rpc Status(HealthMessage) returns (StatusResponse) {}
|
2024-03-22 20:14:04 +00:00
|
|
|
|
|
|
|
rpc StoresSet(StoresSetOptions) returns (Result) {}
|
|
|
|
rpc StoresDelete(StoresDeleteOptions) returns (Result) {}
|
|
|
|
rpc StoresGet(StoresGetOptions) returns (StoresGetResult) {}
|
|
|
|
rpc StoresFind(StoresFindOptions) returns (StoresFindResult) {}
|
2024-04-24 22:19:02 +00:00
|
|
|
|
|
|
|
rpc Rerank(RerankRequest) returns (RerankResult) {}
|
2024-10-01 12:41:20 +00:00
|
|
|
|
|
|
|
rpc GetMetrics(MetricsRequest) returns (MetricsResponse);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Define the empty request
|
|
|
|
message MetricsRequest {}
|
|
|
|
|
|
|
|
message MetricsResponse {
|
|
|
|
int32 slot_id = 1;
|
|
|
|
string prompt_json_for_slot = 2; // Stores the prompt as a JSON string.
|
|
|
|
float tokens_per_second = 3;
|
|
|
|
int32 tokens_generated = 4;
|
|
|
|
int32 prompt_tokens_processed = 5;
|
2024-04-24 22:19:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
message RerankRequest {
|
|
|
|
string query = 1;
|
|
|
|
repeated string documents = 2;
|
|
|
|
int32 top_n = 3;
|
|
|
|
}
|
|
|
|
|
|
|
|
message RerankResult {
|
|
|
|
Usage usage = 1;
|
|
|
|
repeated DocumentResult results = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
message Usage {
|
|
|
|
int32 total_tokens = 1;
|
|
|
|
int32 prompt_tokens = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
message DocumentResult {
|
|
|
|
int32 index = 1;
|
|
|
|
string text = 2;
|
|
|
|
float relevance_score = 3;
|
2024-03-22 20:14:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
message StoresKey {
|
|
|
|
repeated float Floats = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
message StoresValue {
|
|
|
|
bytes Bytes = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
message StoresSetOptions {
|
|
|
|
repeated StoresKey Keys = 1;
|
|
|
|
repeated StoresValue Values = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
message StoresDeleteOptions {
|
|
|
|
repeated StoresKey Keys = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
message StoresGetOptions {
|
|
|
|
repeated StoresKey Keys = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
message StoresGetResult {
|
|
|
|
repeated StoresKey Keys = 1;
|
|
|
|
repeated StoresValue Values = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
message StoresFindOptions {
|
|
|
|
StoresKey Key = 1;
|
|
|
|
int32 TopK = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
message StoresFindResult {
|
|
|
|
repeated StoresKey Keys = 1;
|
|
|
|
repeated StoresValue Values = 2;
|
|
|
|
repeated float Similarities = 3;
|
2023-07-14 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
message HealthMessage {}
|
|
|
|
|
|
|
|
// The request message containing the user's name.
|
|
|
|
message PredictOptions {
|
|
|
|
string Prompt = 1;
|
|
|
|
int32 Seed = 2;
|
|
|
|
int32 Threads = 3;
|
|
|
|
int32 Tokens = 4;
|
|
|
|
int32 TopK = 5;
|
|
|
|
int32 Repeat = 6;
|
|
|
|
int32 Batch = 7;
|
|
|
|
int32 NKeep = 8;
|
|
|
|
float Temperature = 9;
|
|
|
|
float Penalty = 10;
|
|
|
|
bool F16KV = 11;
|
|
|
|
bool DebugMode = 12;
|
|
|
|
repeated string StopPrompts = 13;
|
|
|
|
bool IgnoreEOS = 14;
|
|
|
|
float TailFreeSamplingZ = 15;
|
|
|
|
float TypicalP = 16;
|
|
|
|
float FrequencyPenalty = 17;
|
|
|
|
float PresencePenalty = 18;
|
|
|
|
int32 Mirostat = 19;
|
|
|
|
float MirostatETA = 20;
|
|
|
|
float MirostatTAU = 21;
|
|
|
|
bool PenalizeNL = 22;
|
|
|
|
string LogitBias = 23;
|
|
|
|
bool MLock = 25;
|
|
|
|
bool MMap = 26;
|
|
|
|
bool PromptCacheAll = 27;
|
|
|
|
bool PromptCacheRO = 28;
|
|
|
|
string Grammar = 29;
|
|
|
|
string MainGPU = 30;
|
|
|
|
string TensorSplit = 31;
|
|
|
|
float TopP = 32;
|
|
|
|
string PromptCachePath = 33;
|
|
|
|
bool Debug = 34;
|
2023-07-14 23:19:43 +00:00
|
|
|
repeated int32 EmbeddingTokens = 35;
|
|
|
|
string Embeddings = 36;
|
2023-07-25 17:05:27 +00:00
|
|
|
float RopeFreqBase = 37;
|
|
|
|
float RopeFreqScale = 38;
|
|
|
|
float NegativePromptScale = 39;
|
|
|
|
string NegativePrompt = 40;
|
2023-09-14 15:44:16 +00:00
|
|
|
int32 NDraft = 41;
|
2023-11-11 12:14:59 +00:00
|
|
|
repeated string Images = 42;
|
2024-04-11 17:20:22 +00:00
|
|
|
bool UseTokenizerTemplate = 43;
|
|
|
|
repeated Message Messages = 44;
|
2024-09-19 09:21:59 +00:00
|
|
|
repeated string Videos = 45;
|
2024-09-19 10:26:53 +00:00
|
|
|
repeated string Audios = 46;
|
2024-09-28 15:23:56 +00:00
|
|
|
string CorrelationId = 47;
|
2023-07-14 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// The response message containing the result
|
|
|
|
message Reply {
|
2023-07-27 16:41:04 +00:00
|
|
|
bytes message = 1;
|
2024-04-15 17:47:11 +00:00
|
|
|
int32 tokens = 2;
|
|
|
|
int32 prompt_tokens = 3;
|
2023-07-14 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
message ModelOptions {
|
|
|
|
string Model = 1;
|
|
|
|
int32 ContextSize = 2;
|
|
|
|
int32 Seed = 3;
|
|
|
|
int32 NBatch = 4;
|
|
|
|
bool F16Memory = 5;
|
|
|
|
bool MLock = 6;
|
|
|
|
bool MMap = 7;
|
|
|
|
bool VocabOnly = 8;
|
|
|
|
bool LowVRAM = 9;
|
|
|
|
bool Embeddings = 10;
|
|
|
|
bool NUMA = 11;
|
|
|
|
int32 NGPULayers = 12;
|
|
|
|
string MainGPU = 13;
|
|
|
|
string TensorSplit = 14;
|
2023-07-14 23:19:43 +00:00
|
|
|
int32 Threads = 15;
|
|
|
|
string LibrarySearchPath = 16;
|
2023-07-27 19:56:05 +00:00
|
|
|
float RopeFreqBase = 17;
|
|
|
|
float RopeFreqScale = 18;
|
2023-08-02 22:51:08 +00:00
|
|
|
float RMSNormEps = 19;
|
|
|
|
int32 NGQA = 20;
|
2023-08-07 20:39:10 +00:00
|
|
|
string ModelFile = 21;
|
|
|
|
|
|
|
|
// AutoGPTQ
|
|
|
|
string Device = 22;
|
|
|
|
bool UseTriton = 23;
|
|
|
|
string ModelBaseName = 24;
|
2023-08-07 23:10:05 +00:00
|
|
|
bool UseFastTokenizer = 25;
|
2023-08-09 06:38:51 +00:00
|
|
|
|
|
|
|
// Diffusers
|
|
|
|
string PipelineType = 26;
|
|
|
|
string SchedulerType = 27;
|
|
|
|
bool CUDA = 28;
|
2023-08-15 23:11:42 +00:00
|
|
|
float CFGScale = 29;
|
2023-08-17 21:38:59 +00:00
|
|
|
bool IMG2IMG = 30;
|
|
|
|
string CLIPModel = 31;
|
|
|
|
string CLIPSubfolder = 32;
|
|
|
|
int32 CLIPSkip = 33;
|
2023-12-13 18:20:22 +00:00
|
|
|
string ControlNet = 48;
|
2023-08-22 16:48:06 +00:00
|
|
|
|
|
|
|
string Tokenizer = 34;
|
2023-08-25 19:58:46 +00:00
|
|
|
|
|
|
|
// LLM (llama.cpp)
|
|
|
|
string LoraBase = 35;
|
|
|
|
string LoraAdapter = 36;
|
2023-11-11 17:40:48 +00:00
|
|
|
float LoraScale = 42;
|
|
|
|
|
2023-08-25 19:58:46 +00:00
|
|
|
bool NoMulMatQ = 37;
|
2023-09-14 15:44:16 +00:00
|
|
|
string DraftModel = 39;
|
2024-03-22 20:14:04 +00:00
|
|
|
|
2023-09-04 17:25:23 +00:00
|
|
|
string AudioPath = 38;
|
2023-09-22 13:52:38 +00:00
|
|
|
|
|
|
|
// vllm
|
|
|
|
string Quantization = 40;
|
2024-03-01 21:48:53 +00:00
|
|
|
float GPUMemoryUtilization = 50;
|
|
|
|
bool TrustRemoteCode = 51;
|
|
|
|
bool EnforceEager = 52;
|
|
|
|
int32 SwapSpace = 53;
|
|
|
|
int32 MaxModelLen = 54;
|
2024-04-20 14:37:02 +00:00
|
|
|
int32 TensorParallelSize = 55;
|
2024-10-23 13:46:06 +00:00
|
|
|
string LoadFormat = 58;
|
2023-11-11 12:14:59 +00:00
|
|
|
|
|
|
|
string MMProj = 41;
|
2023-11-11 17:40:48 +00:00
|
|
|
|
|
|
|
string RopeScaling = 43;
|
|
|
|
float YarnExtFactor = 44;
|
|
|
|
float YarnAttnFactor = 45;
|
|
|
|
float YarnBetaFast = 46;
|
|
|
|
float YarnBetaSlow = 47;
|
2024-01-25 23:13:21 +00:00
|
|
|
|
|
|
|
string Type = 49;
|
2024-05-13 17:07:51 +00:00
|
|
|
|
|
|
|
bool FlashAttention = 56;
|
|
|
|
bool NoKVOffload = 57;
|
2024-10-31 11:12:22 +00:00
|
|
|
|
|
|
|
string ModelPath = 59;
|
2024-11-05 14:14:33 +00:00
|
|
|
|
|
|
|
repeated string LoraAdapters = 60;
|
|
|
|
repeated float LoraScales = 61;
|
2023-07-14 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
message Result {
|
|
|
|
string message = 1;
|
|
|
|
bool success = 2;
|
2023-07-14 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
message EmbeddingResult {
|
|
|
|
repeated float embeddings = 1;
|
2023-07-14 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
message TranscriptRequest {
|
|
|
|
string dst = 2;
|
|
|
|
string language = 3;
|
|
|
|
uint32 threads = 4;
|
2024-06-24 17:21:22 +00:00
|
|
|
bool translate = 5;
|
2023-07-14 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
message TranscriptResult {
|
|
|
|
repeated TranscriptSegment segments = 1;
|
|
|
|
string text = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
message TranscriptSegment {
|
|
|
|
int32 id = 1;
|
|
|
|
int64 start = 2;
|
|
|
|
int64 end = 3;
|
|
|
|
string text = 4;
|
|
|
|
repeated int32 tokens = 5;
|
|
|
|
}
|
|
|
|
|
|
|
|
message GenerateImageRequest {
|
|
|
|
int32 height = 1;
|
|
|
|
int32 width = 2;
|
|
|
|
int32 mode = 3;
|
|
|
|
int32 step = 4;
|
|
|
|
int32 seed = 5;
|
|
|
|
string positive_prompt = 6;
|
|
|
|
string negative_prompt = 7;
|
|
|
|
string dst = 8;
|
2023-08-17 21:38:59 +00:00
|
|
|
string src = 9;
|
2023-08-14 21:12:00 +00:00
|
|
|
|
|
|
|
// Diffusers
|
2023-08-17 21:38:59 +00:00
|
|
|
string EnableParameters = 10;
|
|
|
|
int32 CLIPSkip = 11;
|
2023-07-14 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
message TTSRequest {
|
|
|
|
string text = 1;
|
|
|
|
string model = 2;
|
|
|
|
string dst = 3;
|
2024-03-14 22:08:34 +00:00
|
|
|
string voice = 4;
|
2024-06-01 18:26:27 +00:00
|
|
|
optional string language = 5;
|
2023-07-14 23:19:43 +00:00
|
|
|
}
|
2023-08-18 19:23:14 +00:00
|
|
|
|
2024-08-24 00:20:28 +00:00
|
|
|
message SoundGenerationRequest {
|
|
|
|
string text = 1;
|
|
|
|
string model = 2;
|
|
|
|
string dst = 3;
|
|
|
|
optional float duration = 4;
|
|
|
|
optional float temperature = 5;
|
|
|
|
optional bool sample = 6;
|
|
|
|
optional string src = 7;
|
|
|
|
optional int32 src_divisor = 8;
|
|
|
|
}
|
|
|
|
|
2023-08-18 19:23:14 +00:00
|
|
|
message TokenizationResponse {
|
|
|
|
int32 length = 1;
|
|
|
|
repeated int32 tokens = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
message MemoryUsageData {
|
|
|
|
uint64 total = 1;
|
|
|
|
map<string, uint64> breakdown = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
message StatusResponse {
|
|
|
|
enum State {
|
|
|
|
UNINITIALIZED = 0;
|
|
|
|
BUSY = 1;
|
|
|
|
READY = 2;
|
|
|
|
ERROR = -1;
|
|
|
|
}
|
|
|
|
State state = 1;
|
|
|
|
MemoryUsageData memory = 2;
|
2024-03-22 20:14:04 +00:00
|
|
|
}
|
2024-04-11 17:20:22 +00:00
|
|
|
|
|
|
|
message Message {
|
|
|
|
string role = 1;
|
|
|
|
string content = 2;
|
|
|
|
}
|