talk-llama : sync llama.cpp

This commit is contained in:
Georgi Gerganov
2024-08-08 14:16:50 +03:00
parent 5b7979a1e6
commit 22058f2dbc
14 changed files with 5066 additions and 4423 deletions

View File

@ -4,6 +4,8 @@
#include <string>
#include <vector>
// TODO: prefix all symbols with "llama_"
struct codepoint_flags {
enum {
UNDEFINED = 0x0001,
@ -46,6 +48,7 @@ struct codepoint_flags {
}
};
size_t unicode_len_utf8(char src);
std::string unicode_cpt_to_utf8(uint32_t cp);
uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);