talk-llama : sync llama.cpp

This commit is contained in:
Georgi Gerganov
2024-07-08 14:14:17 +03:00
parent d3f6c34976
commit dbf9c15e30
7 changed files with 3226 additions and 635 deletions

View File

@ -48,6 +48,7 @@ struct codepoint_flags {
std::string unicode_cpt_to_utf8(uint32_t cp);
uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);
std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);
std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts);