ggml : sync latest ggml

- New Q4 and Q5 formats
- Various improvements
This commit is contained in:
Georgi Gerganov
2023-05-14 18:04:23 +03:00
parent d652cf12ec
commit e693074aa6
10 changed files with 4550 additions and 2140 deletions

View File

@ -53,6 +53,9 @@ struct gpt_vocab {
std::map<token, id> token_to_id;
std::map<id, token> id_to_token;
std::vector<std::string> special_tokens;
void add_special_token(const std::string & token);
};
// poor-man's JSON parsing