ExternalVendorCode/lollms-webui
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms-webui.git synced 2024-12-24 06:36:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

added a convert script with tokenizer

Browse Source
This commit is contained in:
Saifeddine ALOUI 2023-04-19 11:09:53 +02:00
parent 512e089b17
commit 8f5326bd68
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scripts/convert model_with_tokenizer.bat Normal file
View File

@ -0,0 +1,13 @@
@echo off
@echo off
set filename=../models/%1
set newname=../models/%1.original
echo %modelPath%
echo Converting the model to the new format...
if not exist tmp\llama.cpp git clone https://github.com/ggerganov/llama.cpp.git tmp\llama.cpp
cd tmp\llama.cpp
cd ../..
move /y "%filename%" "%newname%"
echo Converting ...
python tmp\llama.cpp\convert.py "%newname%" --outfile "%filename%" --vocab-dir %2