ExternalVendorCode/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-03 01:03:15 +00:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity

ci(checksum_checker): do get sha from hf API when available (#2380)

Browse Source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto 2024-05-22 23:51:02 +02:00 committed by GitHub
parent 371d0cc1f7
commit 21a12c2cdd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 27 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.github/checksum_checker.sh vendored
View File

@ -16,7 +16,7 @@ function check_and_update_checksum() {
# Download the file and calculate new checksum using Python # Download the file and calculate new checksum using Python
new_checksum=$(python3 -c " new_checksum=$(python3 -c "
import hashlib import hashlib
from huggingface_hub import hf_hub_download from huggingface_hub import hf_hub_download, get_paths_info
import requests import requests
import sys import sys
import os import os
@ -46,14 +46,26 @@ def calculate_sha256(file_path):
download_type, repo_id_or_url = parse_uri(uri) download_type, repo_id_or_url = parse_uri(uri)
new_checksum = None
# Decide download method based on URI type # Decide download method based on URI type
if download_type == 'huggingface': if download_type == 'huggingface':
# Use HF API to pull sha
for file in get_paths_info(repo_id_or_url, [file_name], repo_type='model'):
try:
new_checksum = file.lfs.sha256
break
except Exception as e:
print(f'Error from Hugging Face Hub: {str(e)}', file=sys.stderr)
sys.exit(2)
if new_checksum is None:
try: try:
file_path = hf_hub_download(repo_id=repo_id_or_url, filename=file_name) file_path = hf_hub_download(repo_id=repo_id_or_url, filename=file_name)
except Exception as e: except Exception as e:
print(f'Error from Hugging Face Hub: {str(e)}', file=sys.stderr) print(f'Error from Hugging Face Hub: {str(e)}', file=sys.stderr)
sys.exit(2) sys.exit(2)
else: else:
print(f'Downloading file from {repo_id_or_url}')
response = requests.get(repo_id_or_url) response = requests.get(repo_id_or_url)
if response.status_code == 200: if response.status_code == 200:
with open(file_name, 'wb') as f: with open(file_name, 'wb') as f:
@ -66,9 +78,13 @@ else:
print(f'Error downloading file: {response.status_code}', file=sys.stderr) print(f'Error downloading file: {response.status_code}', file=sys.stderr)
sys.exit(1) sys.exit(1)
print(calculate_sha256(file_path)) if new_checksum is None:
# Clean up the downloaded file new_checksum = calculate_sha256(file_path)
os.remove(file_path) print(new_checksum)
os.remove(file_path)
else:
print(new_checksum)
") ")
if [[ "$new_checksum" == "" ]]; then if [[ "$new_checksum" == "" ]]; then

4
gallery/index.yaml
View File

@ -1062,10 +1062,10 @@
files: files:
- filename: Bunny-Llama-3-8B-Q4_K_M-mmproj.gguf - filename: Bunny-Llama-3-8B-Q4_K_M-mmproj.gguf
sha256: 96d033387a91e56cf97fa5d60e02c0128ce07c8fa83aaaefb74ec40541615ea5 sha256: 96d033387a91e56cf97fa5d60e02c0128ce07c8fa83aaaefb74ec40541615ea5
uri: huggingace://BAAI/Bunny-Llama-3-8B-V-gguf/mmproj-model-f16.gguf uri: huggingface://BAAI/Bunny-Llama-3-8B-V-gguf/mmproj-model-f16.gguf
- filename: Bunny-Llama-3-8B-Q4_K_M.gguf - filename: Bunny-Llama-3-8B-Q4_K_M.gguf
sha256: 88f0a61f947dbf129943328be7262ae82e3a582a0c75e53544b07f70355a7c30 sha256: 88f0a61f947dbf129943328be7262ae82e3a582a0c75e53544b07f70355a7c30
uri: huggingace://BAAI/Bunny-Llama-3-8B-V-gguf/ggml-model-Q4_K_M.gguf uri: huggingface://BAAI/Bunny-Llama-3-8B-V-gguf/ggml-model-Q4_K_M.gguf
- !!merge <<: *llama3 - !!merge <<: *llama3
name: "llava-llama-3-8b-v1_1" name: "llava-llama-3-8b-v1_1"
description: | description: |