Skip to content

Commit

Permalink
refactor: Removed redundant hash check
Browse files Browse the repository at this point in the history
fix: Fix ci
  • Loading branch information
hh-space-invader committed Jan 8, 2025
1 parent ddd765f commit 6992790
Showing 1 changed file with 2 additions and 15 deletions.
17 changes: 2 additions & 15 deletions fastembed/common/model_management.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,18 +116,6 @@ def download_files_from_huggingface(
Path: The path to the model directory.
"""

def _should_update_metadata(
model_dir: Path, stored_metadata: dict[str, Any], model_file: str
) -> bool:
model_path = model_dir / model_file
if not model_path.exists():
return True
current_hash = _get_file_hash(model_path)
return (
model_file not in stored_metadata
or stored_metadata[model_file]["hash"] != current_hash
)

def _get_file_hash(file_path: Path) -> str:
sha256_hash = hashlib.sha256()
with file_path.open("rb") as f:
Expand Down Expand Up @@ -167,7 +155,7 @@ def _save_file_metadata(model_dir: Path) -> None:
"preprocessor_config.json",
]

model_file = next((file for file in extra_patterns if file.endswith(".onnx")), "")
model_file = next((file for file in extra_patterns if file.endswith((".onnx"))), "")
allow_patterns.extend(extra_patterns)

snapshot_dir = Path(cache_dir) / f"models--{hf_source_repo.replace('/', '--')}"
Expand All @@ -176,8 +164,7 @@ def _save_file_metadata(model_dir: Path) -> None:
if snapshot_dir.exists() and metadata_file.exists():
stored_metadata = json.loads(metadata_file.read_text())
if _verify_files_from_metadata(snapshot_dir, stored_metadata):
if not _should_update_metadata(snapshot_dir, stored_metadata, model_file):
disable_progress_bars()
disable_progress_bars()

result = snapshot_download(
repo_id=hf_source_repo,
Expand Down

0 comments on commit 6992790

Please sign in to comment.