diff --git a/hfutils/index/fetch.py b/hfutils/index/fetch.py index 29e3811779..50f01c091c 100644 --- a/hfutils/index/fetch.py +++ b/hfutils/index/fetch.py @@ -470,7 +470,9 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st :type headers: Dict[str, str], optional :param endpoint: The Hugging Face API endpoint. :type endpoint: str, optional - :param force_download: Force download the file to destination path. Defualt to `False`. + :param force_download: Force download the file to destination path. + Defualt to `False`, downloading will be skipped if the local file + is fully matched with expected file. :type force_download: bool :param hf_token: The Hugging Face access token. :type hf_token: str, optional diff --git a/hfutils/index/local_fetch.py b/hfutils/index/local_fetch.py index c86ea1c01b..4343da7c13 100644 --- a/hfutils/index/local_fetch.py +++ b/hfutils/index/local_fetch.py @@ -190,7 +190,9 @@ def tar_file_download(archive_file: str, file_in_archive: str, local_file: str, :type idx_file: Optional[str] :param chunk_size: The size of chunks to read and write, in bytes. Default is 1MB. :type chunk_size: int - :param force_download: Force download the file to destination path. Defualt to `False`. + :param force_download: Force download the file to destination path. + Defualt to `False`, downloading will be skipped if the local file + is fully matched with expected file. :type force_download: bool :raises FileNotFoundError: If the specified file is not found in the archive. diff --git a/test/index/test_fetch.py b/test/index/test_fetch.py index 11ebbdfc9d..6581ef64bd 100644 --- a/test/index/test_fetch.py +++ b/test/index/test_fetch.py @@ -114,6 +114,30 @@ def test_hf_tar_file_download_small(self): ) file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json') + def test_hf_tar_file_download_small_exist(self): + with isolated_directory({ + '.meta.json': get_testfile('skin_mashu', '.meta.json') + }): + hf_tar_file_download( + repo_id='narugo/test_cos5t_tars', + archive_in_repo='mashu_skins.tar', + file_in_archive='.meta.json', + local_file='.meta.json' + ) + file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json') + + def test_hf_tar_file_download_small_replace(self): + with isolated_directory({ + '.meta.json': get_testfile('skin_mashu', '愚人节_奥特瑙斯.png') + }): + hf_tar_file_download( + repo_id='narugo/test_cos5t_tars', + archive_in_repo='mashu_skins.tar', + file_in_archive='.meta.json', + local_file='.meta.json' + ) + file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json') + def test_hf_tar_file_download_lfs(self): with isolated_directory(): hf_tar_file_download( @@ -124,6 +148,30 @@ def test_hf_tar_file_download_lfs(self): ) file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png') + def test_hf_tar_file_download_lfs_exist(self): + with isolated_directory({ + '愚人节_奥特瑙斯.png': get_testfile('skin_mashu', '愚人节_奥特瑙斯.png') + }): + hf_tar_file_download( + repo_id='narugo/test_cos5t_tars', + archive_in_repo='mashu_skins.tar', + file_in_archive='./愚人节_奥特瑙斯.png', + local_file='愚人节_奥特瑙斯.png' + ) + file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png') + + def test_hf_tar_file_download_lfs_replace(self): + with isolated_directory({ + '愚人节_奥特瑙斯.png': get_testfile('skin_mashu', '.meta.json') + }): + hf_tar_file_download( + repo_id='narugo/test_cos5t_tars', + archive_in_repo='mashu_skins.tar', + file_in_archive='./愚人节_奥特瑙斯.png', + local_file='愚人节_奥特瑙斯.png' + ) + file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png') + def test_hf_tar_file_download_not_found(self): with isolated_directory(), pytest.raises(FileNotFoundError): hf_tar_file_download( diff --git a/test/index/test_local_fetch.py b/test/index/test_local_fetch.py index 92f12a07d4..45dc1e4603 100644 --- a/test/index/test_local_fetch.py +++ b/test/index/test_local_fetch.py @@ -100,6 +100,28 @@ def test_tar_file_download_small(self, local_narugo_test_cos5t_tars): ) file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json') + def test_tar_file_download_small_exist(self, local_narugo_test_cos5t_tars): + with isolated_directory({ + '.meta.json': get_testfile('skin_mashu', '.meta.json') + }): + tar_file_download( + archive_file=os.path.join(local_narugo_test_cos5t_tars, 'mashu_skins.tar'), + file_in_archive='.meta.json', + local_file='.meta.json' + ) + file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json') + + def test_tar_file_download_small_replace(self, local_narugo_test_cos5t_tars): + with isolated_directory({ + '.meta.json': get_testfile('skin_mashu', '愚人节_奥特瑙斯.png') + }): + tar_file_download( + archive_file=os.path.join(local_narugo_test_cos5t_tars, 'mashu_skins.tar'), + file_in_archive='.meta.json', + local_file='.meta.json' + ) + file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json') + def test_tar_file_download_lfs(self, local_narugo_test_cos5t_tars): with isolated_directory(): tar_file_download( @@ -109,6 +131,28 @@ def test_tar_file_download_lfs(self, local_narugo_test_cos5t_tars): ) file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png') + def test_tar_file_download_lfs_exist(self, local_narugo_test_cos5t_tars): + with isolated_directory({ + '愚人节_奥特瑙斯.png': get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), + }): + tar_file_download( + archive_file=os.path.join(local_narugo_test_cos5t_tars, 'mashu_skins.tar'), + file_in_archive='./愚人节_奥特瑙斯.png', + local_file='愚人节_奥特瑙斯.png', + ) + file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png') + + def test_tar_file_download_lfs_replace(self, local_narugo_test_cos5t_tars): + with isolated_directory({ + '愚人节_奥特瑙斯.png': get_testfile('skin_mashu', '.meta.json'), + }): + tar_file_download( + archive_file=os.path.join(local_narugo_test_cos5t_tars, 'mashu_skins.tar'), + file_in_archive='./愚人节_奥特瑙斯.png', + local_file='愚人节_奥特瑙斯.png', + ) + file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png') + def test_tar_file_download_not_found(self, local_narugo_test_cos5t_tars): with isolated_directory(), pytest.raises(FileNotFoundError): tar_file_download(