Skip to content

Commit

Permalink
dev(narugo): add unittest
Browse files Browse the repository at this point in the history
  • Loading branch information
@narugo1992
narugo1992 committed Sep 1, 2024
1 parent 6749c41 commit 9ee1475
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 2 deletions.
4 changes: 3 additions & 1 deletion hfutils/index/fetch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,9 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
:type headers: Dict[str, str], optional
:param endpoint: The Hugging Face API endpoint.
:type endpoint: str, optional
:param force_download: Force download the file to destination path. Defualt to `False`.
:param force_download: Force download the file to destination path.
Defualt to `False`, downloading will be skipped if the local file
is fully matched with expected file.
:type force_download: bool
:param hf_token: The Hugging Face access token.
:type hf_token: str, optional
Expand Down
4 changes: 3 additions & 1 deletion hfutils/index/local_fetch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,9 @@ def tar_file_download(archive_file: str, file_in_archive: str, local_file: str,
:type idx_file: Optional[str]
:param chunk_size: The size of chunks to read and write, in bytes. Default is 1MB.
:type chunk_size: int
:param force_download: Force download the file to destination path. Defualt to `False`.
:param force_download: Force download the file to destination path.
Defualt to `False`, downloading will be skipped if the local file
is fully matched with expected file.
:type force_download: bool
:raises FileNotFoundError: If the specified file is not found in the archive.
Expand Down
48 changes: 48 additions & 0 deletions test/index/test_fetch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,30 @@ def test_hf_tar_file_download_small(self):
)
file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json')

def test_hf_tar_file_download_small_exist(self):
with isolated_directory({
'.meta.json': get_testfile('skin_mashu', '.meta.json')
}):
hf_tar_file_download(
repo_id='narugo/test_cos5t_tars',
archive_in_repo='mashu_skins.tar',
file_in_archive='.meta.json',
local_file='.meta.json'
)
file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json')

def test_hf_tar_file_download_small_replace(self):
with isolated_directory({
'.meta.json': get_testfile('skin_mashu', '愚人节_奥特瑙斯.png')
}):
hf_tar_file_download(
repo_id='narugo/test_cos5t_tars',
archive_in_repo='mashu_skins.tar',
file_in_archive='.meta.json',
local_file='.meta.json'
)
file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json')

def test_hf_tar_file_download_lfs(self):
with isolated_directory():
hf_tar_file_download(
Expand All @@ -124,6 +148,30 @@ def test_hf_tar_file_download_lfs(self):
)
file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png')

def test_hf_tar_file_download_lfs_exist(self):
with isolated_directory({
'愚人节_奥特瑙斯.png': get_testfile('skin_mashu', '愚人节_奥特瑙斯.png')
}):
hf_tar_file_download(
repo_id='narugo/test_cos5t_tars',
archive_in_repo='mashu_skins.tar',
file_in_archive='./愚人节_奥特瑙斯.png',
local_file='愚人节_奥特瑙斯.png'
)
file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png')

def test_hf_tar_file_download_lfs_replace(self):
with isolated_directory({
'愚人节_奥特瑙斯.png': get_testfile('skin_mashu', '.meta.json')
}):
hf_tar_file_download(
repo_id='narugo/test_cos5t_tars',
archive_in_repo='mashu_skins.tar',
file_in_archive='./愚人节_奥特瑙斯.png',
local_file='愚人节_奥特瑙斯.png'
)
file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png')

def test_hf_tar_file_download_not_found(self):
with isolated_directory(), pytest.raises(FileNotFoundError):
hf_tar_file_download(
Expand Down
44 changes: 44 additions & 0 deletions test/index/test_local_fetch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,28 @@ def test_tar_file_download_small(self, local_narugo_test_cos5t_tars):
)
file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json')

def test_tar_file_download_small_exist(self, local_narugo_test_cos5t_tars):
with isolated_directory({
'.meta.json': get_testfile('skin_mashu', '.meta.json')
}):
tar_file_download(
archive_file=os.path.join(local_narugo_test_cos5t_tars, 'mashu_skins.tar'),
file_in_archive='.meta.json',
local_file='.meta.json'
)
file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json')

def test_tar_file_download_small_replace(self, local_narugo_test_cos5t_tars):
with isolated_directory({
'.meta.json': get_testfile('skin_mashu', '愚人节_奥特瑙斯.png')
}):
tar_file_download(
archive_file=os.path.join(local_narugo_test_cos5t_tars, 'mashu_skins.tar'),
file_in_archive='.meta.json',
local_file='.meta.json'
)
file_compare(get_testfile('skin_mashu', '.meta.json'), '.meta.json')

def test_tar_file_download_lfs(self, local_narugo_test_cos5t_tars):
with isolated_directory():
tar_file_download(
Expand All @@ -109,6 +131,28 @@ def test_tar_file_download_lfs(self, local_narugo_test_cos5t_tars):
)
file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png')

def test_tar_file_download_lfs_exist(self, local_narugo_test_cos5t_tars):
with isolated_directory({
'愚人节_奥特瑙斯.png': get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'),
}):
tar_file_download(
archive_file=os.path.join(local_narugo_test_cos5t_tars, 'mashu_skins.tar'),
file_in_archive='./愚人节_奥特瑙斯.png',
local_file='愚人节_奥特瑙斯.png',
)
file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png')

def test_tar_file_download_lfs_replace(self, local_narugo_test_cos5t_tars):
with isolated_directory({
'愚人节_奥特瑙斯.png': get_testfile('skin_mashu', '.meta.json'),
}):
tar_file_download(
archive_file=os.path.join(local_narugo_test_cos5t_tars, 'mashu_skins.tar'),
file_in_archive='./愚人节_奥特瑙斯.png',
local_file='愚人节_奥特瑙斯.png',
)
file_compare(get_testfile('skin_mashu', '愚人节_奥特瑙斯.png'), '愚人节_奥特瑙斯.png')

def test_tar_file_download_not_found(self, local_narugo_test_cos5t_tars):
with isolated_directory(), pytest.raises(FileNotFoundError):
tar_file_download(
Expand Down

0 comments on commit 9ee1475

Please sign in to comment.