From 71d7a87282bf0db18e1a2a176f620dbeb1b6bbe5 Mon Sep 17 00:00:00 2001 From: Lachlan Cahill Date: Sun, 19 Nov 2023 11:43:48 +1300 Subject: [PATCH 1/4] For some models like Mistral and Flan-UL2, when downloaded from the hub the binary files write to a snapshots/ subdirectory. Then when the files are committed, they cannot be found and the commit fails with an error. --- bindings/python/convert.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bindings/python/convert.py b/bindings/python/convert.py index a61476dd..eaca956a 100644 --- a/bindings/python/convert.py +++ b/bindings/python/convert.py @@ -145,7 +145,13 @@ def convert_multi( sf_filename = rename(pt_filename) sf_filename = os.path.join(folder, sf_filename) convert_file(pt_filename, sf_filename, discard_names=discard_names) - local_filenames.append(sf_filename) + + if 'snapshots' in sf_filename.split(os.sep): + new_sf_path = os.path.join(folder, os.path.basename(sf_filename)) + shutil.move(sf_filename, new_sf_path) + local_filenames.append(new_sf_path) + else: + local_filenames.append(sf_filename) index = os.path.join(folder, "model.safetensors.index.json") with open(index, "w") as f: From a6ab717c63ef0448a4169b393eeb1998bdd0184f Mon Sep 17 00:00:00 2001 From: Lachlan Cahill Date: Sun, 19 Nov 2023 11:45:28 +1300 Subject: [PATCH 2/4] Changed `local.split("/")[-1]` to os.path.basename(local) for compatibility with all OS including Windows. --- bindings/python/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/convert.py b/bindings/python/convert.py index eaca956a..421ae4dc 100644 --- a/bindings/python/convert.py +++ b/bindings/python/convert.py @@ -162,7 +162,7 @@ def convert_multi( local_filenames.append(index) operations = [ - CommitOperationAdd(path_in_repo=local.split("/")[-1], path_or_fileobj=local) for local in local_filenames + CommitOperationAdd(path_in_repo=os.path.basename(local), path_or_fileobj=local) for local in local_filenames ] errors: List[Tuple[str, "Exception"]] = [] From 3d56e2cd95928a37883b40a18a158ec7611239d2 Mon Sep 17 00:00:00 2001 From: Lachlan Cahill Date: Tue, 21 Nov 2023 21:30:02 +1300 Subject: [PATCH 3/4] Removed snapshots section --- bindings/python/convert.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bindings/python/convert.py b/bindings/python/convert.py index 421ae4dc..389e894c 100644 --- a/bindings/python/convert.py +++ b/bindings/python/convert.py @@ -146,12 +146,7 @@ def convert_multi( sf_filename = os.path.join(folder, sf_filename) convert_file(pt_filename, sf_filename, discard_names=discard_names) - if 'snapshots' in sf_filename.split(os.sep): - new_sf_path = os.path.join(folder, os.path.basename(sf_filename)) - shutil.move(sf_filename, new_sf_path) - local_filenames.append(new_sf_path) - else: - local_filenames.append(sf_filename) + local_filenames.append(sf_filename) index = os.path.join(folder, "model.safetensors.index.json") with open(index, "w") as f: From 5f34420ec128de1de98e0d95e6f90352717e056b Mon Sep 17 00:00:00 2001 From: Lachlan Cahill Date: Tue, 21 Nov 2023 21:43:14 +1300 Subject: [PATCH 4/4] Removed additional line break. --- bindings/python/convert.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bindings/python/convert.py b/bindings/python/convert.py index 389e894c..0ef9623b 100644 --- a/bindings/python/convert.py +++ b/bindings/python/convert.py @@ -145,7 +145,6 @@ def convert_multi( sf_filename = rename(pt_filename) sf_filename = os.path.join(folder, sf_filename) convert_file(pt_filename, sf_filename, discard_names=discard_names) - local_filenames.append(sf_filename) index = os.path.join(folder, "model.safetensors.index.json")