Skip to content

Commit

Permalink
transformers version?
Browse files Browse the repository at this point in the history
  • Loading branch information
markus583 committed Jun 19, 2024
1 parent f3dd236 commit 08432dc
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 7 deletions.
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
author_email="[email protected]",
install_requires=[
"onnxruntime>=1.13.1",
"transformers>=4.22.2",
"transformers>=4.22.2,<=4.35",
"numpy>=1.0,<2.0",
"scikit-learn>=1",
"tqdm",
Expand Down
17 changes: 11 additions & 6 deletions wtpsplit/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from wtpsplit.extract import BertCharORTWrapper, PyTorchWrapper, SaTORTWrapper, extract
from wtpsplit.utils import Constants, indices_to_sentences, sigmoid

__version__ = "1.0.0"
__version__ = "2.0.0"


class WtP:
Expand Down Expand Up @@ -711,6 +711,7 @@ def get_default_threshold(model_str: str):


if __name__ == "__main__":
# FIXME: remove
# sat_lora = SaT("sat-3l", style_or_domain="ud", language="en")
# out = sat_lora.split(
# "Hello this is a test But this is different now Now the next one starts looool",
Expand All @@ -720,12 +721,16 @@ def get_default_threshold(model_str: str):
# print(out)
# splits = list(sat_lora.split(["Paragraph-A Paragraph-B", "Paragraph-C100 Paragraph-D"]))
# print(splits)
# sat_sm = SaT("sat-12l-sm")
# splits = sat_sm.split("This is a test sentence. This is another test sentence.", threshold=0.25)
# print(splits)
sat_ort_sm = SaT("/home/Markus/wtpsplit/scripts/sat-12l-sm", ort_providers=["CPUExecutionProvider"])
splits = sat_ort_sm.split("This is a test sentence. This is another test sentence.", threshold=0.25)
sat_sm = SaT("sat-3l-sm")
splits = sat_sm.split("this is a test this is another test")
print(splits)
# sat_ort_sm = SaT("/home/Markus/wtpsplit/scripts/xlm-roberta-base", ort_providers=["CPUExecutionProvider"])
# splits = sat_ort_sm.split("This is a test sentence. This is another test sentence.", threshold=0.25)
# print(splits)
# sat_ort = SaT("/home/Markus/wtpsplit/scripts/sat-12l-no-limited-lookahead", ort_providers=["CPUExecutionProvider"])
# # sat_ort = SaT("/home/Markus/wtpsplit/scripts/sat-1l-sm", ort_providers=["CPUExecutionProvider"])
# splits = sat_ort.split("This is a test sentence. This is another test sentence.", threshold=0.25)
# print(splits)
# wtp = WtP("wtp-bert-mini", ort_providers=["CPUExecutionProvider"])

# splits = wtp.split("This is a test sentence This is another test sentence.", threshold=0.005)

0 comments on commit 08432dc

Please sign in to comment.