test_data_tokenizers.py

import pytest
import random
import collections
import pickle
from uuid import uuid4
import os
import unicodedata
import tempfile
from pkg_resources import parse_version
import gluonnlp
from gluonnlp.data.tokenizers import WhitespaceTokenizer, MosesTokenizer, JiebaTokenizer,\
    SpacyTokenizer, SubwordNMTTokenizer, YTTMTokenizer, SentencepieceTokenizer, \
    HuggingFaceBPETokenizer, HuggingFaceByteBPETokenizer, HuggingFaceWordPieceTokenizer, \
    HuggingFaceTokenizer
from gluonnlp.base import get_repo_url
from gluonnlp.data import Vocab, load_vocab
from gluonnlp.utils.misc import download
from gluonnlp.models.t5 import T5Tokenizer


EN_SAMPLES = ['Four score and seven years ago our fathers brought forth on this continent, '
              'a new nation, conceived in Liberty, and dedicated to the proposition '
              'that all men are created equal.',
              'In spite of the debate going on for months about the photos of Özil with the '
              'Turkish President Recep Tayyip Erdogan, he regrets the return of '
              'the 92-match national player Özil.']
DE_SAMPLES = ['Goethe stammte aus einer angesehenen bürgerlichen Familie; sein Großvater'
              ' mütterlicherseits war als Stadtschultheiß höchster Justizbeamter der'
              ' Stadt Frankfurt, sein Vater Doktor der Rechte und kaiserlicher Rat.',
              '"Das ist eine Frage, die natürlich davon abhängt, dass man einmal ins '
              'Gespräch kommt, dass man mit ihm auch darüber spricht, warum er das eine '
              'oder andere offenbar so empfunden hat, wie das in seinem Statement niedergelegt'
              ' ist", sagte Grindel im Fußball-Podcast "Phrasenmäher" der "Bild-Zeitung.']
ZH_SAMPLES = ['苟活者在淡红的血色中，会依稀看见微茫的希望；真的猛士，将更奋然而前行。',
              '参加工作，哈尔滨工业大学无线电工程系电子仪器及测量技术专业毕业。']

SUBWORD_TEST_SAMPLES = ["Hello, y'all! How are you Ⅷ 😁 😁 😁 ?",
                        'GluonNLP is great！！！!!!',
                        "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# 'abc'"]


def random_inject_space(sentence):
    words = sentence.split()
    ret = ''
    for i, word in enumerate(words):
        ret += word
        if i < len(words) - 1:
            n_space_tokens = random.randint(1, 10)
            for j in range(n_space_tokens):
                ret += random.choice([' ', '\t', '\r', '\n'])
    return ret


def verify_encode_token_with_offsets(tokenizer, all_sentences, gt_offsets=None):
    if gt_offsets is None:
        for sentences in [all_sentences[0], all_sentences]:
            enc_tokens = tokenizer.encode(sentences, str)
            tokens, offsets = tokenizer.encode_with_offsets(sentences, str)
            if isinstance(sentences, list):
                for ele_tokens, ele_enc_tokens, ele_offsets, ele_sentence in\
                        zip(tokens, enc_tokens, offsets, sentences):
                    for tok, offset, enc_tok in zip(ele_tokens, ele_offsets, ele_enc_tokens):
                        assert ele_sentence[offset[0]:offset[1]] == tok
                        assert tok == enc_tok
            else:
                for tok, offset, enc_tok in zip(tokens, offsets, enc_tokens):
                    assert sentences[offset[0]:offset[1]] == tok
                    assert tok == enc_tok
    else:
        for sentences, ele_gt_offsets in [(all_sentences[0], gt_offsets[0]),
                                          (all_sentences, gt_offsets)]:
            enc_tokens = tokenizer.encode(sentences, str)
            tokens, offsets = tokenizer.encode_with_offsets(sentences, str)
            assert ele_gt_offsets == offsets
            assert enc_tokens == tokens


def verify_sentencepiece_tokenizer_with_offsets(tokenizer, all_sentences):
    for sentences in [all_sentences[0], all_sentences]:
        enc_tokens = tokenizer.encode(sentences, str)
        tokens, offsets = tokenizer.encode_with_offsets(sentences, str)
        if isinstance(sentences, list):
            for ele_tokens, ele_enc_tokens, ele_offsets, ele_sentence\
                    in zip(tokens, enc_tokens, offsets, sentences):
                for i, (tok, offset, enc_tok) in enumerate(zip(ele_tokens, ele_offsets,
                                                               ele_enc_tokens)):
                    assert tok == enc_tok
                    ele_sel_tok = unicodedata.normalize('NFKC',
                                                        ele_sentence[offset[0]:offset[1]]).strip()
                    if tokenizer.is_first_subword(tok):
                        real_tok = tok[1:]
                    else:
                        real_tok = tok
                    assert ele_sel_tok == real_tok,\
                        'ele_sel_tok={}, real_tok={}'.format(ele_sel_tok, real_tok)


def verify_encode_with_offsets_consistency(tokenizer, all_sentences):
    for sentences in [all_sentences[0], all_sentences]:
        enc_tokens = tokenizer.encode(sentences, int)
        tokens, offsets = tokenizer.encode_with_offsets(sentences, int)
        str_tokens, str_offsets = tokenizer.encode_with_offsets(sentences, str)
        assert offsets == str_offsets
        assert tokens == enc_tokens


def verify_encode_token(tokenizer, all_sentences, all_gt_tokens):
    for sentences, gt_tokens in [(all_sentences[0], all_gt_tokens[0]),
                                 (all_sentences, all_gt_tokens)]:
        tokenizer_encode_ret = tokenizer.encode(sentences)
        assert tokenizer_encode_ret == gt_tokens,\
            'Whole Encoded: {}, \nWhole GT: {}'.format(tokenizer_encode_ret, gt_tokens)


def verify_decode(tokenizer, all_sentences, out_type=str):
    for sentences in [all_sentences[0], all_sentences]:
        assert tokenizer.decode(tokenizer.encode(sentences, out_type)) == sentences


def verify_decode_spm(tokenizer, all_sentences, gt_int_decode_sentences):
    for sentences, case_gt_int_decode in [(all_sentences[0], gt_int_decode_sentences[0]),
                                          (all_sentences, gt_int_decode_sentences)]:
        if isinstance(sentences, str):
            gt_str_decode_sentences = sentences
            if tokenizer.lowercase:
                gt_str_decode_sentences = gt_str_decode_sentences.lower()
            gt_str_decode_sentences = unicodedata.normalize('NFKC', gt_str_decode_sentences)
        elif isinstance(sentences, list):
            gt_str_decode_sentences = []
            for ele in sentences:
                ele_gt_decode = ele
                if tokenizer.lowercase:
                    ele_gt_decode = ele_gt_decode.lower()
                ele_gt_decode = unicodedata.normalize('NFKC', ele_gt_decode)
                gt_str_decode_sentences.append(ele_gt_decode)
        else:
            raise NotImplementedError
        assert tokenizer.decode(tokenizer.encode(sentences, str)) == gt_str_decode_sentences
        assert tokenizer.decode(tokenizer.encode(sentences, int)) == case_gt_int_decode


def verify_decode_subword_nmt(tokenizer, all_sentences, gt_int_decode, gt_str_decode):
    for sentences, case_gt_int_decode, case_gt_str_decode in [(all_sentences[0], gt_int_decode[0], gt_str_decode[0]),
                                                              (all_sentences, gt_int_decode, gt_str_decode)]:
        assert tokenizer.decode(tokenizer.encode(sentences, str)) == case_gt_str_decode
        assert tokenizer.decode(tokenizer.encode(sentences, int)) == case_gt_int_decode


def verify_decode_hf(tokenizer, all_sentences, gt_decode_sentences):
    for sentences, case_gt_decode in [(all_sentences[0], gt_decode_sentences[0]),
                                      (all_sentences, gt_decode_sentences)]:
        assert tokenizer.decode(tokenizer.encode(sentences, str)) == case_gt_decode
        assert tokenizer.decode(tokenizer.encode(sentences, int)) == case_gt_decode
        if isinstance(sentences, list):
            for sentence in sentences:
                assert tokenizer.vocab.to_tokens(tokenizer.encode(sentence, int))\
                       == tokenizer.encode(sentence, str)
                assert tokenizer.vocab[tokenizer.encode(sentence, str)]\
                       == tokenizer.encode(sentence, int)
        else:
            assert tokenizer.vocab.to_tokens(tokenizer.encode(sentences, int)) \
                   == tokenizer.encode(sentences, str)
            assert tokenizer.vocab[tokenizer.encode(sentences, str)] \
                   == tokenizer.encode(sentences, int)


def verify_decode_no_vocab_raise(tokenizer):
    # When the vocab is not attached, should raise ValueError
    for sentences in [EN_SAMPLES[0], EN_SAMPLES]:
        with pytest.raises(ValueError):
            tokenizer.encode(sentences, int)
    with pytest.raises(ValueError):
        tokenizer.decode([0])
    with pytest.raises(ValueError):
        tokenizer.decode([[0], [1]])


def verify_pickleble(tokenizer, cls):
    print(tokenizer)
    # Verify if the tokenizer is pickleable and has the same behavior after dumping/loading
    tokenizer_p = pickle.loads(pickle.dumps(tokenizer))
    assert isinstance(tokenizer_p, cls)
    assert tokenizer.encode(SUBWORD_TEST_SAMPLES, str) == tokenizer_p.encode(SUBWORD_TEST_SAMPLES, str)

def test_whitespace_tokenizer():
    tokenizer = WhitespaceTokenizer()
    gt_en_tokenized = [['Four', 'score', 'and', 'seven', 'years', 'ago', 'our', 'fathers', 'brought',
                        'forth', 'on', 'this', 'continent,', 'a', 'new', 'nation,', 'conceived',
                        'in', 'Liberty,', 'and', 'dedicated', 'to', 'the', 'proposition', 'that',
                        'all', 'men', 'are', 'created', 'equal.'],
                       ['In', 'spite', 'of', 'the', 'debate', 'going', 'on', 'for', 'months',
                        'about', 'the', 'photos', 'of', 'Özil', 'with', 'the', 'Turkish',
                        'President', 'Recep', 'Tayyip', 'Erdogan,', 'he', 'regrets', 'the',
                        'return', 'of', 'the', '92-match', 'national', 'player', 'Özil.']]
    gt_de_tokenized = [['Goethe', 'stammte', 'aus', 'einer', 'angesehenen', 'bürgerlichen',
                        'Familie;', 'sein', 'Großvater', 'mütterlicherseits', 'war', 'als',
                        'Stadtschultheiß', 'höchster', 'Justizbeamter', 'der', 'Stadt',
                        'Frankfurt,', 'sein', 'Vater', 'Doktor', 'der', 'Rechte', 'und',
                        'kaiserlicher', 'Rat.'],
                       ['"Das', 'ist', 'eine', 'Frage,', 'die', 'natürlich', 'davon', 'abhängt,',
                        'dass', 'man', 'einmal', 'ins', 'Gespräch', 'kommt,', 'dass', 'man', 'mit',
                        'ihm', 'auch', 'darüber', 'spricht,', 'warum', 'er', 'das', 'eine', 'oder',
                        'andere', 'offenbar', 'so', 'empfunden', 'hat,', 'wie', 'das', 'in',
                        'seinem', 'Statement', 'niedergelegt', 'ist",', 'sagte', 'Grindel', 'im',
                        'Fußball-Podcast', '"Phrasenmäher"', 'der', '"Bild-Zeitung.']]
    for _ in range(2):
        # Inject noise and test for encode
        noisy_en_samples = [random_inject_space(ele) for ele in EN_SAMPLES]
        noisy_de_samples = [random_inject_space(ele) for ele in DE_SAMPLES]
        verify_encode_token(tokenizer, noisy_en_samples + noisy_de_samples,
                            gt_en_tokenized + gt_de_tokenized)
        # Test for decode
        verify_decode(tokenizer, EN_SAMPLES + DE_SAMPLES, str)
        # Test for encode_with_offsets
        verify_encode_token_with_offsets(tokenizer, noisy_en_samples + noisy_de_samples)
    verify_decode_no_vocab_raise(tokenizer)

    # Test for output_type = int
    vocab = Vocab(collections.Counter(sum(gt_en_tokenized + gt_de_tokenized,
                                          [])))
    tokenizer.set_vocab(vocab)
    verify_decode(tokenizer, EN_SAMPLES + DE_SAMPLES, int)
    verify_pickleble(tokenizer, WhitespaceTokenizer)
    verify_encode_token_with_offsets(tokenizer, EN_SAMPLES + DE_SAMPLES)


def test_moses_tokenizer():
    en_tokenizer = MosesTokenizer('en')
    de_tokenizer = MosesTokenizer('de')
    gt_en_tokenized = [['Four', 'score', 'and', 'seven', 'years', 'ago', 'our', 'fathers',
                        'brought', 'forth', 'on', 'this', 'continent', ',', 'a', 'new', 'nation',
                        ',', 'conceived', 'in', 'Liberty', ',', 'and', 'dedicated', 'to', 'the',
                        'proposition', 'that', 'all', 'men', 'are', 'created', 'equal', '.'],
                       ['In', 'spite', 'of', 'the', 'debate', 'going', 'on', 'for', 'months',
                        'about', 'the', 'photos', 'of', 'Özil', 'with', 'the', 'Turkish',
                        'President', 'Recep', 'Tayyip', 'Erdogan', ',', 'he', 'regrets', 'the',
                        'return', 'of', 'the', '92-match', 'national', 'player', 'Özil', '.']]
    gt_de_tokenized = [['Goethe', 'stammte', 'aus', 'einer', 'angesehenen', 'bürgerlichen',
                        'Familie', ';', 'sein', 'Großvater', 'mütterlicherseits', 'war', 'als',
                        'Stadtschultheiß', 'höchster', 'Justizbeamter', 'der', 'Stadt',
                        'Frankfurt', ',', 'sein', 'Vater', 'Doktor', 'der', 'Rechte', 'und',
                        'kaiserlicher', 'Rat', '.'],
                       ['&quot;', 'Das', 'ist', 'eine', 'Frage', ',', 'die', 'natürlich', 'davon',
                        'abhängt', ',', 'dass', 'man', 'einmal', 'ins', 'Gespräch', 'kommt', ',',
                        'dass', 'man', 'mit', 'ihm', 'auch', 'darüber', 'spricht', ',', 'warum',
                        'er', 'das', 'eine', 'oder', 'andere', 'offenbar', 'so', 'empfunden',
                        'hat', ',', 'wie', 'das', 'in', 'seinem', 'Statement', 'niedergelegt',
                        'ist', '&quot;', ',', 'sagte', 'Grindel', 'im', 'Fußball-Podcast',
                        '&quot;', 'Phrasenmäher', '&quot;', 'der', '&quot;', 'Bild-Zeitung', '.']]
    verify_encode_token(en_tokenizer, EN_SAMPLES, gt_en_tokenized)
    verify_encode_token(de_tokenizer, DE_SAMPLES, gt_de_tokenized)
    verify_decode(en_tokenizer, EN_SAMPLES, str)
    verify_decode(de_tokenizer, DE_SAMPLES, str)
    vocab = Vocab(collections.Counter(sum(gt_en_tokenized + gt_de_tokenized, [])))
    verify_decode_no_vocab_raise(en_tokenizer)
    verify_decode_no_vocab_raise(de_tokenizer)
    en_tokenizer.set_vocab(vocab)
    de_tokenizer.set_vocab(vocab)
    verify_decode(en_tokenizer, EN_SAMPLES, int)
    verify_decode(de_tokenizer, DE_SAMPLES, int)
    verify_pickleble(en_tokenizer, MosesTokenizer)
    verify_pickleble(de_tokenizer, MosesTokenizer)


def test_jieba_tokenizer():
    tokenizer = JiebaTokenizer()
    gt_zh_tokenized = [['苟活', '者', '在', '淡红', '的', '血色', '中', '，',
                        '会', '依稀', '看见', '微茫', '的', '希望', '；', '真的',
                        '猛士', '，', '将', '更奋', '然而', '前行', '。'],
                       ['参加', '工作', '，', '哈尔滨工业大学', '无线电', '工程系', '电子仪器',
                        '及', '测量', '技术', '专业', '毕业', '。']]
    verify_encode_token(tokenizer, ZH_SAMPLES, gt_zh_tokenized)
    verify_decode(tokenizer, ZH_SAMPLES, str)
    vocab = Vocab(collections.Counter(sum(gt_zh_tokenized, [])))
    verify_decode_no_vocab_raise(tokenizer)
    tokenizer.set_vocab(vocab)
    verify_decode(tokenizer, ZH_SAMPLES, int)
    verify_pickleble(tokenizer, JiebaTokenizer)


def test_spacy_tokenizer():
    en_tokenizer = SpacyTokenizer('en')
    de_tokenizer = SpacyTokenizer('de')
    gt_en_tokenized = [['Four', 'score', 'and', 'seven', 'years', 'ago', 'our', 'fathers',
                        'brought', 'forth', 'on', 'this', 'continent', ',', 'a', 'new', 'nation',
                        ',', 'conceived', 'in', 'Liberty', ',', 'and', 'dedicated', 'to', 'the',
                        'proposition', 'that', 'all', 'men', 'are', 'created', 'equal', '.'],
                       ['In', 'spite', 'of', 'the', 'debate', 'going', 'on', 'for', 'months',
                        'about', 'the', 'photos', 'of', 'Özil', 'with', 'the', 'Turkish',
                        'President', 'Recep', 'Tayyip', 'Erdogan', ',', 'he', 'regrets', 'the',
                        'return', 'of', 'the', '92-match', 'national', 'player', 'Özil', '.']]
    gt_de_tokenized = [['Goethe', 'stammte', 'aus', 'einer', 'angesehenen', 'bürgerlichen',
                        'Familie', ';', 'sein', 'Großvater', 'mütterlicherseits', 'war', 'als',
                        'Stadtschultheiß', 'höchster', 'Justizbeamter', 'der', 'Stadt', 'Frankfurt',
                        ',', 'sein', 'Vater', 'Doktor', 'der', 'Rechte', 'und', 'kaiserlicher',
                        'Rat', '.'],
                       ['"', 'Das', 'ist', 'eine', 'Frage', ',', 'die', 'natürlich', 'davon',
                        'abhängt', ',', 'dass', 'man', 'einmal', 'ins', 'Gespräch', 'kommt', ',',
                        'dass', 'man', 'mit', 'ihm', 'auch', 'darüber', 'spricht', ',', 'warum',
                        'er', 'das', 'eine', 'oder', 'andere', 'offenbar', 'so', 'empfunden', 'hat',
                        ',', 'wie', 'das', 'in', 'seinem', 'Statement', 'niedergelegt', 'ist', '"',
                        ',', 'sagte', 'Grindel', 'im', 'Fußball-Podcast', '"', 'Phrasenmäher', '"',
                        'der', '"', 'Bild-Zeitung', '.']]
    verify_encode_token(en_tokenizer, EN_SAMPLES, gt_en_tokenized)
    verify_encode_token(de_tokenizer, DE_SAMPLES, gt_de_tokenized)
    vocab = Vocab(collections.Counter(sum(gt_en_tokenized + gt_de_tokenized, [])))
    en_tokenizer.set_vocab(vocab)
    de_tokenizer.set_vocab(vocab)
    verify_pickleble(en_tokenizer, SpacyTokenizer)
    verify_pickleble(de_tokenizer, SpacyTokenizer)
    verify_encode_token_with_offsets(en_tokenizer, EN_SAMPLES)
    verify_encode_token_with_offsets(de_tokenizer, DE_SAMPLES)

    # Test for loading spacy tokenizer from specifying the "model" flag
    en_tokenizer = SpacyTokenizer(model='en_core_web_lg')
    out = en_tokenizer.encode(EN_SAMPLES)


def test_yttm_tokenizer():
    with tempfile.TemporaryDirectory() as dir_path:
        model_path = os.path.join(dir_path, 'yttm.model')
        download(url=get_repo_url() + 'tokenizer_test_models/yttm/test_ende_yttm-6f2c39.model',
                 path=model_path)
        tokenizer = YTTMTokenizer(model_path=model_path)
        gt_tokenized = [['▁He', 'll', 'o', ',', '▁y', "'", 'all', '!', '▁How', '▁are', '▁you', '▁',
                         'Ⅷ', '▁', '😁', '▁', '😁', '▁', '😁', '▁?'],
                        ['▁Gl', 'u', 'on', 'N', 'L', 'P', '▁is', '▁great', '！', '！', '！', '!',
                         '!', '!'],
                        ['▁Gl', 'u', 'on', 'N', 'L', 'P', '-A', 'm', 'az', 'on', '-H', 'a', 'ib',
                         'in', '-L', 'e', 'on', 'ard', '-S', 'hen', 'g', '-S', 'h', 'u', 'ai',
                         '-', 'X', 'ing', 'j', 'ian', '.', '.', '.', '.', '.', '/', ':', '!',
                         '@', '#', '▁', "'", 'ab', 'c', "'"]]
        gt_offsets = [[(0, 2), (2, 4), (4, 5), (5, 6), (6, 8), (8, 9), (9, 12), (12, 13), (13, 17),
                       (17, 21), (21, 25), (25, 26), (26, 27), (27, 28), (28, 29), (29, 30), (30, 31),
                       (31, 32), (32, 33), (33, 35)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 7), (7, 8), (8, 11), (11, 17), (17, 18),
                       (18, 19), (19, 20), (20, 21), (21, 22), (22, 23)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 7), (7, 8), (8, 10), (10, 11), (11, 13),
                       (13, 15), (15, 17), (17, 18), (18, 20), (20, 22), (22, 24), (24, 25), (25, 27),
                       (27, 30), (30, 32), (32, 35), (35, 36), (36, 38), (38, 39), (39, 40), (40, 42),
                       (42, 43), (43, 44), (44, 47), (47, 48), (48, 51), (51, 52), (52, 53), (53, 54),
                       (54, 55), (55, 56), (56, 57), (57, 58), (58, 59), (59, 60), (60, 61), (61, 62),
                       (62, 63), (63, 65), (65, 66), (66, 67)]]
        gt_int_decode = ['Hello, y<UNK>all! How are you <UNK> <UNK> <UNK> <UNK> ?',
                         'GluonNLP is great！！！!!!',
                         'GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# <UNK>abc<UNK>']
        gt_str_decode = ["Hello, y'all! How are you Ⅷ 😁 😁 😁 ?",
                         'GluonNLP is great！！！!!!',
                         "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# 'abc'"]
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, YTTMTokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        # Begin to verify decode
        for sample_sentences, ele_gt_int_decode, ele_gt_str_decode in [(SUBWORD_TEST_SAMPLES[0], gt_int_decode[0], gt_str_decode[0]),
                                                                       (SUBWORD_TEST_SAMPLES, gt_int_decode, gt_str_decode)]:
            int_decode = tokenizer.decode(tokenizer.encode(sample_sentences, int))
            str_decode = tokenizer.decode(tokenizer.encode(sample_sentences, str))
            assert int_decode == ele_gt_int_decode
            assert str_decode == ele_gt_str_decode
        os.remove(model_path)
        assert tokenizer.decode([]) == ''
        assert tokenizer.decode([[]]) == ['']


@pytest.mark.seed(123)
def test_sentencepiece_tokenizer():
    with tempfile.TemporaryDirectory() as dir_path:
        model_path = os.path.join(dir_path, 'spm.model')
        download(url=get_repo_url()
                     + 'tokenizer_test_models/sentencepiece/case1/test_ende-a9bee4.model',
                 path=model_path)
        # Case1
        tokenizer = SentencepieceTokenizer(model_path)
        gt_tokenized = [['▁Hel', 'lo', ',', '▁y', "'", 'all', '!', '▁How', '▁are', '▁you',
                         '▁', 'VI', 'II', '▁', '😁', '▁', '😁', '▁', '😁', '▁?'],
                        ['▁G', 'lu', 'on', 'N', 'L', 'P', '▁is', '▁great', '!', '!', '!', '!',
                         '!', '!'],
                        ['▁G', 'lu', 'on', 'N', 'L', 'P', '-', 'A', 'ma', 'zo', 'n', '-', 'H', 'ai',
                         'bin', '-', 'L', 'e', 'on', 'ard', '-', 'S', 'hen', 'g', '-', 'S', 'hu', 'ai',
                         '-', 'X', 'ing', 'j', 'ian', '.', '.', '.', '.', '.', '/', ':', '!', '@',
                         '#', '▁', "'", 'ab', 'c', "'"]]
        gt_offsets = [[(0, 3), (3, 5), (5, 6), (6, 8), (8, 9), (9, 12), (12, 13), (13, 17), (17, 21),
                       (21, 25), (25, 26), (26, 26), (26, 27), (27, 28), (28, 29), (29, 30), (30, 31),
                       (31, 32), (32, 33), (33, 35)],
                      [(0, 1), (1, 3), (3, 5), (5, 6), (6, 7), (7, 8), (8, 11), (11, 17), (17, 18),
                       (18, 19), (19, 20), (20, 21), (21, 22), (22, 23)],
                      [(0, 1), (1, 3), (3, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 12),
                       (12, 14), (14, 15), (15, 16), (16, 17), (17, 19), (19, 22), (22, 23), (23, 24),
                       (24, 25), (25, 27), (27, 30), (30, 31), (31, 32), (32, 35), (35, 36), (36, 37),
                       (37, 38), (38, 40), (40, 42), (42, 43), (43, 44), (44, 47), (47, 48), (48, 51),
                       (51, 52), (52, 53), (53, 54), (54, 55), (55, 56), (56, 57), (57, 58), (58, 59),
                       (59, 60), (60, 61), (61, 62), (62, 63), (63, 65), (65, 66), (66, 67)]]
        gt_int_decode = ['Hello, y ⁇ all! How are you VIII  ⁇   ⁇   ⁇  ?',
                         'GluonNLP is great!!!!!!',
                         'GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:! ⁇ #  ⁇ abc ⁇ ']
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, SentencepieceTokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_spm(tokenizer, SUBWORD_TEST_SAMPLES, gt_int_decode)

        # Case2, lower_case
        gt_lower_case_int_decode = ['hello, y ⁇ all! how are you viii  ⁇   ⁇   ⁇  ?',
                                    'gluonnlp is great!!!!!!',
                                    'gluonnlp-amazon-haibin-leonard-sheng-shuai-xingjian...../:! ⁇ #  ⁇ abc ⁇ ']
        tokenizer = SentencepieceTokenizer(model_path, lowercase=True)
        verify_decode_spm(tokenizer, SUBWORD_TEST_SAMPLES, gt_lower_case_int_decode)

        # Case3, Use the sentencepiece regularization commands, we test whether we can obtain different encoding results
        tokenizer = SentencepieceTokenizer(model_path, lowercase=True, nbest=-1, alpha=1.0)
        has_different_encode_out = False
        encode_out = None
        for _ in range(10):
            if encode_out is None:
                encode_out = tokenizer.encode(SUBWORD_TEST_SAMPLES[0])
            else:
                ele_out = tokenizer.encode(SUBWORD_TEST_SAMPLES[0])
                if ele_out != encode_out:
                    has_different_encode_out = True
                    break
        assert has_different_encode_out
        os.remove(model_path)

    # Case of T5 Tokenizer
    with tempfile.TemporaryDirectory() as dir_path: 
        vocab_path = os.path.join(dir_path, 't5_spm.model')
        download(
            url=get_repo_url() + 'tokenizer_test_models/sentencepiece/case_t5/test_t5spm-5f05e7.model',
            path=vocab_path
        )
        extra_ids = 100
        tokenizer = T5Tokenizer(vocab_path, extra_ids)
        gt_tokenized = [
            ['▁Hello', ',', '▁', 'y', "'", 'all', '!', '▁How', '▁are', '▁you', '▁VIII', '▁', '😁', 
             '▁', '😁', '▁', '😁', '▁', '?'], 
            ['▁', 'Glu', 'on', 'N', 'LP', '▁is', '▁great', '!', '!!!!!'], 
            ['▁', 'Glu', 'on', 'N', 'LP', '-', 'Am', 'a', 'zon', '-', 'H', 'a', 'i', 'bin', '-', 
             'Le', 'on', 'ard', '-', 'She', 'ng', '-', 'Sh', 'u', 'a', 'i', '-', 'X', 'ing', 'j', 
              'i', 'an', '.....', '/', ':', '!', '@', '#', '▁', "'", 'a', 'b', 'c', "'"]
        ]
        gt_offsets = [
            [(0, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 12), (12, 13), (13, 17), (17, 21), (21, 25), 
             (25, 27), (27, 28), (28, 29), (29, 30), (30, 31), (31, 32), (32, 33), (33, 34), (34, 35)], 
            [(0, 0), (0, 3), (3, 5), (5, 6), (6, 8), (8, 11), (11, 17), (17, 18), (18, 23)], 
            [(0, 0), (0, 3), (3, 5), (5, 6), (6, 8), (8, 9), (9, 11), (11, 12), (12, 15), (15, 16), 
             (16, 17), (17, 18), (18, 19), (19, 22), (22, 23), (23, 25), (25, 27), (27, 30), (30, 31), 
             (31, 34), (34, 36), (36, 37), (37, 39), (39, 40), (40, 41), (41, 42), (42, 43), (43, 44), 
             (44, 47), (47, 48), (48, 49), (49, 51), (51, 56), (56, 57), (57, 58), (58, 59), (59, 60), 
             (60, 61), (61, 62), (62, 63), (63, 64), (64, 65), (65, 66), (66, 67)]
        ]
        gt_int_decode = [
            "Hello, y'all! How are you VIII  ⁇   ⁇   ⁇  ?",
            'GluonNLP is great!!!!!!', 
            "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# 'abc'"
        ]
        inserted_special_tokens = list('<extra_id_{}>'.format(i) for i in range(extra_ids - 1, -1, -1))
        assert list(
            tokenizer.vocab.to_tokens(i) for i in range(len(tokenizer._sp_model), len(tokenizer._vocab))
        ) == inserted_special_tokens, 'Some <extra_id> tokens are not properly inserted.'
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, SentencepieceTokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_spm(tokenizer, SUBWORD_TEST_SAMPLES, gt_int_decode)
        os.remove(vocab_path)


def test_subword_nmt_tokenizer():
    with tempfile.TemporaryDirectory() as dir_path:
        model_path = os.path.join(dir_path, 'subword_nmt.model')
        download(url=get_repo_url() + 'tokenizer_test_models/subword-nmt/test_ende-d189ff.model',
                 path=model_path)
        vocab_path = os.path.join(dir_path, 'subword_nmt.vocab')
        download(url=get_repo_url() + 'tokenizer_test_models/subword-nmt/test_ende_vocab-900f81.json',
                 path=vocab_path)

        # Case 1
        tokenizer = SubwordNMTTokenizer(model_path, vocab_path)
        gt_tokenized = [["Hel", "lo", ",</w>", "y", "\'", "all", "!</w>", "How</w>", "are</w>", "you</w>",
                         "Ⅷ</w>", "😁</w>", "😁</w>", "😁</w>", "?</w>"],
                        ["Gl", "u", "on", "N", "L", "P</w>", "is</w>", "great", "！", "！", "！", "!!",
                         "!</w>"],
                        ["Gl", "u", "on", "N", "L", "P", "-", "Amaz", "on-", "H", "ai", "b", "in-", "Le",
                         "on", "ard", "-", "Sh", "eng", "-", "Sh", "u", "ai", "-", "X", "ing", "ji",
                         "an", "..", "...", "/", ":", "!", "@", "#</w>", "\'", "ab", "c", "\'</w>"]]
        gt_offsets = [[(0, 3), (3, 5), (5, 6), (7, 8), (8, 9), (9, 12), (12, 13), (14, 17), (18, 21),
                       (22, 25), (26, 27), (28, 29), (30, 31), (32, 33), (34, 35)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 7), (7, 8), (9, 11), (12, 17), (17, 18),
                       (18, 19), (19, 20), (20, 22), (22, 23)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 13), (13, 16),
                       (16, 17), (17, 19), (19, 20), (20, 23), (23, 25), (25, 27), (27, 30), (30, 31),
                       (31, 33), (33, 36), (36, 37), (37, 39), (39, 40), (40, 42), (42, 43), (43, 44),
                       (44, 47), (47, 49), (49, 51), (51, 53), (53, 56), (56, 57), (57, 58), (58, 59),
                       (59, 60), (60, 61), (62, 63), (63, 65), (65, 66), (66, 67)]]
        gt_int_decode = ["Hello, y\'all! How are you Ⅷ 😁 😁 😁 ?",
                         "GluonNLP is great！！！!!!",
                         "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# \'abc\'"]
        gt_str_decode = SUBWORD_TEST_SAMPLES
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, SubwordNMTTokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_subword_nmt(tokenizer, SUBWORD_TEST_SAMPLES, gt_int_decode, gt_str_decode)

        # Case 2, bpe_dropout
        # We use str decode here because we may not perfectly recover the original sentence with int decode.
        tokenizer = SubwordNMTTokenizer(model_path, vocab_path, bpe_dropout=0.5)
        verify_decode(tokenizer, SUBWORD_TEST_SAMPLES, out_type=str)

        os.remove(model_path)
        os.remove(vocab_path)


def test_huggingface_bpe_tokenizer():
    with tempfile.TemporaryDirectory() as dir_path:
        model_path = os.path.join(dir_path, 'test_hf_bpe.model')
        download(url=get_repo_url() + 'tokenizer_test_models/hf_bpe/test_hf_bpe.model',
                 path=model_path)
        vocab_path = os.path.join(dir_path, 'test_hf_bpe.vocab')
        download(url=get_repo_url() + 'tokenizer_test_models/hf_bpe/test_hf_bpe.vocab',
                 path=vocab_path)
        hf_vocab_path = os.path.join(dir_path, 'test_hf_bpe.hf_vocab')
        download(url=get_repo_url() + 'tokenizer_test_models/hf_bpe/test_hf_bpe.hf_vocab',
                 path=hf_vocab_path)

        # Case 1, default lowercase=False
        tokenizer = HuggingFaceBPETokenizer(model_path, vocab_path)
        gt_tokenized = [['Hello</w>', ',</w>', 'y</w>', "'</w>", 'all</w>', '!</w>', 'How</w>',
                         'are</w>', 'you</w>', '<unk>', '<unk>', '<unk>', '<unk>', '?</w>'],
                        ['Gl', 'u', 'on', 'N', 'LP</w>', 'is</w>', 'great</w>', '！</w>', '！</w>',
                         '！</w>', '!</w>', '!</w>', '!</w>'],
                        ['Gl', 'u', 'on', 'N', 'LP</w>', '-</w>', 'Amazon</w>', '-</w>', 'H', 'ai',
                         'bin</w>', '-</w>', 'Leonard</w>', '-</w>', 'Sh', 'en', 'g</w>', '-</w>',
                         'Sh', 'u', 'ai</w>', '-</w>', 'X', 'ing', 'j', 'ian</w>', '.</w>', '.</w>',
                         '.</w>', '.</w>', '.</w>', '/</w>', ':</w>', '!</w>', '@</w>', '#</w>',
                         "'</w>", 'ab', 'c</w>', "'</w>"]]
        gt_offsets = [[(0, 5), (5, 6), (7, 8), (8, 9), (9, 12), (12, 13), (14, 17), (18, 21), (22, 25),
                       (26, 27), (28, 29), (30, 31), (32, 33), (34, 35)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 8), (9, 11), (12, 17), (17, 18), (18, 19),
                       (19, 20), (20, 21), (21, 22), (22, 23)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 8), (8, 9), (9, 15), (15, 16), (16, 17),
                       (17, 19), (19, 22), (22, 23), (23, 30), (30, 31), (31, 33), (33, 35), (35, 36),
                       (36, 37), (37, 39), (39, 40), (40, 42), (42, 43), (43, 44), (44, 47), (47, 48),
                       (48, 51), (51, 52), (52, 53), (53, 54), (54, 55), (55, 56), (56, 57), (57, 58),
                       (58, 59), (59, 60), (60, 61), (62, 63), (63, 65), (65, 66), (66, 67)]]
        # gt_int_decode = gt_str_decode for hf
        # hf removed the unk tokens in decode result
        gt_decode = ["Hello , y ' all ! How are you ?",
                     'GluonNLP is great ！ ！ ！ ! ! !',
                     "GluonNLP - Amazon - Haibin - Leonard - Sheng - Shuai - Xingjian . . . . . / : ! @ # ' abc '"]
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, HuggingFaceBPETokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)

        # Case 2, lowercase=True
        gt_lowercase_decode = ["hello , y ' all ! how are you ?",
                               'gluonnlp is great ！ ！ ！ ! ! !',
                               "gluonnlp - amazon - haibin - leonard - sheng - shuai - xingjian . . . . . / : ! @ # ' abc '"]
        tokenizer = HuggingFaceBPETokenizer(model_path, vocab_path, lowercase=True)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_lowercase_decode)

        # Case 3, using original hf vocab
        tokenizer = HuggingFaceBPETokenizer(model_path, hf_vocab_path)
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, HuggingFaceBPETokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)

        os.remove(model_path)
        os.remove(vocab_path)
        os.remove(hf_vocab_path)


def test_huggingface_bytebpe_tokenizer():
    with tempfile.TemporaryDirectory() as dir_path:
        model_path = os.path.join(dir_path, 'hf_bytebpe.model')
        download(url=get_repo_url() + 'tokenizer_test_models/hf_bytebpe/test_hf_bytebpe.model',
                 path=model_path)
        vocab_path = os.path.join(dir_path, 'hf_bytebpe.vocab')
        download(url=get_repo_url() + 'tokenizer_test_models/hf_bytebpe/test_hf_bytebpe.vocab',
                 path=vocab_path)
        hf_vocab_path = os.path.join(dir_path, 'hf_bytebpe.hf_vocab')
        download(url=get_repo_url() + 'tokenizer_test_models/hf_bytebpe/test_hf_bytebpe.hf_vocab',
                 path=hf_vocab_path)

        # Case 1, default lowercase=False
        tokenizer = HuggingFaceByteBPETokenizer(model_path, vocab_path)
        gt_tokenized = [['Hello', ',', 'Ġy', "'", 'all', '!', 'ĠHow', 'Ġare', 'Ġyou',
                         'Ġâ', 'ħ', '§', 'ĠðŁĺ', 'ģ', 'ĠðŁĺ', 'ģ', 'ĠðŁĺ', 'ģ', 'Ġ?'],
                        ['Gl', 'u', 'on', 'N', 'LP', 'Ġis', 'Ġgreat', 'ï¼', 'ģ', 'ï¼',
                         'ģ', 'ï¼', 'ģ', '!!!'],
                        ['Gl', 'u', 'on', 'N', 'LP', '-', 'Amazon', '-', 'Ha', 'ib', 'in',
                         '-', 'Le', 'on', 'ard', '-', 'She', 'ng', '-', 'Sh', 'u',
                         'ai', '-', 'X', 'ing', 'j', 'ian', '.....', '/', ':', '!', '@',
                         '#', "Ġ'", 'ab', 'c', "'"]]
        # the defination of the offsets of bytelevel seems not clear
        gt_offsets = [[(0, 5), (5, 6), (6, 8), (8, 9), (9, 12), (12, 13), (13, 17), (17, 21),
                       (21, 25), (25, 27), (26, 27), (26, 27), (27, 29), (28, 29), (29, 31),
                       (30, 31), (31, 33), (32, 33), (33, 35)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 8), (8, 11), (11, 17), (17, 18),
                       (17, 18), (18, 19), (18, 19), (19, 20), (19, 20), (20, 23)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 8), (8, 9), (9, 15), (15, 16),
                       (16, 18), (18, 20), (20, 22), (22, 23), (23, 25), (25, 27), (27, 30),
                       (30, 31), (31, 34), (34, 36), (36, 37), (37, 39), (39, 40), (40, 42),
                       (42, 43), (43, 44), (44, 47), (47, 48), (48, 51), (51, 56),
                       (56, 57), (57, 58), (58, 59), (59, 60), (60, 61), (61, 63),
                       (63, 65), (65, 66), (66, 67)]]
        gt_decode = ["Hello, y'all! How are you Ⅷ 😁 😁 😁 ?",
                     'GluonNLP is great！！！!!!',
                     "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# 'abc'"]
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, HuggingFaceByteBPETokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)

        # Case 2, lowercase=True
        gt_lowercase_int_decode = ["hello, y'all! how are you ⅷ 😁 😁 😁 ?",
                                   'gluonnlp is great！！！!!!',
                                   "gluonnlp-amazon-haibin-leonard-sheng-shuai-xingjian...../:!@# 'abc'"]
        tokenizer = HuggingFaceByteBPETokenizer(model_path, vocab_path, lowercase=True)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_lowercase_int_decode)

        # Case 3, using original hf vocab
        tokenizer = HuggingFaceByteBPETokenizer(model_path, hf_vocab_path)
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, HuggingFaceByteBPETokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)

        os.remove(model_path)
        os.remove(vocab_path)
        os.remove(hf_vocab_path)


def test_huggingface_wordpiece_tokenizer():
    with tempfile.TemporaryDirectory() as dir_path:
        vocab_path = os.path.join(dir_path, 'hf_wordpiece.vocab')
        download(url=get_repo_url()
                     + 'tokenizer_test_models/hf_wordpiece/test_hf_wordpiece.vocab',
                 path=vocab_path)
        hf_vocab_path = os.path.join(dir_path, 'hf_wordpiece.hf_vocab')
        download(url=get_repo_url()
                     + 'tokenizer_test_models/hf_wordpiece/test_hf_wordpiece.hf_vocab',
                 path=hf_vocab_path)

        # Case 1, lowercase=True
        tokenizer = HuggingFaceWordPieceTokenizer(vocab_path, lowercase=True)
        gt_tokenized = [["hello", ",", "y", "'", "all", "!", "how", "are", "you",
                         "<unk>", "<unk>", "<unk>", "<unk>", "?"],
                        ["gl", "##uo", "##nn", "##l", "##p", "is", "great", "\uff01",
                         "\uff01", "\uff01", "!", "!", "!"],
                        ["gl", "##uo", "##nn", "##l", "##p", "-", "amazon", "-", "hai",
                         "##bin", "-", "leonard", "-", "shen", "##g", "-", "shu", "##ai", "-",
                         "xin", "##g", "##ji", "##an", ".", ".", ".", ".", ".", "/", ":", "!",
                         "@", "#", "'", "abc", "'"]]
        gt_offsets = [[(0, 5), (5, 6), (7, 8), (8, 9), (9, 12), (12, 13), (14, 17), (18, 21),
                       (22, 25), (26, 27), (28, 29), (30, 31), (32, 33), (34, 35)],
                      [(0, 2), (2, 4), (4, 6), (6, 7), (7, 8), (9, 11), (12, 17), (17, 18),
                       (18, 19), (19, 20), (20, 21), (21, 22), (22, 23)],
                      [(0, 2), (2, 4), (4, 6), (6, 7), (7, 8), (8, 9), (9, 15), (15, 16), (16, 19),
                       (19, 22), (22, 23), (23, 30), (30, 31), (31, 35), (35, 36), (36, 37), (37, 40),
                       (40, 42), (42, 43), (43, 46), (46, 47), (47, 49), (49, 51), (51, 52), (52, 53),
                       (53, 54), (54, 55), (55, 56), (56, 57), (57, 58), (58, 59), (59, 60), (60, 61),
                       (62, 63), (63, 66), (66, 67)]]
        gt_decode = ["hello, y'all! how are you?",
                     "gluonnlp is great ！ ！ ！!!!",
                     "gluonnlp - amazon - haibin - leonard - sheng - shuai - xingjian..... / :! @ #'abc '"]
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, HuggingFaceWordPieceTokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)

        # Case 2, lowercase=False
        gt_lowercase_decode = [", y'all! are you?",
                               "is great ！ ！ ！!!!",
                               "- - - - - -..... / :! @ #'abc '"]
        tokenizer = HuggingFaceWordPieceTokenizer(vocab_path, lowercase=False)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_lowercase_decode)

        # Case 3, using original hf vocab
        tokenizer = HuggingFaceWordPieceTokenizer(hf_vocab_path, lowercase=True)
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, HuggingFaceWordPieceTokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)

        os.remove(vocab_path)
        os.remove(hf_vocab_path)


@pytest.mark.skipif(parse_version(gluonnlp.utils.lazy_imports.try_import_huggingface_tokenizers().__version__)
                    >= parse_version('0.9.0.dev0'), reason="Test is only valid for tokenizers 0.8.x")
def test_huggingface_wordpiece_tokenizer_v08():
    """Test for huggingface tokenizer >=0.8"""
    with tempfile.TemporaryDirectory() as dir_path:
        model_path = os.path.join(dir_path, 'hf_wordpiece_new_0.8.model')
        download(url=get_repo_url() +
                     'tokenizer_test_models/hf_wordpiece_new_0.8/hf_wordpiece.model',
                 path=model_path,
                 sha1_hash='66ccadf6e5e354ff9604e4a82f107a2ac873abd5')
        vocab_path = os.path.join(dir_path, 'hf_wordpiece_new_0.8.vocab')
        download(url=get_repo_url() +
                     'tokenizer_test_models/hf_wordpiece_new_0.8/hf_wordpiece.vocab',
                 path=vocab_path,
                 sha1_hash='dd6fdf4bbc74eaa8806d12cb3d38a4d9a306aea8')
        tokenizer = HuggingFaceTokenizer(model_path, vocab_path)
        gt_tokenized = [['Hel', '##lo', ',', 'y', '[UNK]', 'all', '!',
                         'How', 'are', 'you', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '?'],
                        ['Gl', '##u', '##on', '##N', '##L', '##P', 'is', 'great', '[UNK]',
                         '[UNK]', '[UNK]', '!', '!', '!'],
                        ['Gl', '##u', '##on', '##N', '##L', '##P', '-',
                         'Am', '##az', '##on', '-', 'Ha', '##ibi', '##n', '-', 'Leon', '##ard',
                         '-', 'She', '##n', '##g', '-', 'Sh', '##ua', '##i', '-', 'X',
                         '##ing', '##j', '##ian', '.', '.', '.', '.', '.', '/', ':', '!',
                         '@', '#', '[UNK]', 'ab', '##c', '[UNK]']]
        gt_offsets = [[(0, 3), (3, 5), (5, 6), (7, 8), (8, 9), (9, 12), (12, 13),
                       (14, 17), (18, 21), (22, 25), (26, 27), (28, 29), (30, 31),
                       (32, 33), (34, 35)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 7), (7, 8), (9, 11), (12, 17),
                       (17, 18), (18, 19), (19, 20), (20, 21), (21, 22), (22, 23)],
                      [(0, 2), (2, 3), (3, 5), (5, 6), (6, 7), (7, 8), (8, 9),
                       (9, 11), (11, 13), (13, 15), (15, 16), (16, 18), (18, 21),
                       (21, 22), (22, 23), (23, 27), (27, 30), (30, 31), (31, 34),
                       (34, 35), (35, 36), (36, 37), (37, 39), (39, 41), (41, 42),
                       (42, 43), (43, 44), (44, 47), (47, 48), (48, 51), (51, 52),
                       (52, 53), (53, 54), (54, 55), (55, 56), (56, 57), (57, 58),
                       (58, 59), (59, 60), (60, 61), (62, 63), (63, 65), (65, 66),
                       (66, 67)]]
        gt_decode = ['Hello, y all! How are you?',
                     'GluonNLP is great!!!',
                     'GluonNLP - Amazon - Haibin - Leonard - Sheng - Shuai - Xingjian..... / '
                     ':! @ # abc']
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, HuggingFaceTokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)


@pytest.mark.skipif(parse_version(gluonnlp.utils.lazy_imports.try_import_huggingface_tokenizers().__version__)
                    >= parse_version('0.9.0.dev0'), reason="Test is only valid for tokenizers 0.8.x")
def test_huggingface_bpe_tokenizer_v08():
    """Test for huggingface BPE tokenizer >=0.8"""
    with tempfile.TemporaryDirectory() as dir_path:
        model_path = os.path.join(dir_path, 'hf_bpe_new_0.8.model')
        download(url=get_repo_url() +
                     'tokenizer_test_models/hf_bpe_new_0.8/hf_bpe.model',
                 path=model_path,
                 sha1_hash='ecda90979561ca4c5a8d769b5e3c9fa2270d5317')
        vocab_path = os.path.join(dir_path, 'hf_bpe_new_0.8.vocab')
        download(url=get_repo_url() +
                     'tokenizer_test_models/hf_bpe_new_0.8/hf_bpe.vocab',
                 path=vocab_path,
                 sha1_hash='b92dde0b094f405208f3ec94b5eae88430bf4262')
        tokenizer = HuggingFaceTokenizer(model_path, vocab_path)
        gt_tokenized = [['H', 'ello</w>', ',</w>', 'y</w>', 'all</w>', '!</w>',
                         'How</w>', 'are</w>', 'you</w>', '?</w>'],
                        ['G', 'lu', 'on', 'N', 'L', 'P</w>', 'is</w>', 'great</w>',
                         '!</w>', '!</w>', '!</w>'],
                        ['G', 'lu', 'on', 'N', 'L', 'P</w>', '-</w>', 'Amaz', 'on</w>',
                         '-</w>', 'Ha', 'i', 'bin</w>', '-</w>', 'Leon', 'ard</w>', '-</w>',
                         'Sh', 'eng</w>', '-</w>', 'S', 'hu', 'ai</w>', '-</w>', 'X', 'ing',
                         'j', 'ian</w>', '.</w>', '.</w>', '.</w>', '.</w>', '.</w>', '/</w>',
                         ':</w>', '!</w>', '@</w>', '#</w>', 'ab', 'c</w>']]
        gt_offsets = [[(0, 1), (1, 5), (5, 6), (7, 8), (9, 12), (12, 13), (14, 17),
                       (18, 21), (22, 25), (34, 35)],
                      [(0, 1), (1, 3), (3, 5), (5, 6), (6, 7), (7, 8), (9, 11), (12, 17),
                       (20, 21), (21, 22), (22, 23)],
                      [(0, 1), (1, 3), (3, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 13), (13, 15),
                       (15, 16), (16, 18), (18, 19), (19, 22), (22, 23), (23, 27), (27, 30),
                       (30, 31), (31, 33), (33, 36), (36, 37), (37, 38), (38, 40), (40, 42),
                       (42, 43), (43, 44), (44, 47), (47, 48), (48, 51), (51, 52), (52, 53),
                       (53, 54), (54, 55), (55, 56), (56, 57), (57, 58), (58, 59), (59, 60),
                       (60, 61), (63, 65), (65, 66)]]
        gt_decode = ['Hello , y all ! How are you ?',
                     'GluonNLP is great ! ! !',
                     'GluonNLP - Amazon - Haibin - Leonard - Sheng - Shuai - Xingjian'
                     ' . . . . . / : ! @ # abc']
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, HuggingFaceTokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)


@pytest.mark.skipif(parse_version(gluonnlp.utils.lazy_imports.try_import_huggingface_tokenizers().__version__)
                    >= parse_version('0.9.0.dev0'), reason="Test is only valid for tokenizers 0.8.x")
def test_huggingface_bytebpe_tokenizer_v08():
    """Test for huggingface bytebpe tokenizer >=0.8"""
    with tempfile.TemporaryDirectory() as dir_path:
        model_path = os.path.join(dir_path, 'hf_bytebpe_new_0.8.model')
        download(url=get_repo_url() +
                     'tokenizer_test_models/hf_bytebpe_new_0.8/hf_bytebpe.model',
                 path=model_path,
                 sha1_hash='a1c4da1f6c21df923e150f56dbb5b7a53c61808b')
        vocab_path = os.path.join(dir_path, 'hf_bytebpe_new_0.8.vocab')
        download(url=get_repo_url() +
                     'tokenizer_test_models/hf_bytebpe_new_0.8/hf_bytebpe.vocab',
                 path=vocab_path,
                 sha1_hash='7831b19078a3222f450e65b2188dc0770473123b')
        tokenizer = HuggingFaceTokenizer(model_path, vocab_path)
        gt_tokenized = [['He', 'llo', ',', 'Ġy', "'", 'all', '!', 'ĠHow', 'Ġare', 'Ġyou',
                         'Ġâ', 'ħ', '§', 'Ġ', 'ð', 'Ł', 'ĺ', 'ģ', 'Ġ', 'ð', 'Ł', 'ĺ',
                         'ģ', 'Ġ', 'ð', 'Ł', 'ĺ', 'ģ', 'Ġ?'],
                        ['G', 'l', 'u', 'on', 'N', 'L', 'P', 'Ġis', 'Ġgreat', 'ï', '¼', 'ģ',
                         'ï', '¼', 'ģ', 'ï', '¼', 'ģ', '!', '!', '!'],
                        ['G', 'l', 'u', 'on', 'N', 'L', 'P', '-', 'Am', 'az', 'on', '-',
                         'Ha', 'ib', 'in', '-', 'Le', 'on', 'ard', '-', 'S', 'hen', 'g', '-',
                         'Sh', 'u', 'ai', '-', 'X', 'ing', 'j', 'ian',
                         '..', '...', '/', ':', '!', '@', '#', 'Ġ', "'", 'ab', 'c', "'"]]
        gt_offsets = [[(0, 2), (2, 5), (5, 6), (6, 8), (8, 9), (9, 12), (12, 13), (13, 17),
                       (17, 21), (21, 25), (25, 27), (26, 27), (26, 27), (27, 28), (28, 29),
                       (28, 29), (28, 29), (28, 29), (29, 30), (30, 31), (30, 31), (30, 31),
                       (30, 31), (31, 32), (32, 33), (32, 33), (32, 33), (32, 33), (33, 35)],
                      [(0, 1), (1, 2), (2, 3), (3, 5), (5, 6), (6, 7), (7, 8), (8, 11), (11, 17),
                       (17, 18), (17, 18), (17, 18), (18, 19), (18, 19), (18, 19), (19, 20),
                       (19, 20), (19, 20), (20, 21), (21, 22), (22, 23)],
                      [(0, 1), (1, 2), (2, 3), (3, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 11),
                       (11, 13), (13, 15), (15, 16), (16, 18), (18, 20), (20, 22), (22, 23),
                       (23, 25), (25, 27), (27, 30), (30, 31), (31, 32), (32, 35), (35, 36),
                       (36, 37), (37, 39), (39, 40), (40, 42), (42, 43), (43, 44),
                       (44, 47), (47, 48), (48, 51), (51, 53), (53, 56), (56, 57),
                       (57, 58), (58, 59), (59, 60), (60, 61), (61, 62), (62, 63),
                       (63, 65), (65, 66), (66, 67)]]
        gt_decode = ["Hello, y'all! How are you Ⅷ 😁 😁 😁 ?",
                     'GluonNLP is great！！！!!!',
                     "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# 'abc'"]
        verify_encode_token(tokenizer, SUBWORD_TEST_SAMPLES, gt_tokenized)
        verify_pickleble(tokenizer, HuggingFaceTokenizer)
        verify_encode_token_with_offsets(tokenizer, SUBWORD_TEST_SAMPLES, gt_offsets)
        verify_decode_hf(tokenizer, SUBWORD_TEST_SAMPLES, gt_decode)


def test_tokenizers_create():
    tokenizer = gluonnlp.data.tokenizers.create('moses', 'en')
    tokenizer.encode('hello world!')