How improve Textcat-multi classify result accuracy used by jieba or pkusage?

[wuye251]

I'm use spacy.chinese model with jieba segmenter. But I find result is very bad. I don't know the reason. Put my code below. Thanks the good soul help! Spacy version:3.6.1 Python Version:3.10.

import spacy
from spacy.training import Example
from spacy.lang.zh import Chinese
import spacy
from spacy.training import Example
from spacy.lang.zh import Chinese
from spacy.pipeline.textcat_multilabel import DEFAULT_MULTI_TEXTCAT_MODEL
import random
import json

# use the pretrained model will get error when i'm train with new dataset.
# optimizer = nlp.initialize() will get pkusge error:
# Unable to load pkuseg model from: assets/pkuseg_model
# FileNotFoundError: [Errno 2] No such file or directory: 'assets/pkuseg_model/unigram_word.txt'
# Maybe i need to download it?
# nlp = spacy.load("zh_core_web_sm") 

# cfg = {"segmenter": "jieba"}
# nlp = Chinese.from_config({"nlp": {"tokenizer": cfg}})

# cfg = {"segmenter": "pkuseg"}
# nlp = Chinese.from_config({"nlp": {"tokenizer": cfg}})
# nlp.tokenizer.initialize(pkuseg_model="mixed")

# # so i'm use blank model to train and test.
# # i don't know which of the two is better.
# nlp = spacy.blank("zh")

# # add textcat component to the pipeline
# textcat = nlp.add_pipe("textcat_multilabel", config={"model": DEFAULT_MULTI_TEXTCAT_MODEL})

# Jieba
cfg = {"segmenter": "jieba"}
nlp = Chinese.from_config({"nlp": {"tokenizer": cfg}})
textcat = nlp.add_pipe("textcat_multilabel", config={"model": DEFAULT_MULTI_TEXTCAT_MODEL})
# # PKUSeg with "mixed" model provided by pkuseg
# cfg = {"segmenter": "pkuseg"}
# nlp = Chinese.from_config({"nlp": {"tokenizer": cfg}})
# nlp.tokenizer.initialize(pkuseg_model="mixed")
# textcat = nlp.add_pipe("textcat_multilabel", config={"model": DEFAULT_SINGLE_TEXTCAT_MODEL})


# add lable.
# my dataset data.
d = json.loads('[{"label_name":"差评", "content":"讨厌"},{"label_name":"好评","content":"真不错，挺好的","threshold":0.7},{"label_name":"好评","content":"很好很好很好很好很好很好很好很好很好很好很好很好很好很好","threshold":0.7},{"label_name":"好评","content":"很喜欢","threshold":0.7},{"label_name":"好评","content":"nice","threshold":0.7},{"label_name":"好评","content":"很棒","threshold":0.7},{"label_name":"好评","content":"超赞","threshold":0.7},{"label_name":"好评","content":"爱了","threshold":0.7},{"label_name":"好评","content":"好","threshold":0.7},{"label_name":"好评","content":"差","threshold":0.7},{"label_name":"好评","content":"不行","threshold":0.7},{"label_name":"好评","content":"次品","threshold":0.7},{"label_name":"好评","content":"很糟糕","threshold":0.7},{"label_name":"好评","content":"无语","threshold":0.7},{"label_name":"咨询商品","content":"这个多少钱？","threshold":0.7},{"label_name":"咨询商品","content":"有几种颜色？","threshold":0.7},{"label_name":"咨询商品","content":"包邮吗？","threshold":0.7},{"label_name":"咨询商品","content":"有运费险吗？","threshold":0.7},{"label_name":"咨询商品","content":"有什么尺码？","threshold":0.7},{"label_name":"售后-投诉","content":"质量真差","threshold":0},{"label_name":"售后-投诉","content":"颜色不准","threshold":0},{"label_name":"售后-投诉","content":"面料粗糙","threshold":0},{"label_name":"售后-点赞","content":"质量差","threshold":0.72},{"label_name":"售后-点赞","content":"差评","threshold":0.72},{"label_name":"售后-点赞","content":"很棒","threshold":0.72},{"label_name":"售后-点赞","content":"很好看","threshold":0.72},{"label_name":"售后-点赞","content":"超级合适","threshold":0.72},{"label_name":"售后-退换","content":"拍大了想换下","threshold":0.73},{"label_name":"售后-退换","content":"拍小了想退下","threshold":0.73},{"label_name":"售后-退换","content":"不想要了","threshold":0.73},{"label_name":"查询信用卡账单","content":"查询信用卡账单","threshold":0.7},{"label_name":"查询信用卡账单","content":"查询账单","threshold":0.7},{"label_name":"查询信用卡账单","content":"我的信用卡账单","threshold":0.7},{"label_name":"订票","content":"我想订票","threshold":0.7},{"label_name":"订票","content":"订票","threshold":0.7},{"label_name":"订票","content":"订火车票","threshold":0.7},{"label_name":"订票","content":"订飞机票","threshold":0.7},{"label_name":"球类运动","content":"打篮球","threshold":0},{"label_name":"球类运动","content":"玩乒乓球","threshold":0},{"label_name":"球类运动","content":"踢足球","threshold":0},{"label_name":"球类运动","content":"打羽毛球","threshold":0},{"label_name":"天气","content":"多云","threshold":0},{"label_name":"天气","content":"晴天","threshold":0},{"label_name":"天气","content":"阴天","threshold":0},{"label_name":"天气","content":"晴转阴","threshold":0},{"label_name":"天气","content":"阴转小雨","threshold":0},{"label_name":"国家-编辑","content":"中国","threshold":0.7},{"label_name":"国家-编辑","content":"美国","threshold":0.7},{"label_name":"国家-编辑","content":"韩国","threshold":0.7},{"label_name":"国家-编辑","content":"日本","threshold":0.7},{"label_name":"电商-客服-售后","content":"测试3","threshold":0},{"label_name":"球类运动","content":"打排球","threshold":0},{"label_name":"国家-编辑","content":"新加坡","threshold":0.7},{"label_name":"负例语料联动","content":"仅填写必填项","threshold":0.7},{"label_name":"负例语料联动","content":"仅填写必填项","threshold":0.7},{"label_name":"表扬","content":"你在说什么","threshold":0.7},{"label_name":"负例语料联动","content":"必填项","threshold":0.7},{"label_name":"负例语料联动","content":"我是标签下的第一条负例---标签处编辑","threshold":0.7},{"label_name":"负例语料联动","content":"我是标签下的第二条负例--负例维护处编辑","threshold":0.7},{"label_name":"负例语料联动","content":"我是标签下的第三条负例","threshold":0.7},{"label_name":"负例语料联动","content":"我是负例维护下新增的第一条负例","threshold":0.7},{"label_name":"负例语料联动","content":"新增的内容","threshold":0.7},{"label_name":"电商-客服-售后","content":"阿道夫啊23rxxc","threshold":0},{"label_name":"电商-客服-售后","content":"dasasasasf_+_++d","threshold":0},{"label_name":"电商-客服-售后","content":"dddd.0.0.0.","threshold":0},{"label_name":"电商-客服-售后","content":"3323","threshold":0},{"label_name":"电商-客服-售后","content":"232ggggg","threshold":0},{"label_name":"电商-客服-售后","content":"ddddddd","threshold":0},{"label_name":"电商-客服-售后","content":"23r23","threshold":0},{"label_name":"电商-客服-售后","content":"一二三四五六七八九十一二三四五六七八九十一二三四五六七八九十一二三四五六七八九十一二三四五六七八九十一二三四五六七八九十一二三四五六七八九十一二三四五六七八九十一二三四五六七八九十一二三四五六七八九十","threshold":0},{"label_name":"电商-客服-售前","content":"电商-客服-售前1","threshold":0},{"label_name":"电商-客服-售前","content":"电商-客服-售前2","threshold":0},{"label_name":"电商-客服-售前","content":"电商-客服-售前3","threshold":0},{"label_name":"验证未训练时字段展示","content":"验证未训练时字段展示","threshold":0},{"label_name":"验证未训练时字段展示","content":"验证未训练时字段展示","threshold":0},{"label_name":"验证未训练时字段展示","content":"验证未训练时字段展示","threshold":0}]')
train_data = list()
text_label = dict()
for v in d:
    train_data.append((v["content"],{"cats":{v["label_name"]:1.0}}))
    text_label[v["label_name"]] = 1

for k in text_label:
    textcat.add_label(k)

# convert data to Example objects.
train_examples = []
for text, annotations in train_data:
    doc = nlp.make_doc(text)
    example = Example.from_dict(doc, annotations)
    train_examples.append(example)

# start training.
n_iter = 8
# optimizer = nlp.resume_training()
# optimizer = nlp.initialize()
# other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "textcat_multilabel"]
# with nlp.disable_pipes(*other_pipes):  # only use textcat_multilabel for training
with nlp.select_pipes(enable=["textcat_multilabel"]):
    optimizer = nlp.initialize()
    for i in range(n_iter):
        losses = {}
        random.shuffle(train_examples) 
        for example in train_examples:
            # nlp.rehearse([example], sgd=optimizer)
            nlp.update([example], sgd=optimizer, losses=losses)


scores = textcat.predict([eg.predicted for eg in train_examples])
loss, d_loss = textcat.get_loss(train_examples, scores)

# test or predict.
test_data = [
    "今天吃什么",
    "我讨厌这部电影",
    "查询账单"
]

for text in test_data:
    doc = nlp(text)
    print(text, doc.cats)

output

今天吃什么 {'差评': 0.8335129022598267, '好评': 0.9697579741477966, '咨询商品': 0.9898377656936646, '售后-投诉': 0.68087238073349, '售后-点赞': 0.8260092735290527, '售后-退换': 0.9918733239173889, '查询信用卡账单': 0.3684355318546295, '订票': 0.9947301149368286, '球类运动': 0.2311471402645111, '天气': 0.9035037755966187, '国家-编辑': 0.5038042664527893, '电商-客服-售后': 0.7437667846679688, '负例语料联动': 0.9357748031616211, '表扬': 0.9886539578437805, '电商-客服-售前': 0.28681936860084534, '验证未训练时字段展示': 0.8502825498580933}

我讨厌这部电影 {'差评': 0.9159942269325256, '好评': 0.9901401400566101, '咨询商品': 0.9551239013671875, '售后-投诉': 0.8971258997917175, '售后-点赞': 0.3419587016105652, '售后-退换': 0.7432483434677124, '查询信用卡账单': 0.8540969491004944, '订票': 0.9098125100135803, '球类运动': 0.29566851258277893, '天气': 0.4786415100097656, '国家-编辑': 0.5186322331428528, '电商-客服-售后': 0.9906345009803772, '负例语料联动': 0.9211296439170837, '表扬': 0.5615023374557495, '电商-客服-售前': 0.33090463280677795, '验证未训练时字段展示': 0.2705373764038086}

查询账单 {'差评': 0.9369145035743713, '好评': 0.9800768494606018, '咨询商品': 0.9792625904083252, '售后-投诉': 0.5187459588050842, '售后-点赞': 0.8109645247459412, '售后-退换': 0.9886360168457031, '查询信用卡账单': 0.9981147050857544, '订票': 0.8363357186317444, '球类运动': 0.9094472527503967, '天气': 0.6167519092559814, '国家-编辑': 0.3675537109375, '电商-客服-售后': 0.9292000532150269, '负例语料联动': 0.8542944192886353, '表扬': 0.49599653482437134, '电商-客服-售前': 0.8967403769493103, '验证未训练时字段展示': 0.7517518997192383}