用keras.preprocessing.text.Tokenizer標(biāo)記中文文本

keras.preprocessing.text.Tokenizer不能正確處理中文文本。如何修改它以處理中文文本？

from keras.preprocessing.text import Tokenizer
def fit_get_tokenizer(data, max_words):
    tokenizer = Tokenizer(num_words=max_words, filters='!"#%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n')
    tokenizer.fit_on_texts(data)
    return tokenizer
tokenizer = fit_get_tokenizer(df.sentence,max_words=150000)
print('Total number of words: ', len(tokenizer.word_index))
vocabulary_inv = {}
for word in tokenizer.word_index:
    vocabulary_inv[tokenizer.word_index[word]] = word
print(vocabulary_inv)