前言

仅记录学习过程，有问题欢迎讨论

利用rnn实现分词效果(感觉十分依赖词数据)
使用jieba分词好的数据做样本

• pip install jieba

代码

import jieba
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader"""
基于pytorch的网络编写一个分词模型
我们使用jieba分词的结果作为训练数据
看看是否可以得到一个效果接近的神经网络模型中文分词缺点：
1.对词表极为依赖，如果没有词表，则无法进行；如果词表中缺少需要的词，结果也不会正确
2.切分过程中不会关注整个句子表达的意思，只会将句子看成一个个片段
3.如果文本中出现一定的错别字，会造成一连串影响
4.对于人名等的无法枚举实体词无法有效的处理"""class TorchModel(nn.Module):def __init__(self, vocab, input_dim, hidden_size, rnn_layer_size):super(TorchModel, self).__init__()self.emb = nn.Embedding(len(vocab) + 1, input_dim)# 多层rnn效果会比 单层好self.rnn = nn.RNN(input_size=input_dim,hidden_size=hidden_size,num_layers=rnn_layer_size,batch_first=True)# 不能使用pool# self.pool = nn.AvgPool1d(sentence_length)# 输出为0/1 2分类的self.classify = nn.Linear(hidden_size, 2)# -1 不参与计算self.loss = nn.CrossEntropyLoss(ignore_index=-1)def forward(self, x, y=None):x = self.emb(x)x, _ = self.rnn(x)# 用 polling 层# x= self.pool(x.transpose(1,2)).squeeze()y_pred = self.classify(x)if y is not None:# y_pred : n,class_num [[1,2,3][3,2,1]]# y : n                 [0     ,1     ]# 20*20*2===>view ===> 400 * 2   y===> 400 *1return self.loss(y_pred.view(-1, 2), y.view(-1))else:return y_pred# 使用jieba获取切分好的数据 来作为样本数据
# 我爱你们 === 1,1,0,1
def sequence_to_label(sentence):words = jieba.lcut(sentence)labels = [0] * len(sentence)pointer = 0for word in words:pointer += len(word)labels[pointer - 1] = 1return labels# 读取给定词表数据 构建字符集
def build_vocab(path):vocab = {}with open(path, encoding="utf8") as f:for index, line in enumerate(f):char = line.strip()vocab[char] = index + 1vocab['unk'] = len(vocab) + 1return vocabclass Dataset:def __init__(self, vocab, corpus_path, max_length):self.vocab = vocabself.corpus_path = corpus_pathself.max_length = max_lengthself.load()# 构建数据集def load(self):# data 的结构为 [x,y]self.data = []with open(self.corpus_path, encoding="utf8") as f:for line in f:vocab = self.vocab# 转化为 切分好的数据 yy = sequence_to_label(line)# 转化为数字x = [vocab.get(char, vocab['unk']) for char in line]# 都 标准化为最大长度x, y = self.padding(x, y)self.data.append([torch.LongTensor(x), torch.LongTensor(y)])# 使用部分数据做展示，使用全部数据训练时间会相应变长if len(self.data) > 10000:breakdef padding(self, x, y):# 长了就截取x = x[:self.max_length]# 短了就 补0x += [0] * (self.max_length - len(x))y = y[:self.max_length]# y 不能用 0y += [-1] * (self.max_length - len(y))return x, y# 为了给 data_load 使用 做小批量数据分割def __len__(self):return len(self.data)def __getitem__(self, item):return self.data[item]def build_dataset(vocab, corpus_path, max_length, batch_size):dataset = Dataset(vocab, corpus_path, max_length)# shuffle 随机打乱样本data_loader = DataLoader(dataset, shuffle=True, batch_size=batch_size)  # torchreturn data_loaderdef main():batch_size = 20lr = 1e-3epoch_size = 10vocab = build_vocab("D:\\NLP\\test\\week4\\chars.txt")hidden_size = 100# 每个字符的维度input_dim = 20rnn_layer_size = 2# 样本最大长度max_length = 20model = TorchModel(vocab, input_dim, hidden_size, rnn_layer_size)optim = torch.optim.Adam(model.parameters(), lr=lr)# 语料库（样本数据）路径corpus_path = "D:\\NLP\\test\\week4\\corpus.txt"dataiter = build_dataset(vocab, corpus_path, max_length, batch_size)for epoch in range(epoch_size):epoch_loss = []model.train()for x, y_true in dataiter:loss = model(x, y_true)loss.backward()optim.step()optim.zero_grad()epoch_loss.append(loss.item())print("第%d轮 loss = %f" % (epoch + 1, np.mean(epoch_loss)))# save modeltorch.save(model.state_dict(), "model.pth")return# 最终预测
def predict(model_path, vocab_path, input_strings):# 配置保持和训练时一致char_dim = 20  # 每个字的维度hidden_size = 100  # 隐含层维度num_rnn_layers = 2  # rnn层数vocab = build_vocab(vocab_path)  # 建立字表（字符集）model = TorchModel(vocab, char_dim, hidden_size, num_rnn_layers)  # 建立模型model.load_state_dict(torch.load(model_path))  # 加载训练好的模型权重model.eval()for input_string in input_strings:# 逐条预测x = [vocab.get(char, vocab['unk']) for char in input_string]with torch.no_grad():result = model.forward(torch.LongTensor([x]))[0]result = torch.argmax(result, dim=-1)  # 预测出的01序列# 在预测为1的地方切分，将切分后文本打印出来for index, p in enumerate(result):if p == 1:print(input_string[index], end=" ")else:print(input_string[index], end="")print()if __name__ == '__main__':main()# input_strings = ["同时国内有望出台新汽车刺激方案",#                  "沪胶后市有望延续强势",#                  "经过两个交易日的强势调整后",#                  "昨日上海天然橡胶期货价格再度大幅上扬"]# predict("model.pth", "D:\\NLP\\test\\week4\\chars.txt", input_strings)