标签：Bert torch loss labels preds pytorch train total fine
基于Bert预训练模型的文本分类fine tune
环境
- python==3.7
- torch==1.7.1
- transformers==4.9.2
- scikit-learn==0.21.3
- tensorboard==2.5.0
- pandas
- numpy
构建数据集
将数据放到如下图格式的dataframe中，label对应的数字为每种类别的下标。
random seed设置
import torch
import numpy as np
random_seed = 2018
np.random.seed(random_seed)
torch.manual_seed(random_seed)
torch.cuda.manual_seed_all(random_seed)
torch.backends.cudnn.deterministic = True
数据集划分
# 训练集 验证集 测试集划分
# stratify参数为分层抽样，根据label的种类分布，按比例抽样
train_text, temp_text, train_labels, temp_labels = train_test_split(df['text'], df['label'], random_state=random_seed, test_size=0.3, stratify=df['label'])
val_text, test_text, val_labels, test_labels = train_test_split(temp_text, temp_labels, random_state=random_seed, test_size=0.5, stratify=temp_labels)
预训练模型加载
import transformers
from transformers import AutoModel, BertTokenizerFast
# 加载预训练模型
pretrained_model_path = "bert-base-uncased"
bert = AutoModel.from_pretrained(pretrained_model_path)
tokenizer = BertTokenizerFast.from_pretrained(pretrained_model_path)
句子长度分布
import pandas as pd
# 查看训练集中的句子长度，决定pad的长度
seq_len = [len(i.split()) for i in train_text]
seq_df = pd.DataFrame(seq_len)
seq_df.describe()
如上图所示，75%的句子长度都在46，因此后面pad长度可设置为50左右
tokenizer
padding_len = 50
# 经过tokenizer之后会将文字变成数字的形式，其中也包含了attention mask
tokens_train = tokenizer.batch_encode_plus(train_text.tolist(), max_length = padding_len, pad_to_max_length=True, truncation=True)
tokens_test = tokenizer.batch_encode_plus(test_text.tolist(), max_length = padding_len, pad_to_max_length=True, truncation=True)
tokens_val = tokenizer.batch_encode_plus(val_text.tolist(), max_length = padding_len, pad_to_max_length=True, truncation=True)
# to tensors
train_seq = torch.tensor(tokens_train['input_ids'])
train_mask = torch.tensor(tokens_train['attention_mask'])
train_y = torch.tensor(train_labels.tolist())
val_seq = torch.tensor(tokens_val['input_ids'])
val_mask = torch.tensor(tokens_val['attention_mask'])
val_y = torch.tensor(val_labels.tolist())
test_seq = torch.tensor(tokens_test['input_ids'])
test_mask = torch.tensor(tokens_test['attention_mask'])
test_y = torch.tensor(test_labels.tolist())
Dataset Dataloder
# create dataloader
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
batch_size = 16
train_data = TensorDataset(train_seq, train_mask, train_y)
# sampler for sampling the data during training
train_sampler = RandomSampler(train_data)
# dataLoader for train set
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
val_data = TensorDataset(val_seq, val_mask, val_y)
val_sampler = SequentialSampler(val_data)
val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size=batch_size)
冻结预训练模型参数
# freeze all the parameters
for param in bert.parameters():
param.requires_grad = False
模型定义
# 模型定义
class BertClassifier(nn.Module):
def __init__(self, bert_pretrained_model):
super(BertClassifier, self).__init__()
self.bert = bert_pretrained_model
self.dropout = nn.Dropout(0.1)
self.relu = nn.ReLU()
self.fc1 = nn.Linear(768, 512)
self.fc2 = nn.Linear(512, class_num)
self.softmax = nn.LogSoftmax(dim=1)
def forward(self, sent_id, mask):
bert_res = self.bert(sent_id, attention_mask=mask)
# _, cls_hs = bert_res
cls_hs = bert_res.pooler_output
x = self.fc1(cls_hs)
x = self.relu(x)
x = self.dropout(x)
x = self.fc2(x)
x = self.softmax(x)
return x
初始化
device = torch.device("cuda:1") if torch.cuda.is_available() else torch.device("cpu")
model = BertClassifier(bert)
model.to(device)
optimizer
from transformers import AdamW
# bert论文推荐学习率2e-5, 3e-5, 5e-5
optimizer = AdamW(model.parameters(), lr=2e-5)
查看每种类别的weights
针对label分布不均衡，在损失函数里传weights
from sklearn.utils.class_weight import compute_class_weight
group_size = df.groupby(["label"]).size()
print(group_size)
class_weights = compute_class_weight('balanced', np.unique(train_labels), train_labels)
损失函数
weights= torch.tensor(class_weights, dtype=torch.float)
weights = weights.to(device)
cross_entropy = nn.NLLLoss(weight=weights)
训练
from datetime import datetime
from sklearn.metrics import f1_score
from torch.utils.tensorboard import SummaryWriter
saved_path = '/data/yuhengshi/saved_model/9_aug_other_roberta_val185.plk'
writer = SummaryWriter("/data/yuhengshi/saved_model")
def evaluate():
print(f"""evaluate start""")
model.eval()
total_loss, total_accuracy = 0, 0
total_preds = []
total_labels = []
for step, batch in enumerate(val_dataloader):
batch = [t.to(device) for t in batch]
sent_id, mask, labels = batch
with torch.no_grad():
preds = model(sent_id, mask)
loss = cross_entropy(preds, labels)
total_loss = total_loss + loss.item()
preds = preds.detach().cpu().numpy()
total_preds.append(preds)
labels = labels.detach().cpu().numpy()
total_labels.append(labels)
# # 每50次
# if step % 50 == 0 and step != 0:
# print()
avg_loss = total_loss/len(val_dataloader)
total_preds = np.concatenate(total_preds, axis=0)
total_labels = np.concatenate(total_labels, axis=0)
val_f1 = f1_score(total_labels, np.argmax(total_preds, axis=1), average='micro')
return avg_loss, val_f1, total_preds, total_labels
def train(epochs=10):
best_valid_loss = float('inf')
start_time = datetime.now()
for epoch in range(epochs):
print(f"""Epoch {epoch}/{epochs} start""")
model.train()
total_loss, total_accuracy, total_batch = 0, 0, 0
# 存预测结果
total_preds = []
total_labels = []
for step, batch in enumerate(train_dataloader):
batch = [x.to(device) for x in batch]
sent_id, mask, labels = batch
model.zero_grad()
preds = model(sent_id, mask)
loss = cross_entropy(preds, labels)
total_loss = total_loss + loss.item()
loss.backward()
# 防梯度爆炸
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
# model predictions are stored on GPU. So, push it to CPU
preds = preds.detach().cpu().numpy()
total_preds.append(preds)
total_labels.append(labels.detach().cpu().numpy())
# 每50次
if step != 0 and step % 50 == 0:
current_time = datetime.now()
print(f"""Epoch {epoch}/{epochs}, step:{step}, train_loss:{loss}, cost_time:{current_time-start_time}""")
# 每个epoch结束 计算平均loss
total_preds = np.concatenate(total_preds, axis=0)
total_labels = np.concatenate(total_labels, axis=0)
avg_loss = total_loss/len(train_dataloader)
preds = np.argmax(total_preds, axis=1)
train_f1 = f1_score(total_labels, preds, average='micro')
valid_loss, valid_f1, valid_preds, valid_labels = evaluate()
valid_preds = np.argmax(valid_preds, axis=1)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
torch.save(model.state_dict(), saved_path)
current_time = datetime.now()
metrics_report = classification_report(valid_labels, valid_preds)
# tensorboard
writer.add_scalar("loss/train", avg_loss, epoch)
writer.add_scalar("loss/valid", valid_loss, epoch)
writer.add_scalar("f1/train", train_f1, epoch)
writer.add_scalar("f1/valid", valid_f1, epoch)
print(f"Epoch {epoch}/{epochs}, train_loss: {avg_loss}, train_f1:{train_f1}\n"
f"valid_loss: {valid_loss}, valid_f1: {valid_f1}\n"
f"best_valid_loss: {best_valid_loss}, cost_time: {current_time-start_time}\n"
f"{metrics_report}")
print(f"=====================================")
total_batch += 1
writer.close()
train()
预测
def predict(model_path, test_data, test_label):
model.load_state_dict(torch.load(model_path))
with torch.no_grad():
preds = model(test_seq.to(device), test_mask.to(device))
preds = preds.detach().cpu().numpy()
preds = np.argmax(preds, axis=1)
return preds
tensorboard使用
tensorboard --logdir=./log2 --port 8088
标签：Bert,torch,loss,labels,preds,pytorch,train,total,fine 来源： https://blog.csdn.net/yuhengshi/article/details/119961539
本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享； 2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关； 3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关； 4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除； 5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。