标签：BERT 训练 pred paragraphs len mlm tokens positions 69
点击查看代码
import os
import random
import torch
from d2l import torch as d2l
#@save
d2l.DATA_HUB['wikitext-2'] = (
'https://s3.amazonaws.com/research.metamind.io/wikitext/'
'wikitext-2-v1.zip', '3c914d17d80b1459be871a5039ac23e752a53cbe')
#@save
def _read_wiki(data_dir):
file_name = os.path.join(data_dir, 'wiki.train.tokens')
with open(file_name, 'r', encoding = 'utf-8') as f:
lines = f.readlines()
# 大写字母转换为小写字母
paragraphs = [line.strip().lower().split(' . ')
for line in lines if len(line.split(' . ')) >= 2]
random.shuffle(paragraphs)
return paragraphs
# 生成下一句预测任务的数据
#@save
def _get_next_sentence(sentence, next_sentence, paragraphs):
# 预测一个句子对中两个句子是不是相邻
# 50%概率选择相邻句子对，50%概率选择随机句子对
# print('type(paragraphs) : ', type(paragraphs))
# print('len(paragraphs) : ', len(paragraphs))
# print('paragraphs : ', paragraphs)
# print('random.choice(paragraphs) : ', random.choice(paragraphs))
# print('random.choice(random.choice(paragraphs)) : ', random.choice(random.choice(paragraphs)))
if random.random() < 0.5:
is_next = True
else:
# paragraphs是三重列表的嵌套
next_sentence = random.choice(random.choice(paragraphs))
is_next = False
return sentence, next_sentence, is_next
# 下一个句子预测
#@save
def _get_nsp_data_from_paragraph(paragraph, paragraphs, vocab, max_len):
nsp_data_from_paragraph = []
for i in range(len(paragraph) - 1):
tokens_a, tokens_b, is_next = _get_next_sentence(paragraph[i], paragraph[i + 1], paragraphs)
# 考虑1个'<cls>'词元和2个'<sep>'词元
# 只会终止执行本次循环中剩下的代码，直接从下一次循环继续执行。
if len(tokens_a) + len(tokens_b) + 3 > max_len:
continue
# 转换为BERT输入
tokens, segments = d2l.get_tokens_and_segments(tokens_a, tokens_b)
# 生产下一个句子预测数据
nsp_data_from_paragraph.append((tokens, segments, is_next))
return nsp_data_from_paragraph
# 生成遮蔽语言模型任务的数据
# 从BERT输入序列生成遮蔽语言模型的训练样本
#@save
def _replace_mlm_tokens(tokens, candidate_pred_positions, num_mlm_preds, vocab):
"""
:param tokens: BERT输入序列的词元的列表
:param candidate_pred_positions:不包括特殊词元的BERT输入序列的词元索引的列表（特殊词元在遮蔽语言模型任务中不被预测）
:param num_mlm_preds:预测的数量（选择15%要预测的随机词元）
:param vocab:
:return:替换后的输入词元、发生预测的词元索引和这些预测的标签
"""
# 为遮蔽语言模型的输入创建新的词元副本，其中输入可能包含替换的“<mask>”或随机词元
mlm_input_tokens = [token for token in tokens]
pred_positions_and_labels = []
# 打乱后用于在遮蔽语言模型任务中获取15%的随机词元进行预测
random.shuffle(candidate_pred_positions)
for mlm_pred_position in candidate_pred_positions:
if len(pred_positions_and_labels) >= num_mlm_preds:
break
masked_token = None
# 80%的概率：将词替换为“<mask>”词元
if random.random() < 0.8:
masked_token = '<mask>'
else:
# 10%的概率：保持词不变
if random.random() < 0.5:
masked_token = tokens[mlm_pred_position]
# 10%的概率：用随机词替换该词
else:
masked_token = random.choice(vocab.idx_to_token)
mlm_input_tokens[mlm_pred_position] = masked_token
pred_positions_and_labels.append((mlm_pred_position, tokens[mlm_pred_position]))
return mlm_input_tokens, pred_positions_and_labels
# 将BERT输入序列（tokens）作为输入，并返回输入词元的索引、发生预测的词元索引以及这些预测的标签索引。
#@save
def _get_mlm_data_from_tokens(tokens, vocab):
candidate_pred_positions = []
# tokens是一个字符串列表
for i, token in enumerate(tokens):
# 在遮蔽语言模型任务中不会预测特殊词元
if token in ['<cls>', '<sep>']:
continue
candidate_pred_positions.append(i)
# 遮蔽语言模型任务中预测15%的随机词元
num_mlm_preds = max(1, round(len(tokens) * 0.15))
mlm_input_tokens, pred_positions_and_labels = _replace_mlm_tokens(
tokens, candidate_pred_positions, num_mlm_preds, vocab)
# pred_positions_and_labels.append((mlm_pred_position, tokens[mlm_pred_position]))
pred_positions_and_labels = sorted(pred_positions_and_labels,key=lambda x: x[0])
pred_positions = [v[0] for v in pred_positions_and_labels]
mlm_pred_labels = [v[1] for v in pred_positions_and_labels]
# vocab[mlm_input_tokens] - > mask
return vocab[mlm_input_tokens], pred_positions, vocab[mlm_pred_labels]
# 将文本转换为预训练数据集
#@save
def _pad_bert_inputs(examples, max_len, vocab):
max_num_mlm_preds = round(max_len * 0.15)
all_token_ids, all_segments, valid_lens, = [], [], []
# 拼接 all_mlm_weights = 0 否则为1
all_pred_positions, all_mlm_weights, all_mlm_labels = [], [], []
nsp_labels = []
# _get_nsp_data_from_paragraph
# return nsp_data_from_paragraph -- (tokens, segments, is_next)
# _get_mlm_data_from_tokens
# return vocab[mlm_input_tokens], pred_positions, vocab[mlm_pred_labels]
for (token_ids, pred_positions, mlm_pred_label_ids, segments, is_next) in examples:
all_token_ids.append(torch.tensor(token_ids + [vocab['<pad>']] * (max_len - len(token_ids)), dtype=torch.long))
all_segments.append(torch.tensor(segments + [0] * (max_len - len(segments)), dtype=torch.long))
# valid_lens不包括'<pad>'的计数
valid_lens.append(torch.tensor(len(token_ids), dtype=torch.float32))
all_pred_positions.append(torch.tensor(pred_positions + [0] * (max_num_mlm_preds - len(pred_positions)), dtype=torch.long))
# 填充词元的预测将通过乘以0权重在损失中过滤掉
all_mlm_weights.append(torch.tensor([1.0] * len(mlm_pred_label_ids) + [0.0] * (max_num_mlm_preds - len(pred_positions)),
dtype=torch.float32))
all_mlm_labels.append(torch.tensor(mlm_pred_label_ids + [0] * (max_num_mlm_preds - len(mlm_pred_label_ids)), dtype=torch.long))
nsp_labels.append(torch.tensor(is_next, dtype=torch.long))
return (all_token_ids, all_segments, valid_lens, all_pred_positions, all_mlm_weights, all_mlm_labels, nsp_labels)
#@save
class _WikiTextDataset(torch.utils.data.Dataset):
def __init__(self, paragraphs, max_len):
# 输入paragraphs[i]是代表段落的句子字符串列表；
# 而输出paragraphs[i]是代表段落的句子列表，其中每个句子都是词元列表
paragraphs = [d2l.tokenize(paragraph, token='word') for paragraph in paragraphs]
sentences = [sentence for paragraph in paragraphs for sentence in paragraph]
self.vocab = d2l.Vocab(sentences, min_freq=5, reserved_tokens=['<pad>', '<mask>', '<cls>', '<sep>'])
# 获取下一句子预测任务的数据
examples = []
for paragraph in paragraphs:
# def _get_nsp_data_from_paragraph(paragraph, paragraphs, vocab, max_len):
# nsp_data_from_paragraph.append((tokens, segments, is_next))
# return nsp_data_from_paragraph
# extend() 函数用于在列表末尾一次性追加另一个序列中的多个值（用新列表扩展原来的列表）。
examples.extend(_get_nsp_data_from_paragraph(paragraph, paragraphs, self.vocab, max_len))
# 获取遮蔽语言模型任务的数据
# def _get_mlm_data_from_tokens(tokens, vocab):
# return vocab[mlm_input_tokens], pred_positions, vocab[mlm_pred_labels]
examples = [(_get_mlm_data_from_tokens(tokens, self.vocab) + (segments, is_next)) for tokens, segments, is_next in examples]
# 填充输入
# for (token_ids, pred_positions, mlm_pred_label_ids, segments, is_next) in examples:
(self.all_token_ids, self.all_segments, self.valid_lens, self.all_pred_positions, self.all_mlm_weights,
self.all_mlm_labels, self.nsp_labels) = _pad_bert_inputs(examples, max_len, self.vocab)
def __getitem__(self, idx):
return (self.all_token_ids[idx], self.all_segments[idx],
self.valid_lens[idx], self.all_pred_positions[idx],
self.all_mlm_weights[idx], self.all_mlm_labels[idx],
self.nsp_labels[idx])
def __len__(self):
return len(self.all_token_ids)
#@save
def load_data_wiki(batch_size, max_len):
"""加载WikiText-2数据集"""
num_workers = 0
data_dir = d2l.download_extract('wikitext-2', 'wikitext-2')
paragraphs = _read_wiki(data_dir)
train_set = _WikiTextDataset(paragraphs, max_len)
train_iter = torch.utils.data.DataLoader(train_set, batch_size, shuffle=True, num_workers=num_workers)
return train_iter, train_set.vocab
batch_size, max_len = 512, 64
train_iter, vocab = load_data_wiki(batch_size, max_len)
# tokens_X torch.Size([512, 64])
# segments_X torch.Size([512, 64])
# valid_lens_x torch.Size([512])
# pred_positions_X torch.Size([512, 10]) 预测多少个位置 64*0.15
# mlm_weights_X torch.Size([512, 10])
# mlm_Y torch.Size([512, 10])
# nsp_y torch.Size([512])
for (tokens_X, segments_X, valid_lens_x, pred_positions_X, mlm_weights_X, mlm_Y, nsp_y) in train_iter:
print(tokens_X.shape, segments_X.shape, valid_lens_x.shape, pred_positions_X.shape, mlm_weights_X.shape, mlm_Y.shape, nsp_y.shape)
break
标签：BERT,训练,pred,paragraphs,len,mlm,tokens,positions,69 来源： https://www.cnblogs.com/g932150283/p/16597095.html
本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享； 2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关； 3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关； 4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除； 5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。