尧图网站建设 
目标训练基于transformer的单向语言模型并完成文本生成。示意图内容importmathimportosimporttorchimporttorch.nnasnnimporttorch.nn.functionalasF# # 1. 模型超参数# torch.manual_seed(42)batch_size32block_size64# 最大上下文长度n_embd128# 词向量维度n_head4# 注意力头数n_layer3# Transformer 层数dropout0.1learning_rate3e-4max_iters1500eval_interval300device(cudaiftorch.cuda.is_available()elsempsiftorch.backends.mps.is_available()elsecpu)print(当前设备:,device)# # 2. 准备训练数据# # 建议准备自己的中文语料 data.txt。# 没有文件时使用演示语料仅用于验证训练流程。ifos.path.exists(data.txt):withopen(data.txt,r,encodingutf-8)asf:textf.read()else:sentences[今天天气很好我们一起出去散步。,人工智能正在改变我们的生活。,深度学习是机器学习的重要分支。,自然语言处理可以帮助计算机理解文本。,神经网络可以通过训练学习数据中的规律。,Transformer使用注意力机制处理序列信息。,语言模型的任务是预测下一个字符。,机器学习需要大量数据进行训练。,学习编程需要不断地练习和思考。,使用PyTorch可以方便地构建深度学习模型。,我们正在学习如何训练一个语言模型。,文本生成是自然语言处理的重要应用。,]text\n.join(sentences*200)# 字符级 Tokenizercharssorted(list(set(text)))vocab_sizelen(chars)stoi{ch:ifori,chinenumerate(chars)}itos{i:chforch,iinstoi.items()}defencode(s):return[stoi[c]forcins]defdecode(ids):return.join(itos[int(i)]foriinids)datatorch.tensor(encode(text),dtypetorch.long)# 训练集与验证集nint(0.9*len(data))train_datadata[:n]val_datadata[n:]assertmin(len(train_data),len(val_data))block_sizeprint(词表大小:,vocab_size)print(训练字符数:,len(train_data))# # 3. 构建训练批次# defget_batch(split):sourcetrain_dataifsplittrainelseval_data ixtorch.randint(0,len(source)-block_size,(batch_size,))xtorch.stack([source[i:iblock_size]foriinix])# y 相对于 x 向右移动一个位置ytorch.stack([source[i1:iblock_size1]foriinix])returnx.to(device),y.to(device)# # 4. 多头因果自注意力# classCausalSelfAttention(nn.Module):def__init__(self):super().__init__()assertn_embd%n_head0self.num_headsn_head self.head_dimn_embd//n_head self.qkvnn.Linear(n_embd,3*n_embd)self.projnn.Linear(n_embd,n_embd)self.attn_dropoutnn.Dropout(dropout)self.resid_dropoutnn.Dropout(dropout)# 下三角因果 Maskself.register_buffer(mask,torch.tril(torch.ones(block_size,block_size)).view(1,1,block_size,block_size))defforward(self,x):B,T,Cx.shape# 一次线性映射生成 Q、K、Vq,k,vself.qkv(x).split(C,dim2)# [B, T, C] - [B, Head, T, HeadDim]qq.view(B,T,self.num_heads,self.head_dim).transpose(1,2)kk.view(B,T,self.num_heads,self.head_dim).transpose(1,2)vv.view(B,T,self.num_heads,self.head_dim).transpose(1,2)# Scaled Dot-Product Attentionscoresq k.transpose(-2,-1)scoresscores/math.sqrt(self.head_dim)# 遮挡未来 Tokenscoresscores.masked_fill(self.mask[:,:,:T,:T]0,float(-inf))weightsF.softmax(scores,dim-1)weightsself.attn_dropout(weights)outweights v# 拼接所有注意力头outout.transpose(1,2).contiguous()outout.view(B,T,C)returnself.resid_dropout(self.proj(out))# # 5. 前馈神经网络# classFeedForward(nn.Module):def__init__(self):super().__init__()self.netnn.Sequential(nn.Linear(n_embd,4*n_embd),nn.GELU(),nn.Linear(4*n_embd,n_embd),nn.Dropout(dropout))defforward(self,x):returnself.net(x)# # 6. Transformer Block# classTransformerBlock(nn.Module):def__init__(self):super().__init__()self.ln1nn.LayerNorm(n_embd)self.attnCausalSelfAttention()self.ln2nn.LayerNorm(n_embd)self.ffnFeedForward()defforward(self,x):# Pre-Norm 残差连接xxself.attn(self.ln1(x))xxself.ffn(self.ln2(x))returnx# # 7. GPT 单向语言模型# classMiniGPT(nn.Module):def__init__(self):super().__init__()self.token_embeddingnn.Embedding(vocab_size,n_embd)self.position_embeddingnn.Embedding(block_size,n_embd)self.blocksnn.Sequential(*[TransformerBlock()for_inrange(n_layer)])self.ln_fnn.LayerNorm(n_embd)self.lm_headnn.Linear(n_embd,vocab_size)defforward(self,idx,targetsNone):B,Tidx.shapeassertTblock_size token_embself.token_embedding(idx)positionstorch.arange(T,deviceidx.device)pos_embself.position_embedding(positions)xtoken_embpos_emb xself.blocks(x)xself.ln_f(x)logitsself.lm_head(x)lossNoneiftargetsisnotNone:B,T,Vlogits.shape lossF.cross_entropy(logits.reshape(B*T,V),targets.reshape(B*T))returnlogits,losstorch.no_grad()defgenerate(self,idx,max_new_tokens100,temperature1.0,top_kNone):asserttemperature0self.eval()for_inrange(max_new_tokens):# 只使用最后 block_size 个 Tokenidx_condidx[:,-block_size:]logits,_self(idx_cond)# 获取最后一个位置的预测logitslogits[:,-1,:]# 温度调节logitslogits/temperature# Top-K 采样iftop_kisnotNone:kmin(top_k,logits.size(-1))values,_torch.topk(logits,k)logits[logitsvalues[:,[-1]]]-float(inf)probsF.softmax(logits,dim-1)next_tokentorch.multinomial(probs,num_samples1)idxtorch.cat([idx,next_token],dim1)returnidx# # 8. 训练与验证# modelMiniGPT().to(device)print(模型参数量:,sum(p.numel()forpinmodel.parameters()))optimizertorch.optim.AdamW(model.parameters(),lrlearning_rate)torch.no_grad()defestimate_loss():model.eval()results{}forsplitin[train,val]:lossestorch.zeros(20)foriinrange(20):xb,ybget_batch(split)_,lossmodel(xb,yb)losses[i]loss.item()results[split]losses.mean().item()model.train()returnresultsforstepinrange(max_iters):ifstep%eval_interval0orstepmax_iters-1:lossesestimate_loss()print(fstep{step:4d}| ftrain loss:{losses[train]:.4f}| fval loss:{losses[val]:.4f})xb,ybget_batch(train)logits,lossmodel(xb,yb)optimizer.zero_grad(set_to_noneTrue)loss.backward()# 梯度裁剪torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)optimizer.step()# # 9. 保存模型# torch.save({model_state_dict:model.state_dict(),stoi:stoi,itos:itos,block_size:block_size,n_embd:n_embd,n_head:n_head,n_layer:n_layer,},mini_gpt.pth)print(模型已保存mini_gpt.pth)# # 10. 文本生成# prompt人工智能# 字符级词表无法识别训练集外的字符unknownset(prompt)-set(stoi)ifunknown:raiseValueError(f提示词包含词表外字符:{unknown})contexttorch.tensor([encode(prompt)],dtypetorch.long,devicedevice)generatedmodel.generate(context,max_new_tokens100,temperature0.8,top_k10)print(\n 生成结果 )print(decode(generated[0].tolist())) week5
目标训练基于transformer的单向语言模型并完成文本生成。示意图内容importmathimportosimporttorchimporttorch.nnasnnimporttorch.nn.functionalasF# # 1. 模型超参数# torch.manual_seed(42)batch_size32block_size64# 最大上下文长度n_embd128# 词向量维度n_head4# 注意力头数n_layer3# Transformer 层数dropout0.1learning_rate3e-4max_iters1500eval_interval300device(cudaiftorch.cuda.is_available()elsempsiftorch.backends.mps.is_available()elsecpu)print(当前设备:,device)# # 2. 准备训练数据# # 建议准备自己的中文语料 data.txt。# 没有文件时使用演示语料仅用于验证训练流程。ifos.path.exists(data.txt):withopen(data.txt,r,encodingutf-8)asf:textf.read()else:sentences[今天天气很好我们一起出去散步。,人工智能正在改变我们的生活。,深度学习是机器学习的重要分支。,自然语言处理可以帮助计算机理解文本。,神经网络可以通过训练学习数据中的规律。,Transformer使用注意力机制处理序列信息。,语言模型的任务是预测下一个字符。,机器学习需要大量数据进行训练。,学习编程需要不断地练习和思考。,使用PyTorch可以方便地构建深度学习模型。,我们正在学习如何训练一个语言模型。,文本生成是自然语言处理的重要应用。,]text\n.join(sentences*200)# 字符级 Tokenizercharssorted(list(set(text)))vocab_sizelen(chars)stoi{ch:ifori,chinenumerate(chars)}itos{i:chforch,iinstoi.items()}defencode(s):return[stoi[c]forcins]defdecode(ids):return.join(itos[int(i)]foriinids)datatorch.tensor(encode(text),dtypetorch.long)# 训练集与验证集nint(0.9*len(data))train_datadata[:n]val_datadata[n:]assertmin(len(train_data),len(val_data))block_sizeprint(词表大小:,vocab_size)print(训练字符数:,len(train_data))# # 3. 构建训练批次# defget_batch(split):sourcetrain_dataifsplittrainelseval_data ixtorch.randint(0,len(source)-block_size,(batch_size,))xtorch.stack([source[i:iblock_size]foriinix])# y 相对于 x 向右移动一个位置ytorch.stack([source[i1:iblock_size1]foriinix])returnx.to(device),y.to(device)# # 4. 多头因果自注意力# classCausalSelfAttention(nn.Module):def__init__(self):super().__init__()assertn_embd%n_head0self.num_headsn_head self.head_dimn_embd//n_head self.qkvnn.Linear(n_embd,3*n_embd)self.projnn.Linear(n_embd,n_embd)self.attn_dropoutnn.Dropout(dropout)self.resid_dropoutnn.Dropout(dropout)# 下三角因果 Maskself.register_buffer(mask,torch.tril(torch.ones(block_size,block_size)).view(1,1,block_size,block_size))defforward(self,x):B,T,Cx.shape# 一次线性映射生成 Q、K、Vq,k,vself.qkv(x).split(C,dim2)# [B, T, C] - [B, Head, T, HeadDim]qq.view(B,T,self.num_heads,self.head_dim).transpose(1,2)kk.view(B,T,self.num_heads,self.head_dim).transpose(1,2)vv.view(B,T,self.num_heads,self.head_dim).transpose(1,2)# Scaled Dot-Product Attentionscoresq k.transpose(-2,-1)scoresscores/math.sqrt(self.head_dim)# 遮挡未来 Tokenscoresscores.masked_fill(self.mask[:,:,:T,:T]0,float(-inf))weightsF.softmax(scores,dim-1)weightsself.attn_dropout(weights)outweights v# 拼接所有注意力头outout.transpose(1,2).contiguous()outout.view(B,T,C)returnself.resid_dropout(self.proj(out))# # 5. 前馈神经网络# classFeedForward(nn.Module):def__init__(self):super().__init__()self.netnn.Sequential(nn.Linear(n_embd,4*n_embd),nn.GELU(),nn.Linear(4*n_embd,n_embd),nn.Dropout(dropout))defforward(self,x):returnself.net(x)# # 6. Transformer Block# classTransformerBlock(nn.Module):def__init__(self):super().__init__()self.ln1nn.LayerNorm(n_embd)self.attnCausalSelfAttention()self.ln2nn.LayerNorm(n_embd)self.ffnFeedForward()defforward(self,x):# Pre-Norm 残差连接xxself.attn(self.ln1(x))xxself.ffn(self.ln2(x))returnx# # 7. GPT 单向语言模型# classMiniGPT(nn.Module):def__init__(self):super().__init__()self.token_embeddingnn.Embedding(vocab_size,n_embd)self.position_embeddingnn.Embedding(block_size,n_embd)self.blocksnn.Sequential(*[TransformerBlock()for_inrange(n_layer)])self.ln_fnn.LayerNorm(n_embd)self.lm_headnn.Linear(n_embd,vocab_size)defforward(self,idx,targetsNone):B,Tidx.shapeassertTblock_size token_embself.token_embedding(idx)positionstorch.arange(T,deviceidx.device)pos_embself.position_embedding(positions)xtoken_embpos_emb xself.blocks(x)xself.ln_f(x)logitsself.lm_head(x)lossNoneiftargetsisnotNone:B,T,Vlogits.shape lossF.cross_entropy(logits.reshape(B*T,V),targets.reshape(B*T))returnlogits,losstorch.no_grad()defgenerate(self,idx,max_new_tokens100,temperature1.0,top_kNone):asserttemperature0self.eval()for_inrange(max_new_tokens):# 只使用最后 block_size 个 Tokenidx_condidx[:,-block_size:]logits,_self(idx_cond)# 获取最后一个位置的预测logitslogits[:,-1,:]# 温度调节logitslogits/temperature# Top-K 采样iftop_kisnotNone:kmin(top_k,logits.size(-1))values,_torch.topk(logits,k)logits[logitsvalues[:,[-1]]]-float(inf)probsF.softmax(logits,dim-1)next_tokentorch.multinomial(probs,num_samples1)idxtorch.cat([idx,next_token],dim1)returnidx# # 8. 训练与验证# modelMiniGPT().to(device)print(模型参数量:,sum(p.numel()forpinmodel.parameters()))optimizertorch.optim.AdamW(model.parameters(),lrlearning_rate)torch.no_grad()defestimate_loss():model.eval()results{}forsplitin[train,val]:lossestorch.zeros(20)foriinrange(20):xb,ybget_batch(split)_,lossmodel(xb,yb)losses[i]loss.item()results[split]losses.mean().item()model.train()returnresultsforstepinrange(max_iters):ifstep%eval_interval0orstepmax_iters-1:lossesestimate_loss()print(fstep{step:4d}| ftrain loss:{losses[train]:.4f}| fval loss:{losses[val]:.4f})xb,ybget_batch(train)logits,lossmodel(xb,yb)optimizer.zero_grad(set_to_noneTrue)loss.backward()# 梯度裁剪torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)optimizer.step()# # 9. 保存模型# torch.save({model_state_dict:model.state_dict(),stoi:stoi,itos:itos,block_size:block_size,n_embd:n_embd,n_head:n_head,n_layer:n_layer,},mini_gpt.pth)print(模型已保存mini_gpt.pth)# # 10. 文本生成# prompt人工智能# 字符级词表无法识别训练集外的字符unknownset(prompt)-set(stoi)ifunknown:raiseValueError(f提示词包含词表外字符:{unknown})contexttorch.tensor([encode(prompt)],dtypetorch.long,devicedevice)generatedmodel.generate(context,max_new_tokens100,temperature0.8,top_k10)print(\n 生成结果 )print(decode(generated[0].tolist()))