完整代码

一个能干活的 Transformer 翻译模型,全部代码在这里。 分五部分:数据处理、模型搭建、训练、推理、评估。

每个代码块右上角有开关

注释写得比较密,如果只想看主干,点右上角「隐藏注释」。

一、数据处理

{String.raw`import random from collections import Counter import torch from torch.utils.data import Dataset, DataLoader from torch.nn.utils.rnn import pad_sequence # 四个特殊符号,固定占用前四个 id PAD, UNK, BOS, EOS = 0, 1, 2, 3 MAXLEN = 20 def load_pairs(path): """读 制表符分隔 的平行语料,返回 [(英文词列表, 中文字列表), ...]""" pairs = [] with open(path, encoding="utf-8") as f: for line in f: parts = line.rstrip("\n").split("\t") if len(parts) < 2: continue # 英文转小写、按空格切;中文按字切 en = parts[0].strip().lower() zh = parts[1].strip() if not en or not zh: continue en_words = en.split() # 太长的句子丢掉,省得 batch 里 padding 太多 if len(en_words) <= MAXLEN and len(zh) <= MAXLEN: pairs.append((en_words, list(zh))) return pairs def build_vocab(sentences, min_freq=2): """统计词频,建词表。只保留出现 >= min_freq 次的词""" counter = Counter() for s in sentences: counter.update(s) # 前四个位置留给特殊符号 itos = ["", "", "", ""] for token, freq in counter.most_common(): if freq < min_freq: break itos.append(token) # 双向映射:id -> 词,词 -> id stoi = {t: i for i, t in enumerate(itos)} return itos, stoi class TranslationDataset(Dataset): def __init__(self, pairs, src_stoi, tgt_stoi): self.data = [] for src, tgt in pairs: # 两端加上 和 ,未知词用 s = [BOS] + [src_stoi.get(w, UNK) for w in src] + [EOS] t = [BOS] + [tgt_stoi.get(w, UNK) for w in tgt] + [EOS] self.data.append((torch.tensor(s), torch.tensor(t))) def __len__(self): return len(self.data) def __getitem__(self, i): return self.data[i] def collate(batch): """把一个 batch 里长短不一的句子补到同样长""" srcs, tgts = zip(*batch) return ( pad_sequence(srcs, batch_first=True, padding_value=PAD), # (B, T_src) pad_sequence(tgts, batch_first=True, padding_value=PAD), # (B, T_tgt) torch.tensor([len(s) for s in srcs]), # 每句真实长度 )`}

二、模型搭建

{String.raw`import math import torch import torch.nn as nn class PositionalEncoding(nn.Module): """正弦位置编码,37.4 课讲过""" def __init__(self, d_model, max_len=512): super().__init__() # 先建一张全 0 的表,(最大长度, 维度) pe = torch.zeros(max_len, d_model) # 位置编号 0..max_len-1,变成列向量 pos = torch.arange(0, max_len).unsqueeze(1).float() # 每一对维度对应的分母:10000^(2j/d) div = torch.exp( torch.arange(0, d_model, 2).float() / d_model * math.log(10000.0) ) # 偶数维放 sin,奇数维放 cos pe[:, 0::2] = torch.sin(pos * div) pe[:, 1::2] = torch.cos(pos * div) # register_buffer:跟着模型搬 GPU,但不参与训练 self.register_buffer("pe", pe) def forward(self, x): # 只取前 T 行,加到每个样本上 return x + self.pe[: x.size(1)].unsqueeze(0) class MultiHeadAttention(nn.Module): """多头注意力,37.6 课讲过""" def __init__(self, d_model, n_heads, dropout=0.1): super().__init__() self.d_k = d_model // n_heads self.n_heads = n_heads # 四个投影矩阵:Q、K、V、输出 self.w_q = nn.Linear(d_model, d_model) self.w_k = nn.Linear(d_model, d_model) self.w_v = nn.Linear(d_model, d_model) self.w_o = nn.Linear(d_model, d_model) self.dropout = nn.Dropout(dropout) def forward(self, q, k, v, mask=None): B = q.size(0) def split(lin, x): # (B,T,d) -> (B,T,h,dk) -> (B,h,T,dk) # 拆成 h 个头,把头挪到 batch 维后面,方便批量算 return lin(x).view(B, -1, self.n_heads, self.d_k).transpose(1, 2) Q = split(self.w_q, q) K = split(self.w_k, k) V = split(self.w_v, v) # 缩放点积:除以 sqrt(d_k) 防止分数过大 scores = Q @ K.transpose(-2, -1) / math.sqrt(self.d_k) # mask 是 (B,T,T) 或 (1,T,T),unsqueeze 后广播到 h 个头 if mask is not None: scores = scores.masked_fill(mask.unsqueeze(1) == 0, -1e9) attn = self.dropout(torch.softmax(scores, dim=-1)) out = attn @ V # (B,h,T,dk) # 拼回 (B,T,d):先换回来,再把 h 和 dk 合并 out = out.transpose(1, 2).reshape(B, -1, self.n_heads * self.d_k) return self.w_o(out) class FeedForward(nn.Module): """前馈网络:放大 4 倍,ReLU,压回来""" def __init__(self, d_model, d_ff=2048, dropout=0.1): super().__init__() self.net = nn.Sequential( nn.Linear(d_model, d_ff), nn.ReLU(), nn.Dropout(dropout), nn.Linear(d_ff, d_model), ) def forward(self, x): return self.net(x) class EncoderLayer(nn.Module): """编码器一层:自注意力 + 前馈,各套残差和 LayerNorm""" def __init__(self, d_model, n_heads, d_ff, dropout=0.1): super().__init__() self.attn = MultiHeadAttention(d_model, n_heads, dropout) self.ffn = FeedForward(d_model, d_ff, dropout) self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) self.dropout = nn.Dropout(dropout) def forward(self, x, src_mask=None): # 子层 1:Q K V 都是自己(自注意力) x = self.norm1(x + self.dropout(self.attn(x, x, x, src_mask))) # 子层 2:前馈 x = self.norm2(x + self.dropout(self.ffn(x))) return x class DecoderLayer(nn.Module): """解码器一层:掩蔽自注意力 + 跨注意力 + 前馈""" def __init__(self, d_model, n_heads, d_ff, dropout=0.1): super().__init__() self.self_attn = MultiHeadAttention(d_model, n_heads, dropout) self.cross_attn = MultiHeadAttention(d_model, n_heads, dropout) self.ffn = FeedForward(d_model, d_ff, dropout) self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) self.norm3 = nn.LayerNorm(d_model) self.dropout = nn.Dropout(dropout) def forward(self, y, enc_out, tgt_mask=None, src_mask=None): # 子层 1:掩蔽自注意力,Q K V 都是自己,但要挡未来 y = self.norm1(y + self.dropout(self.self_attn(y, y, y, tgt_mask))) # 子层 2:跨注意力,Q 是自己,K 和 V 是编码器输出 y = self.norm2(y + self.dropout(self.cross_attn(y, enc_out, enc_out, src_mask))) # 子层 3:前馈 y = self.norm3(y + self.dropout(self.ffn(y))) return y def causal_mask(size): """下三角矩阵:位置 i 只能看 0..i""" # torch.tril 取下三角,1 = 允许,0 = 屏蔽 return torch.tril(torch.ones(size, size)).bool() class Transformer(nn.Module): def __init__(self, src_vocab, tgt_vocab, d_model=512, n_heads=8, n_layers=6, d_ff=2048, max_len=512, dropout=0.1): super().__init__() # 词嵌入,padding_idx 保证 的向量恒为 0 self.src_embed = nn.Embedding(src_vocab, d_model, padding_idx=PAD) self.tgt_embed = nn.Embedding(tgt_vocab, d_model, padding_idx=PAD) self.pos = PositionalEncoding(d_model, max_len) # 六层编码器、六层解码器,各自用 nn.ModuleList 存 self.encoder = nn.ModuleList([ EncoderLayer(d_model, n_heads, d_ff, dropout) for _ in range(n_layers) ]) self.decoder = nn.ModuleList([ DecoderLayer(d_model, n_heads, d_ff, dropout) for _ in range(n_layers) ]) # 把 d_model 维映射到词表大小 self.fc_out = nn.Linear(d_model, tgt_vocab) def encode(self, src): # 乘 sqrt(d_model) 让词嵌入的量级盖过位置编码 x = self.src_embed(src) * math.sqrt(self.src_embed.embedding_dim) x = self.pos(x) # padding 位置不参与注意力 pad_mask = (src != PAD) for layer in self.encoder: x = layer(x, pad_mask) return x def decode(self, tgt, enc_out, src): y = self.tgt_embed(tgt) * math.sqrt(self.tgt_embed.embedding_dim) y = self.pos(y) # 因果掩码 & 源句的 padding 掩码 look_ahead = causal_mask(tgt.size(1)).to(tgt.device) pad_mask = (src != PAD) for layer in self.decoder: y = layer(y, enc_out, look_ahead, pad_mask) return y def forward(self, src, tgt): enc_out = self.encode(src) dec_out = self.decode(tgt, enc_out, src) return self.fc_out(dec_out)`}

三、训练

{String.raw`def train(model, loader, tgt_vocab, epochs=20, device="cuda"): model.to(device) # 论文的设置:Adam,第二个 beta 用 0.98 optimizer = torch.optim.Adam(model.parameters(), lr=1.0, betas=(0.9, 0.98), eps=1e-9) # 交叉熵,忽略 padding 位置 criterion = nn.CrossEntropyLoss(ignore_index=PAD) d_model = model.src_embed.embedding_dim warmup = 1000 # 原论文 4000,这里训练步数少,调小 step = 0 for epoch in range(1, epochs + 1): model.train() total_loss, total_tokens = 0.0, 0 for src, tgt, _ in loader: src, tgt = src.to(device), tgt.to(device) # 错开一位:输入不含最后,标签不含开头 tgt_in = tgt[:, :-1] tgt_out = tgt[:, 1:] # 前向 logits = model(src, tgt_in) # 拉平成 (样本数, 词表大小) 算交叉熵 loss = criterion( logits.reshape(-1, tgt_vocab), tgt_out.reshape(-1), ) # 学习率 warmup:先升后降 # 必须在 optimizer.step() 之前设,否则第一步会用 lr=1.0 把模型炸掉 step += 1 lr = d_model ** (-0.5) * min(step ** (-0.5), step * warmup ** (-1.5)) for g in optimizer.param_groups: g["lr"] = lr optimizer.zero_grad() loss.backward() # 梯度裁剪,防止偶发的梯度爆炸 torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() # 统计时排除 padding n = (tgt_out != PAD).sum().item() total_loss += loss.item() * n total_tokens += n print(f"epoch {epoch} loss {total_loss / total_tokens:.4f}") return model`}

四、推理

编码器只跑一次

每生成一个词都重跑编码器是最大的浪费。把 enc_out 算一次存起来,循环里只调解码器。

{String.raw`@torch.no_grad() def translate(model, sentence, src_stoi, tgt_itos, device="cuda", max_len=30): model.eval() # 源句转 id,加首尾符号 ids = [BOS] + [src_stoi.get(w, UNK) for w in sentence.lower().split()] + [EOS] src = torch.tensor([ids], device=device) # 编码器只跑这一次,结果留着反复用 enc_out = model.encode(src) # 解码器从 开始 ys = torch.tensor([[BOS]], device=device) for _ in range(max_len): # 只调解码器,把缓存的 enc_out 传进去 dec = model.decode(ys, enc_out, src) # 取最后一个位置的分数,选最大的词 next_id = model.fc_out(dec[:, -1]).argmax(-1, keepdim=True) # 接到已生成序列后面 ys = torch.cat([ys, next_id], dim=1) # 遇到 就停 if next_id.item() == EOS: break # 去掉开头的 ,把 id 变回字 out = ys[0, 1:].tolist() if EOS in out: out = out[: out.index(EOS)] return "".join(tgt_itos[i] for i in out if i > 3)`}

五、评估

{String.raw`import math from collections import Counter def bleu(hyp, ref, n=4): """BLEU-4:n-gram 命中率 + 简短惩罚""" if not hyp: return 0.0 precisions = [] for k in range(1, n + 1): # 统计译文和参考里所有 k-gram hyp_grams = Counter( tuple(hyp[i:i + k]) for i in range(len(hyp) - k + 1) ) ref_grams = Counter( tuple(ref[i:i + k]) for i in range(len(ref) - k + 1) ) # 命中数取 min(译文出现次数, 参考出现次数) hit = sum(min(c, ref_grams[g]) for g, c in hyp_grams.items()) total = sum(hyp_grams.values()) # 加一平滑,避免 0 命中直接归零 precisions.append((hit + 1) / (total + 1) if hit == 0 else hit / total) # 四个精度的几何平均 score = sum(math.log(p) for p in precisions) / n # 简短惩罚:译文比参考短就扣分 if len(hyp) > len(ref): bp = 1.0 else: bp = math.exp(1 - len(ref) / max(1, len(hyp))) return bp * math.exp(score) def evaluate(model, test_pairs, src_stoi, tgt_itos, device="cuda", limit=None): """在测试集上算平均 BLEU""" pairs = test_pairs[:limit] if limit else test_pairs total = 0.0 for src, ref in pairs: hyp = translate(model, " ".join(src), src_stoi, tgt_itos, device) total += bleu(list(hyp), ref) return 100 * total / len(pairs)`}

跑起来

{String.raw`DATA = "D:/main/AIchallenger2017/cmn.txt" # 1. 读数据 pairs = load_pairs(DATA) random.seed(0) random.shuffle(pairs) # 2. 按 90/5/5 切 n = len(pairs) train_pairs = pairs[: int(n * 0.90)] val_pairs = pairs[int(n * 0.90): int(n * 0.95)] test_pairs = pairs[int(n * 0.95):] # 3. 只用训练集建词表,避免数据泄漏 src_itos, src_stoi = build_vocab([p[0] for p in train_pairs]) tgt_itos, tgt_stoi = build_vocab([p[1] for p in train_pairs]) print(f"源词表 {len(src_itos)} 目标词表 {len(tgt_itos)}") # 4. 建 DataLoader train_loader = DataLoader( TranslationDataset(train_pairs, src_stoi, tgt_stoi), batch_size=128, shuffle=True, collate_fn=collate, ) # 5. 建模型并训练 model = Transformer(len(src_itos), len(tgt_itos)) model = train(model, train_loader, len(tgt_itos), epochs=20) # 6. 评估 score = evaluate(model, test_pairs, src_stoi, tgt_itos, limit=400) print(f"BLEU {score:.2f}")`}

下一课

代码齐了。下一课真跑一遍,把结果摆出来—— 重点是长短句的分桶曲线。