ch10监督微调章节中未提到为啥lora模型里面要乘0.01。 #1

import torch
import torch.nn as nn

class LoRALinear(nn.Module):
    def __init__(self, original: nn.Linear, r: int = 8, alpha: int = 16):
        super().__init__()
        self.original = original
        self.original.weight.requires_grad_(False)  # 冻结 W
        out_dim, in_dim = original.weight.shape
        self.A = nn.Parameter(torch.randn(r, in_dim) * 0.01) # 这里的0.01的作用未提及。
        self.B = nn.Parameter(torch.zeros(out_dim, r))
        self.scale = alpha / r

    def forward(self, x):
        base_out = self.original(x)             # Wx
        lora_out = (x @ self.A.T) @ self.B.T    # xA^T B^T = (BA)x 的等价写法
        return base_out + self.scale * lora_out