企业官网建设流程全解析

class FFN(nn.Module): def __init__(self, model_dim, intermediate_dim): super().__init__() # TODO 1: 定义 self.w_up self.w_up=nn.Linear(model_dim,intermediate_dim) # TODO 2: 定义 self.w_down self.w_down=nn.Linear(intermediate_dim,model_dim) def forward(self, x): # TODO 3: 上投影并用 F.relu 激活 # TODO 4: 下投影并 return return self.w_down(F.relu(self.w_up(x)))

class SwiGLUFFN(nn.Module): def __init__(self, model_dim, intermediate_dim): super().__init__() # TODO 1: self.w_gate = ... (D -> H, bias=False) self.w_gate=nn.Linear(model_dim, intermediate_dim, bias=False) # TODO 2: self.w_up = ... (D -> H, bias=False) self.w_up=nn.Linear(model_dim, intermediate_dim, bias=False) # TODO 3: self.w_down = ... (H -> D, bias=False) self.w_down=nn.Linear(intermediate_dim, model_dim, bias=False) def forward(self, x): # TODO 4: gate = F.silu(...) gate=F.silu(self.w_gate(x)) # TODO 5: up = ... up=self.w_up(x) # TODO 6: return self.w_down(gate * up) return self.w_down(gate*up)