p04798526
/
OpenDeltaMirror
mirror of https://github.com/thunlp/OpenDelta.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update lora.py

This commit is contained in:
DingDing 2022-04-14 14:33:03 +08:00 committed by GitHub
parent 41ecfcb595
commit b6e522318f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
opendelta/delta_models/lora.py
View File

@ -28,7 +28,6 @@ class LowRankLinear(nn.Module):
self.r = r
self.lora_alpha = lora_alpha
self.lora_dropout = lora_dropout
self.lin = nn.Linear(in_features, out_features) #
if lora_dropout > 0.:
self.lora_dropout = nn.Dropout(p=lora_dropout)
else:
@ -37,7 +36,6 @@ class LowRankLinear(nn.Module):
self.lora_A = nn.Parameter(weight.new_zeros((r, in_features)))
self.lora_B = nn.Parameter(weight.new_zeros((out_features, r)))
self.scaling = self.lora_alpha / self.r
self.lin.reset_parameters() #
nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
nn.init.zeros_(self.lora_B)
@ -151,4 +149,4 @@ class LoraModel(DeltaBase):
self.delta_modules.append(new_module)
else:
raise NotImplementedError
return new_module
return new_module