2022-02-15 17:59:53 +08:00
|
|
|
from transformers import AutoModelForSequenceClassification
|
|
|
|
|
model = AutoModelForSequenceClassification.from_pretrained("roberta-base")
|
|
|
|
|
# suppose we load BART
|
|
|
|
|
|
|
|
|
|
from opendelta import Visualization
|
|
|
|
|
print("before modify")
|
|
|
|
|
Visualization(model).structure_graph()
|
|
|
|
|
|
|
|
|
|
from opendelta import LoraModel
|
|
|
|
|
import re
|
2022-02-15 22:43:28 +08:00
|
|
|
delta_model = LoraModel(backbone_model=model, modified_modules=['[r](\d)+\.output.dense', 'attention.output.dense'])
|
2022-02-20 17:23:31 +08:00
|
|
|
# delta_model = LoraModel(backbone_model=model, modified_modules=['[r][0-5]\.output.dense'])
|
2022-02-15 17:59:53 +08:00
|
|
|
print("after modify")
|
|
|
|
|
delta_model.log()
|
|
|
|
|
# This will visualize the backbone after modification and other information.
|
