OpenDelta

1
from transformers import AutoModelForSequenceClassification
2
model = AutoModelForSequenceClassification.from_pretrained("roberta-base")
3
# suppose we load BART
4

5
from opendelta import Visualization
6
print("before modify")
7
Visualization(model).structure_graph()
8

9
from opendelta import LoraModel
10
import re
11
delta_model = LoraModel(backbone_model=model, modified_modules=['[r](\d)+\.output.dense', 'attention.output.dense'])
12
# delta_model = LoraModel(backbone_model=model, modified_modules=['[r][0-5]\.output.dense'])
13
print("after modify")
14
delta_model.log()
15
# This will visualize the backbone after modification and other information.