OpenAttack

1
import OpenAttack
2
import nltk
3
from nltk.sentiment.vader import SentimentIntensityAnalyzer
4
import numpy as np
5
import datasets
6

7
def make_model():
8
    class MyClassifier(OpenAttack.Classifier):
9
        def __init__(self):
10
            try:
11
                self.model = SentimentIntensityAnalyzer()
12
            except LookupError:
13
                nltk.download('vader_lexicon')
14
                self.model = SentimentIntensityAnalyzer()
15
        
16
        def get_pred(self, input_):
17
            return self.get_prob(input_).argmax(axis=1)
18

19
        def get_prob(self, input_):
20
            ret = []
21
            for sent in input_:
22
                res = self.model.polarity_scores(sent)
23
                prob = (res["pos"] + 1e-6) / (res["neg"] + res["pos"] + 1e-6)
24
                ret.append(np.array([1 - prob, prob]))
25
            return np.array(ret)
26
    return MyClassifier()
27

28
def dataset_mapping(x):
29
    return {
30
        "x": x["sentence"],
31
        "y": 1 if x["label"] > 0.5 else 0,
32
    }
33

34
def main():
35

36
    print("New Attacker")
37
    attacker = OpenAttack.attackers.PWWSAttacker()
38

39
    print("Build model")
40
    clsf = make_model()
41

42
    dataset = datasets.load_dataset("sst", split="train[:100]").map(function=dataset_mapping)
43

44
    print("Start attack")
45
    attack_eval = OpenAttack.AttackEval( attacker, clsf, metrics=[
46
        OpenAttack.metric.Fluency(),
47
        OpenAttack.metric.GrammaticalErrors(),
48
        OpenAttack.metric.SemanticSimilarity(),
49
        OpenAttack.metric.EditDistance(),
50
        OpenAttack.metric.ModificationRate()
51
    ] )
52
    attack_eval.eval(dataset, visualize=True, progress_bar=True)
53

54
if __name__ == "__main__":
55
    main()
56