paddlenlp

1
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License"
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14

15
import paddle
16
import paddle.nn as nn
17
import paddle.nn.functional as F
18

19

20
class PointwiseMatching(nn.Layer):
21
    def __init__(self, pretrained_model, dropout=None):
22
        super().__init__()
23
        self.ptm = pretrained_model
24
        self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)
25

26
        # num_labels = 2 (similar or dissimilar)
27
        self.classifier = nn.Linear(self.ptm.config["hidden_size"], 2)
28

29
    def forward(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
30

31
        _, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
32

33
        cls_embedding = self.dropout(cls_embedding)
34
        logits = self.classifier(cls_embedding)
35
        probs = F.softmax(logits)
36

37
        return probs
38

39

40
class PairwiseMatching(nn.Layer):
41
    def __init__(self, pretrained_model, dropout=None, margin=0.1):
42
        super().__init__()
43
        self.ptm = pretrained_model
44
        self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)
45
        self.margin = margin
46

47
        # hidden_size -> 1, calculate similarity
48
        self.similarity = nn.Linear(self.ptm.config["hidden_size"], 1)
49

50
    def predict(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
51

52
        _, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
53

54
        cls_embedding = self.dropout(cls_embedding)
55
        sim_score = self.similarity(cls_embedding)
56
        sim_score = F.sigmoid(sim_score)
57

58
        return sim_score
59

60
    def forward(
61
        self,
62
        pos_input_ids,
63
        neg_input_ids,
64
        pos_token_type_ids=None,
65
        neg_token_type_ids=None,
66
        pos_position_ids=None,
67
        neg_position_ids=None,
68
        pos_attention_mask=None,
69
        neg_attention_mask=None,
70
    ):
71

72
        _, pos_cls_embedding = self.ptm(pos_input_ids, pos_token_type_ids, pos_position_ids, pos_attention_mask)
73

74
        _, neg_cls_embedding = self.ptm(neg_input_ids, neg_token_type_ids, neg_position_ids, neg_attention_mask)
75

76
        pos_embedding = self.dropout(pos_cls_embedding)
77
        neg_embedding = self.dropout(neg_cls_embedding)
78

79
        pos_sim = self.similarity(pos_embedding)
80
        neg_sim = self.similarity(neg_embedding)
81

82
        pos_sim = F.sigmoid(pos_sim)
83
        neg_sim = F.sigmoid(neg_sim)
84

85
        labels = paddle.full(shape=[pos_cls_embedding.shape[0]], fill_value=1.0, dtype="float32")
86

87
        loss = F.margin_ranking_loss(pos_sim, neg_sim, labels, margin=self.margin)
88

89
        return loss
90