CSS-LM

1
import logging
2
import math
3

4
import torch
5
import torch.nn.functional as F
6

7

8
logger = logging.getLogger(__name__)
9

10

11
def swish(x):
12
    return x * torch.sigmoid(x)
13

14

15
def _gelu_python(x):
16
    """ Original Implementation of the gelu activation function in Google Bert repo when initially created.
17
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
18
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
19
        This is now written in C in torch.nn.functional
20
        Also see https://arxiv.org/abs/1606.08415
21
    """
22
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
23

24

25
def gelu_new(x):
26
    """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
27
        Also see https://arxiv.org/abs/1606.08415
28
    """
29
    return 0.5 * x * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0))))
30

31

32
if torch.__version__ < "1.4.0":
33
    gelu = _gelu_python
34
else:
35
    gelu = F.gelu
36

37

38
def gelu_fast(x):
39
    return 0.5 * x * (1.0 + torch.tanh(x * 0.7978845608 * (1.0 + 0.044715 * x * x)))
40

41

42
ACT2FN = {
43
    "relu": F.relu,
44
    "swish": swish,
45
    "gelu": gelu,
46
    "tanh": torch.tanh,
47
    "gelu_new": gelu_new,
48
    "gelu_fast": gelu_fast,
49
}
50

51

52
def get_activation(activation_string):
53
    if activation_string in ACT2FN:
54
        return ACT2FN[activation_string]
55
    else:
56
        raise KeyError("function {} not found in ACT2FN mapping {}".format(activation_string, list(ACT2FN.keys())))
57