GENA_LM
1{
2"architectures": [
3"BertForMaskedLM"
4],
5"attention_probs_dropout_prob": 0.1,
6"gradient_checkpointing": false,
7"hidden_act": "gelu",
8"hidden_dropout_prob": 0.1,
9"hidden_size": 1024,
10"initializer_range": 0.02,
11"intermediate_size": 4096,
12"layer_norm_eps": 1e-5,
13"max_position_embeddings": 512,
14"model_type": "bert",
15"num_attention_heads": 16,
16"num_hidden_layers": 24,
17"pad_token_id": 3,
18"pre_layer_norm": true,
19"position_embedding_type": "absolute",
20"transformers_version": "4.6.0.dev0",
21"type_vocab_size": 2,
22"use_cache": true,
23"vocab_size": 32000
24}