gpt-neox

1
{
2
  "pipe_parallel_size": 1,
3
  "model_parallel_size": 1,
4
  "make_vocab_size_divisible_by": 1,
5

6
  # model settings
7
  "num_layers": 32,
8
  "hidden_size": 4096,
9
  "num_attention_heads": 32,
10
  # Codellama was uptrained on 16k token sequence lengths
11
  # with rotary_emb_base adjusted to 1_000_000.
12
  "seq_length": 16384,
13
  "max_position_embeddings": 16384,
14
  "pos_emb": "rotary",
15
  "rotary_pct": 1,
16
  "rotary_emb_base": 1000000,
17
  "no_weight_tying": true,
18
  "gpt_j_residual": false,
19
  "output_layer_parallelism": "column",
20
  "norm": "rmsnorm",
21
  "rms_norm_epsilon": 1.0e-5,
22

23
  "attention_config": [[["flash"], 32]],
24

25
  "scaled_upper_triang_masked_softmax_fusion": true,
26
  "bias_gelu_fusion": false,
27
  "use_bias_in_norms": false,
28
  "use_bias_in_attn_linear": false,
29
  "mlp_type": "llama",
30
  "activation": "silu",
31
}
32