paddlenlp

1
Global:
2
  device: gpu
3
  seed: 1024
4

5
  global_batch_size: 
6
  local_batch_size: 1
7
  micro_batch_size: 1
8

9

10
Engine:
11
  max_steps: 500000
12
  num_train_epochs: 1
13
  accumulate_steps:
14
  logging_freq: 1
15
  eval_freq: 500
16
  eval_iters: 10
17
  test_iters:
18
  mix_precision:
19
    enable: True
20
    dtype: "float16"
21
    level: "O2"
22
    scale_loss: 32768.0
23
    custom_black_list: ["reduce_sum", "c_softmax_with_cross_entropy", "elementwise_div"]
24
    custom_white_list: ["lookup_table", "lookup_table_v2"]
25
  save_load:
26
    save_steps: 1000
27
    save_epoch: 1
28
    output_dir: ./output
29
    ckpt_dir:
30

31

32
Model:
33
  module: "GPTModule"
34
  name: "GPT"
35
  vocab_size_divisible_unit: 128
36
  fused_linear: False
37
  fuse_attn_qkv: True
38
  scale_qk_by_layer_num: True
39
  sequence_parallel: False
40
  use_flash_attn: False
41
  fused_softmax_with_triangular: True
42

43

44
Data:
45
  Train:
46
    dataset:
47
      name: GPTDataset
48
      input_dir: ./data/
49
      split: [969, 30, 1]
50
      max_seq_len: 1024
51
    sampler:
52
      name: GPTBatchSampler
53
      shuffle: False
54
      drop_last: True
55
    loader:
56
      num_workers: 1
57
      return_list: False
58
      collate_fn: gpt_collate_fn
59
  
60
  Eval:
61
    dataset:
62
      name: GPTDataset
63
      input_dir: ./data/
64
      split: [969, 30, 1]
65
      max_seq_len: 1024
66
    sampler:
67
      name: GPTBatchSampler
68
      shuffle: False
69
      drop_last: True
70
    loader:
71
      num_workers: 1
72
      return_list: False
73
      collate_fn: gpt_collate_fn
74

75

76
Optimizer:
77
  name: FusedAdamW
78
  weight_decay: 0.01
79
  beta1: 0.9
80
  beta2: 0.999
81
  epsilon: 1.0e-8
82
  lr:
83
    name: CosineAnnealingWithWarmupDecay
84
    decay_steps: 360000
85
    warmup_rate: 0.01
86
    max_lr: 5.0e-5
87
    min_lr: 1.0e-5
88
    use_increments: True
89
  grad_clip:
90
    name: "ClipGradByGlobalNorm"
91
    clip_norm: 1.0
92
  tensor_fusion: False
93

94

95
Profiler:
96
  enable: False
97
  scheduler: [1, 5]
98
  profiler_log: profiler_log
99
  detailed: False
100

101

102
Distributed:
103
  fuse_sequence_parallel_allreduce: False
104