paddlenlp

1
{
2
    "dp_degree": "auto",
3
    "max_search_time": 900,
4
    "max_time_per_task": 400,
5
    "buffer":17408,
6
    "max_mem_usage":40960,
7
    "metric_cfg": {
8
      "OptimizationDirection": "Maximize",
9
      "name": "interval_samples_per_second"
10
    },
11
    "micro_batch_size": "auto",
12
    "model_cfg": {
13
      "global_batch_size": 8,
14
      "hidden_size": 5120,
15
      "num_attention_heads": 40,
16
      "num_layers": 40,
17
      "vocab_size": 32000
18
    },
19
    "mp_degree": "auto",
20
    "pp_degree": "auto",
21
    "run_cmd": {
22
      "gradient_accumulation_steps": [
23
        "./autoconfig/llama7b_pretrain_params.json",
24
        "gradient_accumulation_steps"
25
      ],
26
      "micro_batch_size": [
27
        "./autoconfig/llama7b_pretrain_params.json",
28
        "per_device_train_batch_size"
29
      ],
30
      "mp_degree": [
31
        "./autoconfig/llama7b_pretrain_params.json",
32
        "tensor_parallel_degree"
33
      ],
34
      "pp_degree": [
35
        "./autoconfig/llama7b_pretrain_params.json",
36
        "pipeline_parallel_degree"
37
      ],
38
      "run_best_stage": {
39
        "continue_training": [
40
          "./autoconfig/llama7b_pretrain_params.json",
41
          "continue_training",
42
          0
43
        ],
44
        "autotuner_benchmark": [
45
          "./autoconfig/llama7b_pretrain_params.json",
46
          "autotuner_benchmark",
47
          0
48
        ]
49
      },
50
      "search_stage": {
51
        "continue_training": [
52
          "./autoconfig/llama7b_pretrain_params.json",
53
          "continue_training",
54
          0
55
        ],
56
        "autotuner_benchmark": [
57
          "./autoconfig/llama7b_pretrain_params.json",
58
          "autotuner_benchmark",
59
          1
60
        ]
61
      },
62
      "sharding_degree": [
63
        "./autoconfig/llama7b_pretrain_params.json",
64
        "sharding_parallel_degree"
65
      ],
66
      "sharding_stage": [
67
        "./autoconfig/llama7b_pretrain_params.json",
68
        "sharding",
69
        "stage"
70
      ],
71
      "use_recompute": [
72
        "./autoconfig/llama7b_pretrain_params.json",
73
        "recompute"
74
      ],
75
      "recompute_granularity": [
76
        "./autoconfig/llama7b_pretrain_params.json",
77
        "recompute_granularity"
78
      ]
79
    },
80
    "sharding_degree": "auto",
81
    "sharding_stage": "auto",
82
    "task_limit": 2000,
83
    "use_recompute": "auto",
84
    "recompute_granularity": "auto",
85
    "invalid_strategy": ["stage3_mp*"],
86
    "schedule_prior": ["mp4"],
87
    "need_baseline": true,
88
    "mode": "Pretrain"
89
  }