paddlenlp

1
{
2
    "dp_degree": "auto",
3
    "max_search_time": 900,
4
    "max_time_per_task": 400,
5
    "metric_cfg": {
6
      "OptimizationDirection": "Maximize",
7
      "name": "interval_samples_per_second"
8
    },
9
    "micro_batch_size": "auto",
10
    "model_cfg": {
11
      "global_batch_size": 8,
12
      "hidden_size": 5120,
13
      "num_attention_heads": 40,
14
      "num_layers": 40,
15
      "vocab_size": 32000
16
    },
17
    "mp_degree": "auto",
18
    "pp_degree": "auto",
19
    "run_cmd": {
20
      "gradient_accumulation_steps": [
21
        "./autoconfig/llama7b_pretrain_params.json",
22
        "gradient_accumulation_steps"
23
      ],
24
      "micro_batch_size": [
25
        "./autoconfig/llama7b_pretrain_params.json",
26
        "per_device_train_batch_size"
27
      ],
28
      "mp_degree": [
29
        "./autoconfig/llama7b_pretrain_params.json",
30
        "tensor_parallel_degree"
31
      ],
32
      "pp_degree": [
33
        "./autoconfig/llama7b_pretrain_params.json",
34
        "pipeline_parallel_degree"
35
      ],
36
      "run_best_stage": {
37
        "continue_training": [
38
          "./autoconfig/llama7b_pretrain_params.json",
39
          "continue_training",
40
          0
41
        ],
42
        "autotuner_benchmark": [
43
          "./autoconfig/llama7b_pretrain_params.json",
44
          "autotuner_benchmark",
45
          0
46
        ]
47
      },
48
      "search_stage": {
49
        "continue_training": [
50
          "./autoconfig/llama7b_pretrain_params.json",
51
          "continue_training",
52
          0
53
        ],
54
        "autotuner_benchmark": [
55
          "./autoconfig/llama7b_pretrain_params.json",
56
          "autotuner_benchmark",
57
          1
58
        ]
59
      },
60
      "sharding_degree": [
61
        "./autoconfig/llama7b_pretrain_params.json",
62
        "sharding_parallel_degree"
63
      ],
64
      "sharding_stage": [
65
        "./autoconfig/llama7b_pretrain_params.json",
66
        "sharding",
67
        "stage"
68
      ],
69
      "use_recompute": [
70
        "./autoconfig/llama7b_pretrain_params.json",
71
        "recompute"
72
      ],
73
      "recompute_granularity": [
74
        "./autoconfig/llama7b_pretrain_params.json",
75
        "recompute_granularity"
76
      ]
77
    },
78
    "sharding_degree": "auto",
79
    "sharding_stage": "auto",
80
    "task_limit": 2000,
81
    "use_recompute": "auto",
82
    "recompute_granularity": "auto",
83
    "invalid_strategy": ["stage3_mp*"],
84
    "schedule_prior": ["mp4"],
85
    "need_baseline": true,
86
    "mode": "Pretrain"
87
  }