paddlenlp

1
{
2
	"model_name_or_path": "facebook/llama-7b",
3
	"tokenizer_name_or_path": "facebook/llama-7b",
4
	"input_dir": "./data",
5
	"output_dir": "./checkpoints/llama_pretrain_ckpts",
6
	"per_device_train_batch_size": 1,
7
	"gradient_accumulation_steps": 8,
8
	"per_device_eval_batch_size": 2,
9
	"tensor_parallel_degree": 8,
10
	"pipeline_parallel_degree": 1,
11
	"sharding": "stage3",
12
	"virtual_pp_degree": 1,
13
	"sequence_parallel": 0,
14
	"use_flash_attention": true,
15
	"use_fused_rms_norm": true,
16
	"use_fused_rope": true,
17
	"max_seq_length": 4096,
18
	"learning_rate": 3e-05,
19
	"min_learning_rate": 3e-06,
20
	"warmup_steps": 30,
21
	"logging_steps": 1,
22
	"max_steps": 100,
23
	"save_steps": 5000,
24
	"eval_steps": 1000,
25
	"weight_decay": 0.01,
26
	"bf16": true,
27
	"fp16_opt_level": "O2",
28
	"warmup_ratio": 0.01,
29
	"max_grad_norm": 1.0,
30
	"dataloader_num_workers": 1,
31
	"continue_training": 0,
32
	"do_train": true,
33
	"do_eval": false,
34
	"do_predict": false,
35
	"disable_tqdm": true,
36
	"recompute": true,
37
	"distributed_dataloader": 1,
38
	"recompute_granularity": "full",
39
	"save_total_limit": 2,
40
	"sharding_parallel_degree": 1,
41
	"autotuner_benchmark": 1
42
}
43