paddlenlp

1
{
2
	"model_name_or_path": "facebook/llama-7b",
3
	"dataset_name_or_path": "./data",
4
	"output_dir": "./checkpoints/llama_sft_ckpts",
5
	"per_device_train_batch_size": 1,
6
	"gradient_accumulation_steps": 4,
7
	"per_device_eval_batch_size": 8,
8
	"eval_accumulation_steps": 16,
9
	"num_train_epochs": 1,
10
    "max_steps": 100,
11
	"learning_rate": 3e-05,
12
	"warmup_steps": 30,
13
	"logging_steps": 1,
14
	"evaluation_strategy": "no",
15
	"save_strategy": "steps",
16
	"src_length": 256,
17
	"max_length": 512,
18
	"bf16": true,
19
	"fp16_opt_level": "O2",
20
	"do_train": true,
21
	"do_eval": false,
22
	"disable_tqdm": true,
23
	"load_best_model_at_end": false,
24
	"eval_with_do_generation": false,
25
	"recompute": false,
26
	"save_total_limit": 1,
27
	"tensor_parallel_degree": 4,
28
	"pipeline_parallel_degree": 1,
29
	"pipeline_parallel_config": "disable_p2p_cache_shape",
30
	"zero_padding": false,
31
	"use_flash_attention": true,
32
	"sharding_parallel_degree": 2,
33
	"sharding": "stage2",
34
	"recompute_granularity": "full",
35
	"autotuner_benchmark": 1,
36
	"benchmark": 1
37
}