skypilot

1
base_model: mistralai/Mistral-7B-v0.1
2
model_type: MistralForCausalLM
3
tokenizer_type: LlamaTokenizer
4
is_mistral_derived_model: true
5

6
load_in_8bit: false
7
load_in_4bit: true
8
strict: false
9

10
datasets:
11
  - path: mhenrichsen/alpaca_2k_test
12
    type: alpaca
13
dataset_prepared_path: last_run_prepared
14
val_set_size: 0.05
15
output_dir: ./qlora-out
16

17
# hub_model_id: manishiitg/mhenrichsen-alpaca_2k_test # TODO: Replace with hub model id
18
# hf_use_auth_token: false # TODO: push as private or public model
19

20
adapter: qlora
21
lora_model_dir:
22

23
sequence_len: 8192
24
sample_packing: true
25
pad_to_sequence_len: true
26

27
lora_r: 32
28
lora_alpha: 16
29
lora_dropout: 0.05
30
lora_target_linear: true
31
lora_fan_in_fan_out:
32
lora_target_modules:
33
  - gate_proj
34
  - down_proj
35
  - up_proj
36
  - q_proj
37
  - v_proj
38
  - k_proj
39
  - o_proj
40

41
wandb_project:
42
wandb_entity:
43
wandb_watch:
44
wandb_run_id:
45
wandb_log_model:
46

47
gradient_accumulation_steps: 4
48
micro_batch_size: 2
49
num_epochs: 1
50
optimizer: adamw_bnb_8bit
51
lr_scheduler: cosine
52
learning_rate: 0.0002
53

54
train_on_inputs: false
55
group_by_length: false
56
bf16: true
57
fp16: false
58
tf32: false
59

60
gradient_checkpointing: true
61
early_stopping_patience:
62
resume_from_checkpoint:
63
local_rank:
64
logging_steps: 1
65
xformers_attention:
66
flash_attention: true
67

68
warmup_steps: 10
69
eval_steps: 0.05
70
eval_table_size:
71
eval_table_max_new_tokens: 128
72
save_steps:
73
debug:
74
deepspeed:
75
weight_decay: 0.0
76
fsdp:
77
fsdp_config:
78
special_tokens:
79
  bos_token: "<s>"
80
  eos_token: "</s>"
81
  unk_token: "<unk>"