llama-factory

1
# coding=utf-8
2
# Calculates the distribution of the input lengths in the dataset.
3
# Usage: python length_cdf.py --model_name_or_path path_to_model --dataset alpaca_en --template default
4

5
from collections import defaultdict
6
from typing import Optional
7

8
import fire
9
from tqdm import tqdm
10

11
from llmtuner.data import get_dataset
12
from llmtuner.hparams import get_train_args
13
from llmtuner.model import load_model_and_tokenizer
14

15

16
def length_cdf(
17
    model_name_or_path: str,
18
    dataset: Optional[str] = "alpaca_en",
19
    dataset_dir: Optional[str] = "data",
20
    template: Optional[str] = "default",
21
    interval: Optional[int] = 1000,
22
):
23
    model_args, data_args, training_args, finetuning_args, _ = get_train_args(
24
        dict(
25
            stage="sft",
26
            model_name_or_path=model_name_or_path,
27
            dataset=dataset,
28
            dataset_dir=dataset_dir,
29
            template=template,
30
            cutoff_len=1_000_000,
31
            output_dir="dummy_dir",
32
            overwrite_cache=True,
33
        )
34
    )
35
    _, tokenizer = load_model_and_tokenizer(model_args, finetuning_args, is_trainable=False, add_valuehead=False)
36
    trainset = get_dataset(tokenizer, model_args, data_args, training_args, stage="sft")
37
    total_num = len(trainset)
38
    length_dict = defaultdict(int)
39
    for sample in tqdm(trainset["input_ids"]):
40
        length_dict[len(sample) // interval * interval] += 1
41

42
    length_tuples = list(length_dict.items())
43
    length_tuples.sort()
44
    count_accu, prob_accu = 0, 0
45
    for length, count in length_tuples:
46
        count_accu += count
47
        prob_accu += count / total_num * 100
48
        print("{:d} ({:.2f}%) samples have length < {}.".format(count_accu, prob_accu, length + interval))
49

50

51
if __name__ == "__main__":
52
    fire.Fire(length_cdf)
53