lm-evaluation-harness

1
import os
2
from typing import List, Union
3

4
from lm_eval.utils import load_yaml_config
5

6

7
# {{{CI}}}
8
# This is the path where the output for the changed files for the tasks folder is stored
9
# FILE_PATH = file_path = ".github/outputs/tasks_all_changed_and_modified_files.txt"
10

11

12
# reads a text file and returns a list of words
13
# used to read the output of the changed txt from tj-actions/changed-files
14
def load_changed_files(file_path: str) -> List[str]:
15
    with open(file_path, "r") as f:
16
        content = f.read()
17
        words_list = [x for x in content.split()]
18
    return words_list
19

20

21
# checks the txt file for list of changed files.
22
# if file ends with .yaml then check yaml and load the config.
23
# if the config task is a string, it's a task config.
24
# if the config task is a list, it's a group config.
25
def parser(full_path: List[str]) -> List[str]:
26
    _output = set()
27
    for x in full_path:
28
        if os.path.exists(x) and x.endswith(".yaml"):
29
            config = load_yaml_config(x, mode="simple")
30
            if isinstance(config["task"], str):
31
                _output.add(config["task"])
32
            elif isinstance(config["task"], list):
33
                _output.add(config["group"])
34
    return list(_output)
35

36

37
def new_tasks() -> Union[List[str], None]:
38
    FILENAME = ".github/outputs/tasks_all_changed_and_modified_files.txt"
39
    if os.path.exists(FILENAME):
40
        # If tasks folder has changed then we get the list of files from FILENAME
41
        # and parse the yaml files to get the task names.
42
        return parser(load_changed_files(FILENAME))
43
    elif os.getenv("API") is not None:
44
        # Or if API has changed then we set the ENV variable API to True
45
        # and run  given tasks.
46
        return ["arc_easy", "hellaswag", "piqa", "wikitext"]
47
    # if both not true just do arc_easy
48
    else:
49
        return
50