promptflow

1
import argparse
2
import os
3
import sys
4
from pathlib import Path
5

6
from utils import Color, run_command, print_red
7

8
if __name__ == "__main__":
9
    parser = argparse.ArgumentParser(description=Color.RED + "Test Coverage for Promptflow!" + Color.END + "\n")
10

11
    parser.add_argument("-p", required=True, nargs="+", help="The paths to calculate code coverage")
12
    parser.add_argument("-t", required=True, nargs="+", help="The path to the tests")
13
    parser.add_argument("-l", required=True, help="Location to run tests in")
14
    parser.add_argument(
15
        "-m",
16
        required=True,
17
        help="Pytest marker to identify the tests to run",
18
        default="all",
19
    )
20
    parser.add_argument(
21
        "-o",
22
        required=False,
23
        help="Pytest output file name",
24
        default="test-results.xml",
25
    )
26
    parser.add_argument("-n", help="Pytest number of process to run the tests", default="auto")
27
    parser.add_argument(
28
        "--model-name",
29
        help="The model file name to run the tests",
30
        type=str,
31
        default="",
32
    )
33
    parser.add_argument("--timeout", help="Timeout for individual tests (seconds)", type=str, default="")
34
    parser.add_argument(
35
        "--coverage-config",
36
        help="The path of code coverage config file",
37
        type=str,
38
        default="",
39
    )
40
    parser.add_argument(
41
        "--disable-cov-branch",
42
        action="store_true",
43
        help="Whether to enable branch coverage calculation",
44
    )
45
    parser.add_argument(
46
        "--ignore-glob",
47
        help="The path of ignored test file",
48
        type=str,
49
        default="",
50
    )
51

52
    args = parser.parse_args()
53
    print("Working directory: " + str(os.getcwd()))
54
    print("Args.p: " + str(args.p))
55
    print("Args.t: " + str(args.t))
56
    print("Args.l: " + str(args.l))
57
    print("Args.m: " + str(args.m))
58
    print("Args.n: " + str(args.n))
59
    print("Args.o: " + str(args.o))
60
    print("Args.model-name: " + str(args.model_name))
61
    print("Args.timeout: " + str(args.timeout))
62
    print("Args.coverage-config: " + str(args.coverage_config))
63
    print("Args.ignore-glob: " + str(args.ignore_glob))
64
    print("Args.disable-cov-branch: " + str(args.disable_cov_branch))
65

66
    test_paths_list = [str(Path(path).absolute()) for path in args.t]
67

68
    # display a list of all Python packages installed in the current Python environment
69
    run_command(["pip", "list"])
70
    run_command(["pip", "show", "promptflow", "promptflow-sdk"])
71

72
    pytest_command = ["pytest", f"--junitxml={args.o}"]
73
    pytest_command += test_paths_list
74
    if args.coverage_config:
75
        if args.p:
76
            cov_path_list = [f"--cov={path}" for path in args.p]
77
            pytest_command += cov_path_list
78
        if not args.disable_cov_branch:
79
            pytest_command += ["--cov-branch"]
80
        pytest_command += [  # noqa: W503
81
            "--cov-report=term",
82
            "--cov-report=html",
83
            "--cov-report=xml",
84
        ]
85
        pytest_command = pytest_command + [f"--cov-config={args.coverage_config}"]
86
    if args.ignore_glob:
87
        pytest_command = pytest_command + [f"--ignore-glob={args.ignore_glob}"]
88
    pytest_command += [
89
        "-n",
90
        args.n,
91
        "--dist",
92
        "loadfile",
93
        "--log-level=info",
94
        "--log-format=%(asctime)s %(levelname)s %(message)s",
95
        "--log-date-format=[%Y-%m-%d %H:%M:%S]",
96
        "--durations=5",
97
        "-ra",
98
        "-vv",
99
    ]
100

101
    if args.timeout:
102
        pytest_command = pytest_command + [
103
            "--timeout",
104
            args.timeout,
105
            "--timeout_method",
106
            "thread",
107
        ]
108

109
    if args.m != "all":
110
        pytest_command = pytest_command + ["-m", args.m]
111

112
    if args.model_name:
113
        pytest_command = pytest_command + ["--model-name", args.model_name]
114

115
    # pytest --junit-xml=test-results.xml --cov=azure.ai.ml --cov-report=html --cov-report=xml -ra ./tests/*/unittests/
116
    error_code, _ = run_command(pytest_command, throw_on_retcode=False)
117
    # https://docs.pytest.org/en/7.1.x/reference/exit-codes.html
118
    if error_code == 1:
119
        print_red("Tests were collected and run but some of the tests failed.")
120
    elif error_code == 2:
121
        print_red("Test execution was interrupted by the user.")
122
    elif error_code == 3:
123
        print_red("Internal error happened while executing tests.")
124
    elif error_code == 4:
125
        print_red("pytest command line usage error.")
126
    elif error_code == 5:
127
        print_red("No tests were collected.")
128
    sys.exit(error_code)
129