GenerativeAIExamples

1
# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
#
4
# Licensed under the Apache License, Version 2.0 (the "License");
5
# you may not use this file except in compliance with the License.
6
# You may obtain a copy of the License at
7
#
8
# http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15

16
import argparse
17
import logging
18

19
from evaluator import eval_llm_judge, eval_ragas
20
from llm_answer_generator import generate_answers
21

22
logging.basicConfig(level=logging.INFO)
23
logger = logging.getLogger(__name__)
24

25
if __name__ == "__main__":
26
    
27
    parser = argparse.ArgumentParser()
28
    parser.add_argument("--base_url", type=str, help="Specify the base URL to be used.")
29
    parser.add_argument(
30
        "--generate_answer",
31
        type=bool,
32
        nargs="?",
33
        default=False,
34
        help="Specify if 'answer' generation by RAG pipeline is required.",
35
    )
36
    parser.add_argument(
37
        "--evaluate",
38
        type=bool,
39
        nargs="?",
40
        default=True,
41
        help="Specify if evaluation is required.",
42
    )
43
    parser.add_argument(
44
        "--docs",
45
        type=str,
46
        nargs="?",
47
        default="",
48
        help="Specify the folder path for dataset.",
49
    )
50
    parser.add_argument(
51
        "--ga_input",
52
        type=str,
53
        nargs="?",
54
        default="",
55
        help="Specify the .json file with QnA pair for generating answers by RAG pipeline.",
56
    )
57
    parser.add_argument(
58
        "--ga_output",
59
        type=str,
60
        nargs="?",
61
        default="",
62
        help="Specify the .JSON file path for generated answers along with QnA.",
63
    )
64
    parser.add_argument(
65
        "--ev_input",
66
        type=str,
67
        nargs="?",
68
        default="",
69
        help="Specify the .JSON file path with 'question','gt_answer','gt_context','answer',and 'contexts'.",
70
    )
71
    parser.add_argument(
72
        "--ev_result",
73
        type=str,
74
        nargs="?",
75
        default="",
76
        help="Specify the file path to store evaluation results.",
77
    )
78
    parser.add_argument(
79
        "--metrics",
80
        type=str,
81
        nargs="?",
82
        default="judge_llm",
83
        choices=["ragas", "judge_llm"],
84
        help="Specify evaluation metrics between ragas and judge-llm.",
85
    )
86
    parser.add_argument(
87
        "--judge_llm_model",
88
        type=str,
89
        nargs="?",
90
        default="ai-mixtral-8x7b-instruct",
91
        help="Specify the LLM model to be used as judge llm for evaluation from ChatNVIDIA catalog."
92
    )
93
    args = parser.parse_args()
94

95
    if args.generate_answer:
96
        generate_answers(
97
            base_url=args.base_url,
98
            dataset_folder_path=args.docs,
99
            qa_generation_file_path=args.ga_input,
100
            eval_file_path=args.ga_output,
101
        )
102

103
    logger.info("\nANSWERS GENERATED\n")
104
    if args.evaluate:
105
        if args.metrics == "ragas":
106
            eval_ragas(
107
                ev_file_path=args.ev_input,
108
                ev_result_path=args.ev_result,
109
                llm_model=args.judge_llm_model,
110
            )
111
            logger.info("\nRAG EVALUATED WITH RAGAS METRICS\n")
112
        elif args.metrics == "judge_llm":
113
            eval_llm_judge(
114
                ev_file_path=args.ev_input,
115
                ev_result_path=args.ev_result,
116
                llm_model=args.judge_llm_model,
117
            )
118
            logger.info("\nRAG EVALUATED WITH JUDGE LLM\n")
119