llava

1
import os
2
import argparse
3
import json
4

5
from llava.eval.m4c_evaluator import EvalAIAnswerProcessor
6

7

8
def parse_args():
9
    parser = argparse.ArgumentParser()
10
    parser.add_argument('--dir', type=str, default="./playground/data/eval/vqav2")
11
    parser.add_argument('--ckpt', type=str, required=True)
12
    parser.add_argument('--split', type=str, required=True)
13
    return parser.parse_args()
14

15

16
if __name__ == '__main__':
17

18
    args = parse_args()
19

20
    src = os.path.join(args.dir, 'answers', args.split, args.ckpt, 'merge.jsonl')
21
    test_split = os.path.join(args.dir, 'llava_vqav2_mscoco_test2015.jsonl')
22
    dst = os.path.join(args.dir, 'answers_upload', args.split, f'{args.ckpt}.json')
23
    os.makedirs(os.path.dirname(dst), exist_ok=True)
24

25
    results = []
26
    error_line = 0
27
    for line_idx, line in enumerate(open(src)):
28
        try:
29
            results.append(json.loads(line))
30
        except:
31
            error_line += 1
32

33
    results = {x['question_id']: x['text'] for x in results}
34
    test_split = [json.loads(line) for line in open(test_split)]
35
    split_ids = set([x['question_id'] for x in test_split])
36

37
    print(f'total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}')
38

39
    all_answers = []
40

41
    answer_processor = EvalAIAnswerProcessor()
42

43
    for x in test_split:
44
        if x['question_id'] not in results:
45
            all_answers.append({
46
                'question_id': x['question_id'],
47
                'answer': ''
48
            })
49
        else:
50
            all_answers.append({
51
                'question_id': x['question_id'],
52
                'answer': answer_processor(results[x['question_id']])
53
            })
54

55
    with open(dst, 'w') as f:
56
        json.dump(all_answers, open(dst, 'w'))
57