57 lines
1.7 KiB
Python
57 lines
1.7 KiB
Python
import os
|
|
import argparse
|
|
import json
|
|
|
|
from llava.eval.m4c_evaluator import EvalAIAnswerProcessor
|
|
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--dir', type=str, default="./playground/data/eval/vqav2")
|
|
parser.add_argument('--ckpt', type=str, required=True)
|
|
parser.add_argument('--split', type=str, required=True)
|
|
return parser.parse_args()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
args = parse_args()
|
|
|
|
src = os.path.join(args.dir, 'answers', args.split, args.ckpt, 'merge.jsonl')
|
|
test_split = os.path.join(args.dir, 'llava_vqav2_mscoco_test2015.jsonl')
|
|
dst = os.path.join(args.dir, 'answers_upload', args.split, f'{args.ckpt}.json')
|
|
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
|
|
results = []
|
|
error_line = 0
|
|
for line_idx, line in enumerate(open(src)):
|
|
try:
|
|
results.append(json.loads(line))
|
|
except:
|
|
error_line += 1
|
|
|
|
results = {x['question_id']: x['text'] for x in results}
|
|
test_split = [json.loads(line) for line in open(test_split)]
|
|
split_ids = set([x['question_id'] for x in test_split])
|
|
|
|
print(f'total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}')
|
|
|
|
all_answers = []
|
|
|
|
answer_processor = EvalAIAnswerProcessor()
|
|
|
|
for x in test_split:
|
|
if x['question_id'] not in results:
|
|
all_answers.append({
|
|
'question_id': x['question_id'],
|
|
'answer': ''
|
|
})
|
|
else:
|
|
all_answers.append({
|
|
'question_id': x['question_id'],
|
|
'answer': answer_processor(results[x['question_id']])
|
|
})
|
|
|
|
with open(dst, 'w') as f:
|
|
json.dump(all_answers, open(dst, 'w'))
|