/
socialgraph
/
dataset
Обзор
Документация
Войти
/
socialgraph
/
dataset
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
CI/CD
Аналитика
Безопасность
master
upload/examples.py
41 строка
1 KB
socialgraph
parse
09 авг 2026, 10:22
09 авг 2026, 10:22
76cada1
Код
Авторство
О чём код?
import json import os from collections import defaultdict from tqdm import tqdm from prepare.utils import get_is_code from tools.const import WORK_DIR from tools.load import get_data from tools.utils import save_examples def run(language): is_code = get_is_code(language) with open(f"{WORK_DIR}/src/examples.jsonl", encoding="utf-8") as src: examples = [json.loads(q) for q in src] for q in tqdm(examples): d = get_data(q['project']) if d and d.get('language').lower() == language: q['language'] = language q['branch'] = d['default_branch'] q['files_count'] = sum(1 for f in d['files'] if is_code(f)) examples = [q for q in examples if q.get('language') == language] save_examples(examples, language) if __name__ == '__main__': import argparse args = argparse.ArgumentParser() args.add_argument( "--language", type=str, default="java" ) args = args.parse_args() run(args.language)