github_service.py - master - dpavlov/datagrid_reporter

github_service.py
262 строки10 KB
Dmitriy Pavlov
notes update
09 июн 2026, 01:00
09 июн 2026, 01:00b3cff75
from dataclasses import dataclass
from datetime import date
from typing import Any, Dict, Iterable

from datagrid_reporter.config import GithubConfig, ProjectConfig
from datagrid_reporter.dates import bounded_horizon, github_timestamp_to_day, utc_today
from datagrid_reporter.http_json import get_json, github_headers
from datagrid_reporter.progress import describe_exception
from datagrid_reporter.storage import LocalStore, SyncWindow


SOURCE = "github_prs"
BASE_URL = "https://api.github.com"


@dataclass(frozen=True)
class GithubSyncResult:
    prs: int = 0
    commits: int = 0
    commit_files: int = 0

    @property
    def total(self) -> int:
        return self.prs + self.commits + self.commit_files


def sync_github_prs(
    store: LocalStore,
    project: ProjectConfig,
    days_override: int | None = None,
    from_day: date | None = None,
    max_days: int | None = None,
    force: bool = False,
    progress=None,
) -> GithubSyncResult:
    if not project.github:
        raise ValueError(f"Project {project.key} has no GitHub config")

    cfg = project.github
    end_day = utc_today()
    window_days, desired_start = bounded_horizon(days_override or cfg.window_days, end_day, from_day, max_days)
    state = store.get_state(SOURCE, project.key)
    _emit(
        progress,
        _describe_window_plan(
            cfg.owner,
            cfg.repo,
            desired_start,
            end_day,
            window_days,
            state,
            force,
        ),
    )
    window = SyncWindow(desired_start, end_day, "forced reload") if force else store.plan_window(SOURCE, project.key, desired_start, end_day)
    _emit(
        progress,
        f"GitHub {cfg.owner}/{cfg.repo}: selected window start={window.start_day}, end={window.end_day}, reason={window.reason}",
    )
    if window.start_day is None:
        _emit(progress, f"GitHub {cfg.owner}/{cfg.repo}: window already covered; skipping recent PR scan")
        store.skip_sync_success(SOURCE, project.key)
        return GithubSyncResult()

    _emit(
        progress,
        f"GitHub {cfg.owner}/{cfg.repo}: syncing PRs updated since {window.start_day} through {end_day}",
    )
    store.begin_sync(SOURCE, project.key, window_days)
    try:
        pr_count = 0
        commit_count = 0
        commit_file_count = 0
        commit_details_left = cfg.max_commit_details_per_sync
        for pr in iter_recent_prs(cfg, window.start_day, progress=progress):
            _emit(progress, f"GitHub PR #{pr['number']}: loading comments/reviews")
            pr = enrich_pr_activity(cfg, pr, progress=progress)
            store.upsert_github_pr(project.key, cfg.owner, cfg.repo, pr)
            pr_count += 1
            _emit(progress, f"GitHub PR #{pr['number']}: stored; scanning commits")
            for position, commit_ref in enumerate(iter_pr_commits(cfg, pr["number"], window.start_day, progress=progress), start=1):
                sha = commit_ref["sha"]
                if store.has_github_commit(project.key, sha) and (
                    store.has_github_commit_files(project.key, sha) or commit_details_left <= 0
                ):
                    store.link_github_pr_commit(project.key, pr["number"], sha, position)
                    continue

                if commit_details_left > 0:
                    _emit(progress, f"GitHub commit {sha[:12]}: loading details; details left={commit_details_left}")
                    commit = get_commit(cfg, sha)
                    files = store.upsert_github_commit(
                        project.key,
                        cfg.owner,
                        cfg.repo,
                        commit,
                        max_files=cfg.max_commit_files_per_commit,
                        max_patch_bytes=cfg.max_patch_bytes,
                    )
                    commit_details_left -= 1
                else:
                    files = store.upsert_github_commit(project.key, cfg.owner, cfg.repo, commit_ref)

                store.link_github_pr_commit(project.key, pr["number"], sha, position)
                commit_count += 1
                commit_file_count += files
            _emit(progress, f"GitHub progress: PRs={pr_count}, commits={commit_count}, files={commit_file_count}")
        store.commit()
        _emit(progress, "GitHub: committed local changes")
        store.complete_sync(SOURCE, project.key, window.start_day, end_day, window_days)
        store.export_project(project.key)
        _emit(progress, "GitHub: exports refreshed")
        return GithubSyncResult(prs=pr_count, commits=commit_count, commit_files=commit_file_count)
    except Exception as exc:
        _emit(progress, f"GitHub sync failed: {describe_exception(exc)}")
        store.fail_sync(SOURCE, project.key, str(exc))
        raise


def iter_recent_prs(cfg: GithubConfig, start_day: date, progress=None) -> Iterable[Dict[str, Any]]:
    page = 1
    while True:
        _emit(progress, f"GitHub PR page {page}: requesting")
        items = get_json(
            f"{BASE_URL}/repos/{cfg.owner}/{cfg.repo}/pulls",
            params={
                "state": cfg.state,
                "sort": "updated",
                "direction": "desc",
                "per_page": cfg.per_page,
                "page": page,
            },
            headers=github_headers(),
        )
        _emit(progress, f"GitHub PR page {page}: received {len(items or [])} items")
        if not items:
            break

        found_in_window = False
        for pr in items:
            updated_day = github_timestamp_to_day(pr["updated_at"])
            if updated_day >= start_day:
                found_in_window = True
                yield pr

        oldest_day = github_timestamp_to_day(items[-1]["updated_at"])
        _emit(progress, f"GitHub PR page {page}: oldest updated day {oldest_day}")
        if oldest_day < start_day or not found_in_window:
            break
        page += 1


def enrich_pr_activity(cfg: GithubConfig, pr: Dict[str, Any], progress=None) -> Dict[str, Any]:
    pr_number = int(pr["number"])
    enriched = dict(pr)
    enriched["comments"] = list(iter_issue_comments(cfg, pr_number, progress=progress))
    enriched["review_comments"] = list(iter_review_comments(cfg, pr_number, progress=progress))
    enriched["reviews"] = list(iter_reviews(cfg, pr_number, progress=progress))
    return enriched


def iter_issue_comments(cfg: GithubConfig, pr_number: int, progress=None) -> Iterable[Dict[str, Any]]:
    yield from _iter_pr_collection(cfg, pr_number, "issues", "comments", progress=progress)


def iter_review_comments(cfg: GithubConfig, pr_number: int, progress=None) -> Iterable[Dict[str, Any]]:
    yield from _iter_pr_collection(cfg, pr_number, "pulls", "comments", progress=progress)


def iter_reviews(cfg: GithubConfig, pr_number: int, progress=None) -> Iterable[Dict[str, Any]]:
    yield from _iter_pr_collection(cfg, pr_number, "pulls", "reviews", progress=progress)


def _iter_pr_collection(
    cfg: GithubConfig,
    pr_number: int,
    area: str,
    collection: str,
    progress=None,
) -> Iterable[Dict[str, Any]]:
    page = 1
    while True:
        _emit(progress, f"GitHub PR #{pr_number} {area}/{collection} page {page}: requesting")
        items = get_json(
            f"{BASE_URL}/repos/{cfg.owner}/{cfg.repo}/{area}/{pr_number}/{collection}",
            params={
                "per_page": cfg.per_page,
                "page": page,
            },
            headers=github_headers(),
        )
        _emit(progress, f"GitHub PR #{pr_number} {area}/{collection} page {page}: received {len(items or [])} items")
        if not items:
            break
        yield from items
        if len(items) < cfg.per_page:
            break
        page += 1


def iter_pr_commits(cfg: GithubConfig, pr_number: int, start_day: date, progress=None) -> Iterable[Dict[str, Any]]:
    page = 1
    while True:
        _emit(progress, f"GitHub PR #{pr_number} commits page {page}: requesting")
        items = get_json(
            f"{BASE_URL}/repos/{cfg.owner}/{cfg.repo}/pulls/{pr_number}/commits",
            params={
                "per_page": cfg.per_page,
                "page": page,
            },
            headers=github_headers(),
        )
        _emit(progress, f"GitHub PR #{pr_number} commits page {page}: received {len(items or [])} items")
        if not items:
            break

        for item in items:
            commit_data = item.get("commit") or {}
            author_data = commit_data.get("author") or {}
            committer_data = commit_data.get("committer") or {}
            timestamp = committer_data.get("date") or author_data.get("date")
            if timestamp and github_timestamp_to_day(timestamp) >= start_day:
                yield item

        if len(items) < cfg.per_page:
            break
        page += 1


def get_commit(cfg: GithubConfig, sha: str) -> Dict[str, Any]:
    return get_json(
        f"{BASE_URL}/repos/{cfg.owner}/{cfg.repo}/commits/{sha}",
        headers=github_headers(),
    )


def _emit(progress, message: str) -> None:
    if progress:
        progress(message)


def _describe_window_plan(
    owner: str,
    repo: str,
    desired_start: date,
    end_day: date,
    window_days: int,
    state: Dict[str, Any] | None,
    force: bool,
) -> str:
    if not state:
        return (
            f"GitHub {owner}/{repo}: requested horizon {window_days}d "
            f"({desired_start}..{end_day}); no prior sync_state"
        )

    return (
        f"GitHub {owner}/{repo}: requested horizon {window_days}d "
        f"({desired_start}..{end_day}); "
        f"last loaded={state.get('min_loaded_day')}..{state.get('max_loaded_day')}, "
        f"last_success={state.get('last_success_at')}, force={force}"
    )