scripts/pr-ci-comment.mjs - canary - githubmirror/next.js

scripts/pr-ci-comment.mjs
920 строк23 KB
Sebastian "Sebbie" Silbermann
[ci] Match the test job summary style to the PR test report comment (#95882)
30 июл 2026, 15:41
30 июл 2026, 15:4134299ad
#!/usr/bin/env node

import { existsSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
import path from 'node:path'
import { buildTestReport } from './test-report.js'

const TEST_COMMENT_MARKER = '<!-- __NEXT_TEST_REPORT_COMMENT__ -->'
const STATS_COMMENT_MARKER = '<!-- __NEXT_STATS_COMMENT__ -->'
const MAX_COMMENT_LENGTH = 62_000
const MAX_RESULT_MESSAGE_LENGTH = 12_000

const LOG_TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\s?/
// Strip terminal color/control codes from test output before posting it to GitHub.
const ANSI_RE =
  // eslint-disable-next-line no-control-regex
  /(?:\u001B\][\s\S]*?(?:\u0007|\u001B\\|\u009C))|(?:[\u001B\u009B][[\]()#;?]*(?:\d{1,4}(?:[;:]\d{0,4})*)?[\dA-PR-TZcf-nq-uy=><~])/g

const COMMENT_AUTHOR = 'github-actions[bot]'

class GitHubClient {
  constructor({ token, owner, repo, dryRun }) {
    this.token = token
    this.owner = owner
    this.repo = repo
    this.dryRun = dryRun
  }

  async request(route, options = {}) {
    const res = await this.fetchApi(route, options)

    if (!res.ok) {
      throw new Error(
        await formatResponseError(res, `${options.method || 'GET'} ${route}`)
      )
    }

    if (res.status === 204) {
      return null
    }

    return res.json()
  }

  async fetchApi(route, options = {}) {
    return fetchWithRateLimitRetry(
      `https://api.github.com${route}`,
      {
        ...options,
        headers: {
          Accept: 'application/vnd.github+json',
          Authorization: `Bearer ${this.token}`,
          'X-GitHub-Api-Version': '2022-11-28',
          ...options.headers,
        },
      },
      `${options.method || 'GET'} ${route}`
    )
  }

  async paginate(route) {
    const results = []
    let page = 1

    while (true) {
      const separator = route.includes('?') ? '&' : '?'
      const data = await this.request(
        `${route}${separator}per_page=100&page=${page}`
      )
      const items = Array.isArray(data)
        ? data
        : data.jobs || data.artifacts || []
      results.push(...items)

      if (items.length < 100) {
        return results
      }

      page += 1
    }
  }

  async listJobsForRunAttempt(runId, runAttempt) {
    return this.paginate(
      `/repos/${this.owner}/${this.repo}/actions/runs/${runId}/attempts/${runAttempt}/jobs`
    )
  }

  async downloadJobLogs(jobId) {
    const route = `/repos/${this.owner}/${this.repo}/actions/jobs/${jobId}/logs`
    const res = await this.fetchApi(route, {
      redirect: 'manual',
    })

    if (!res.ok && res.status !== 302) {
      throw new Error(
        await formatResponseError(res, `GET job logs for job ${jobId}`)
      )
    }

    if (res.status === 302) {
      const location = res.headers.get('location')
      if (!location) {
        throw new Error(`Job ${jobId} log redirect did not include a location`)
      }

      const logsRes = await fetchWithRateLimitRetry(
        location,
        {},
        `GET redirected job logs for job ${jobId}`
      )
      if (!logsRes.ok) {
        throw new Error(
          await formatResponseError(
            logsRes,
            `GET redirected job logs for job ${jobId}`
          )
        )
      }
      return stripLogTimestamps(await logsRes.text())
    }

    return stripLogTimestamps(await res.text())
  }

  async listIssueComments(issueNumber) {
    return this.paginate(
      `/repos/${this.owner}/${this.repo}/issues/${issueNumber}/comments`
    )
  }

  async findExistingBotComment(issueNumber, marker, fallbackHeadings = []) {
    const comments = await this.listIssueComments(issueNumber)
    return [...comments].reverse().find((comment) => {
      if (comment.user?.login !== COMMENT_AUTHOR) {
        return false
      }

      return (
        comment.body?.includes(marker) ||
        fallbackHeadings.some((heading) => comment.body?.includes(heading))
      )
    })
  }

  async upsertIssueComment(issueNumber, marker, body, fallbackHeadings = []) {
    body = fitComment(body)

    const existing = await this.findExistingBotComment(
      issueNumber,
      marker,
      fallbackHeadings
    )

    if (this.dryRun) {
      console.log(
        `[dry-run] ${existing ? 'Would update' : 'Would create'} comment for #${issueNumber}`
      )
      console.log(body)
      return
    }

    if (existing) {
      try {
        await this.request(
          `/repos/${this.owner}/${this.repo}/issues/comments/${existing.id}`,
          {
            method: 'PATCH',
            body: JSON.stringify({ body }),
          }
        )
        console.log(`Updated comment ${existing.html_url}`)
        return
      } catch (err) {
        console.log(
          `Failed to update existing comment ${existing.id}, creating a new one`,
          err
        )
      }
    }

    const created = await this.request(
      `/repos/${this.owner}/${this.repo}/issues/${issueNumber}/comments`,
      {
        method: 'POST',
        body: JSON.stringify({ body }),
      }
    )
    console.log(`Created comment ${created.html_url}`)
  }

  async insertIssueCommentIfMissing(
    issueNumber,
    marker,
    body,
    fallbackHeadings = []
  ) {
    body = fitComment(body)

    const existing = await this.findExistingBotComment(
      issueNumber,
      marker,
      fallbackHeadings
    )

    if (existing) {
      console.log(
        `Existing comment ${existing.html_url} found for #${issueNumber}; requested-phase is create-only, leaving it alone`
      )
      return
    }

    if (this.dryRun) {
      console.log(
        `[dry-run] Would create placeholder comment for #${issueNumber}`
      )
      console.log(body)
      return
    }

    const created = await this.request(
      `/repos/${this.owner}/${this.repo}/issues/${issueNumber}/comments`,
      {
        method: 'POST',
        body: JSON.stringify({ body }),
      }
    )
    console.log(`Created placeholder comment ${created.html_url}`)
  }

  async findPullRequestForCommit(sha) {
    const pulls = await this.request(
      `/repos/${this.owner}/${this.repo}/commits/${sha}/pulls`,
      {
        headers: {
          Accept: 'application/vnd.github+json',
        },
      }
    )

    return pulls?.[0]
  }

  async getPullRequest(number) {
    return this.request(`/repos/${this.owner}/${this.repo}/pulls/${number}`)
  }
}

async function main() {
  const dryRun = process.env.PR_CI_COMMENT_DRY_RUN === '1'
  const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN
  if (!token && !dryRun) {
    throw new Error('Missing GITHUB_TOKEN')
  }

  const eventPath = process.env.GITHUB_EVENT_PATH
  if (!eventPath) {
    throw new Error('Missing GITHUB_EVENT_PATH')
  }

  const event = JSON.parse(await readFile(eventPath, 'utf8'))
  const workflowRun = event.workflow_run
  if (!workflowRun) {
    throw new Error('This script must run from a workflow_run event')
  }

  const [owner, repo] = (
    event.repository?.full_name ||
    process.env.GITHUB_REPOSITORY ||
    ''
  ).split('/')

  if (!owner || !repo) {
    throw new Error('Unable to resolve repository owner/name')
  }

  const github = new GitHubClient({
    token,
    owner,
    repo,
    dryRun,
  })

  const pr = await getPullRequestMetadata(github, workflowRun)

  if (!pr?.number) {
    console.log(
      `No pull request found for workflow run ${workflowRun.id}, skipping`
    )
    return
  }

  const phase = event.action
  if (phase !== 'requested' && phase !== 'completed') {
    console.log(`Ignoring workflow_run action "${phase}"`)
    return
  }

  if (workflowRun.name === 'Generate Stats') {
    await handleStatsWorkflow({ github, workflowRun, pr, phase })
    return
  }

  if (workflowRun.name === 'build-and-test') {
    await handleBuildAndTestWorkflow({
      github,
      workflowRun,
      pr,
      owner,
      repo,
      phase,
    })
    return
  }

  console.log(`Ignoring workflow "${workflowRun.name}"`)
}

async function getPullRequestMetadata(github, workflowRun) {
  const fromArtifact = await readPullRequestMetadataArtifact()
  if (fromArtifact?.number) {
    const metadata = await validatePullRequestMetadata(
      github,
      workflowRun,
      fromArtifact,
      'PR metadata artifact'
    )
    if (metadata) {
      return metadata
    }
  }

  const fromPayload = workflowRun.pull_requests?.[0]
  if (fromPayload?.number) {
    const metadata = await validatePullRequestMetadata(
      github,
      workflowRun,
      {
        number: fromPayload.number,
        headSha: fromPayload.head?.sha || workflowRun.head_sha,
        headRef: fromPayload.head?.ref || workflowRun.head_branch,
        headRepo: fromPayload.head?.repo?.full_name,
        baseRef: fromPayload.base?.ref,
      },
      'workflow_run payload'
    )
    if (metadata) {
      return metadata
    }
  }

  if (workflowRun.head_sha && github.token) {
    const pull = await github.findPullRequestForCommit(workflowRun.head_sha)
    if (pull?.number) {
      return validatePullRequestMetadata(
        github,
        workflowRun,
        {
          number: pull.number,
          headSha: workflowRun.head_sha,
          headRef: workflowRun.head_branch,
          headRepo: workflowRun.head_repository?.full_name,
          baseRef: pull.base?.ref,
        },
        'commit-associated PR'
      )
    }
  }

  return null
}

async function validatePullRequestMetadata(
  github,
  workflowRun,
  candidate,
  source
) {
  if (!candidate?.number) {
    return null
  }

  if (!github.token) {
    return candidate
  }

  try {
    const pull = await github.getPullRequest(candidate.number)
    const expectedHeadSha = candidate.headSha || workflowRun.head_sha
    if (!expectedHeadSha) {
      console.log(
        `Ignoring ${source} for #${candidate.number}: missing workflow head SHA`
      )
      return null
    }

    if (pull.head?.sha !== expectedHeadSha) {
      console.log(
        `Ignoring ${source} for #${candidate.number}: current PR head ${pull.head?.sha} did not match workflow head ${expectedHeadSha}`
      )
      return null
    }

    if (!candidateMatchesWorkflowRun(candidate, workflowRun)) {
      console.log(
        `Ignoring ${source} for #${candidate.number}: metadata did not match workflow run`
      )
      return null
    }

    return {
      number: pull.number,
      headSha: pull.head?.sha || expectedHeadSha,
      headRef: pull.head?.ref || candidate.headRef || workflowRun.head_branch,
      headRepo: pull.head?.repo?.full_name || candidate.headRepo,
      baseRef: pull.base?.ref || candidate.baseRef,
      isFork: pull.head?.repo?.full_name !== pull.base?.repo?.full_name,
    }
  } catch (err) {
    console.log(`Failed to validate ${source}`, err)
    return null
  }
}

function candidateMatchesWorkflowRun(candidate, workflowRun) {
  if (candidate.headSha && candidate.headSha === workflowRun.head_sha) {
    return true
  }

  const workflowHeadRepo = workflowRun.head_repository?.full_name
  return Boolean(
    candidate.headRef &&
      candidate.headRepo &&
      workflowRun.head_branch &&
      workflowHeadRepo &&
      candidate.headRef === workflowRun.head_branch &&
      candidate.headRepo === workflowHeadRepo
  )
}

async function readPullRequestMetadataArtifact() {
  const metadataPath =
    process.env.PR_CI_METADATA_PATH || path.join('pr-ci-metadata', 'pr.json')

  if (!existsSync(metadataPath)) {
    return null
  }

  const metadata = JSON.parse(await readFile(metadataPath, 'utf8'))
  return {
    number: Number(metadata.number),
    headSha: metadata.headSha,
    headRef: metadata.headRef,
    headRepo: metadata.headRepo,
    baseRef: metadata.baseRef,
    isFork: metadata.isFork === true || metadata.isFork === 'true',
  }
}

async function handleStatsWorkflow({ github, workflowRun, pr, phase }) {
  if (phase === 'requested') {
    const sha = pr.headSha || workflowRun.head_sha
    const body = [
      STATS_COMMENT_MARKER,
      '## Stats in progress',
      '',
      `Commit: ${sha}`,
      `[View workflow run](${workflowRun.html_url})`,
      '',
    ].join('\n')

    await github.insertIssueCommentIfMissing(
      pr.number,
      STATS_COMMENT_MARKER,
      body,
      ['## Stats from current PR']
    )
    return
  }

  // Look for a stats block before reacting to the run conclusion. A single
  // bundler timing out cancels its job and marks the whole run "cancelled", but
  // the aggregate still emits stats for the bundlers that finished. Treating a
  // cancelled run as "no data" up front would discard that partial comment, so
  // we post whatever the aggregate produced first and only fall back to a
  // cancelled/skipped notice when there is genuinely no block.
  const jobs = await github.listJobsForRunAttempt(
    workflowRun.id,
    workflowRun.run_attempt || 1
  )
  const candidates = jobs.filter((job) => /aggregate stats/i.test(job.name))

  for (const job of candidates) {
    let logs
    try {
      logs = await github.downloadJobLogs(job.id)
    } catch (err) {
      // A cancelled or skipped aggregate job may have no retrievable logs.
      console.log(`Failed to download logs for job ${job.id}`, err)
      continue
    }

    const stats = extractDelimitedBlock(
      logs,
      '--stats start--',
      '--stats end--'
    )

    if (!stats) {
      continue
    }

    let body = stats
      .replace('âš ï¸', '\u26a0\ufe0f')
      .replace('âœ“', '\u2713')
      .trim()

    if (!body.includes(STATS_COMMENT_MARKER)) {
      body = `${STATS_COMMENT_MARKER}\n${body}`
    }

    body += `\n\nCommit: ${pr.headSha || workflowRun.head_sha}`

    await github.upsertIssueComment(pr.number, STATS_COMMENT_MARKER, body, [
      '## Stats from current PR',
    ])
    return
  }

  const sha = pr.headSha || workflowRun.head_sha

  // No stats block. A cancelled conclusion here means the whole run was
  // cancelled (superseded, or every bundler cancelled) rather than an
  // individual bundler, since a partial run would have produced a block above.
  if (workflowRun.conclusion === 'cancelled') {
    console.log('No stats block found and the run was cancelled.')
    const body = [
      STATS_COMMENT_MARKER,
      '## Stats cancelled',
      '',
      `Commit: ${sha}`,
      `[View workflow run](${workflowRun.html_url})`,
      '',
    ].join('\n')

    await github.upsertIssueComment(pr.number, STATS_COMMENT_MARKER, body, [
      '## Stats from current PR',
    ])
    return
  }

  console.log(
    'No stats block found in the completed stats workflow. Assuming stats were skipped.'
  )

  const body = [
    STATS_COMMENT_MARKER,
    '## Stats skipped',
    '',
    `Commit: ${sha}`,
    `[View workflow run](${workflowRun.html_url})`,
    '',
  ].join('\n')
  await github.upsertIssueComment(pr.number, STATS_COMMENT_MARKER, body, [
    '## Stats from current PR',
  ])
}

async function handleBuildAndTestWorkflow({
  github,
  workflowRun,
  pr,
  owner,
  repo,
  phase,
}) {
  if (phase === 'requested') {
    const sha = pr.headSha || workflowRun.head_sha
    const body = [
      TEST_COMMENT_MARKER,
      '## Tests in progress',
      '',
      `Commit: ${sha}`,
      `[View workflow run](${workflowRun.html_url})`,
      '',
    ].join('\n')

    await github.insertIssueCommentIfMissing(
      pr.number,
      TEST_COMMENT_MARKER,
      body,
      ['## Failing test suites', '## Failing CI jobs']
    )
    return
  }

  if (workflowRun.conclusion === 'success') {
    const body = [
      TEST_COMMENT_MARKER,
      '## Tests Passed',
      '',
      '<!-- ## Failing test suites -->',
      `Commit: ${pr.headSha || workflowRun.head_sha}`,
      '',
    ].join('\n')

    await github.upsertIssueComment(pr.number, TEST_COMMENT_MARKER, body, [
      '## Failing test suites',
      '## Failing CI jobs',
    ])
    return
  }

  const jobs = await github.listJobsForRunAttempt(
    workflowRun.id,
    workflowRun.run_attempt || 1
  )
  const failedJobs = jobs.filter(
    (job) =>
      ['failure', 'timed_out'].includes(job.conclusion) &&
      job.name !== 'thank you, next'
  )

  const failedSuites = []
  const otherFailures = []

  await mapLimit(failedJobs, 4, async (job) => {
    let logs = ''
    try {
      logs = await github.downloadJobLogs(job.id)
    } catch (err) {
      otherFailures.push({ job, reason: err.message })
      return
    }

    const parsedSuites = parseFailedSuitesFromLogs(logs, job)

    if (parsedSuites.length === 0) {
      otherFailures.push({ job })
    } else {
      failedSuites.push(...parsedSuites)
    }
  })

  const body = buildTestReportComment({
    failedSuites,
    otherFailures,
    owner,
    repo,
    sha: pr.headSha || workflowRun.head_sha,
  })

  await github.upsertIssueComment(pr.number, TEST_COMMENT_MARKER, body, [
    '## Failing test suites',
    '## Failing CI jobs',
  ])
}

function parseFailedSuitesFromLogs(logs, job) {
  const blocks = extractJsonBlocks(logs)
  const failedSuites = []

  for (const testData of blocks) {
    const testResults = testData.testResults || []
    for (const testResult of testResults) {
      const failedAssertions = (testResult.assertionResults || []).filter(
        (res) => res.status === 'failed'
      )
      const failed =
        testResult.status === 'failed' ||
        failedAssertions.length > 0 ||
        (testResults.length === 1 && testData.numFailedTests > 0)

      if (!failed) {
        continue
      }

      const groupedFails = new Map()
      for (const fail of failedAssertions) {
        const ancestorKey = (fail.ancestorTitles || []).join(' > ')
        if (!groupedFails.has(ancestorKey)) {
          groupedFails.set(ancestorKey, [])
        }
        groupedFails.get(ancestorKey).push(fail)
      }

      failedSuites.push({
        job,
        mode: testData.processEnv?.NEXT_TEST_MODE,
        testPath: normalizeTestPath(testResult.name),
        resultMessage: truncate(
          stripAnsi(testResult.message || collectFailureMessages(testResult)),
          MAX_RESULT_MESSAGE_LENGTH
        ),
        groups: groupedFails,
      })
    }
  }

  return failedSuites
}

function extractJsonBlocks(logs) {
  const blocks = []
  const re = /--test output start--\s*(\{[\s\S]*?\})\s*--test output end--/g
  let match

  while ((match = re.exec(logs))) {
    try {
      blocks.push(JSON.parse(match[1]))
    } catch (err) {
      console.log('Failed to parse test output block', err)
    }
  }

  return blocks
}

function extractDelimitedBlock(logs, start, end) {
  const startIndex = logs.indexOf(start)
  if (startIndex === -1) {
    return null
  }

  const contentStart = startIndex + start.length
  const endIndex = logs.indexOf(end, contentStart)
  if (endIndex === -1) {
    return null
  }

  return logs.slice(contentStart, endIndex).trim()
}

function buildTestReportComment({
  failedSuites,
  otherFailures,
  owner,
  repo,
  sha,
}) {
  const suites = failedSuites
    .sort((a, b) =>
      `${a.job.name}:${a.testPath}`.localeCompare(`${b.job.name}:${b.testPath}`)
    )
    .map((suite) => {
      // Unlike `run-tests.js`, which reads them from its environment, the
      // suite properties have to be derived from the CI job name here.
      const jobName = suite.job.name.toLowerCase()
      const isCacheComponents = jobName.includes('cache components')

      return {
        jobName: suite.job.name,
        jobUrl: suite.job.html_url,
        mode: suite.mode,
        testPath: suite.testPath,
        isTurbopack: jobName.includes('turbopack') || isCacheComponents,
        isRspack: jobName.includes('rspack'),
        isExperimental: jobName.includes('experimental') || isCacheComponents,
        isPPR: jobName.includes('ppr'),
        failedCases: [...suite.groups.keys()]
          .sort()
          .flatMap((group) =>
            suite.groups
              .get(group)
              .map((fail) => `${group ? `${group} > ` : ''}${fail.title}`)
          ),
        resultMessage: suite.resultMessage,
      }
    })

  return buildTestReport({
    marker: TEST_COMMENT_MARKER,
    owner,
    repo,
    sha,
    suites,
    otherFailures: otherFailures
      .sort((a, b) => a.job.name.localeCompare(b.job.name))
      .map(({ job, reason }) => ({
        name: job.name,
        url: job.html_url,
        reason,
      })),
  })
}

function normalizeTestPath(testName) {
  const normalized = String(testName || '').replaceAll('\\', '/')
  const match = normalized.match(/(?:^|\/)(test\/.*)$/)
  return match?.[1] || normalized
}

function collectFailureMessages(testResult) {
  return (testResult.assertionResults || [])
    .flatMap((assertion) => assertion.failureMessages || [])
    .join('\n\n')
}

function stripLogTimestamps(logs) {
  return logs
    .split('\n')
    .map((line) => line.replace(LOG_TIMESTAMP_RE, ''))
    .join('\n')
}

function stripAnsi(value) {
  return String(value || '').replace(ANSI_RE, '')
}

function truncate(value, maxLength) {
  if (value.length <= maxLength) {
    return value
  }

  return `${value.slice(0, maxLength)}\n\n... truncated ...`
}

function fitComment(body) {
  if (body.length <= MAX_COMMENT_LENGTH) {
    return body
  }

  return `${body.slice(0, MAX_COMMENT_LENGTH)}\n\n... truncated to fit in one GitHub comment ...`
}

async function fetchWithRateLimitRetry(url, options, description) {
  const res = await fetch(url, options)
  const retryDelay = getRetryAfterMs(res.headers)

  if ((res.status === 403 || res.status === 429) && retryDelay !== null) {
    console.log(
      `${description} was rate limited; retrying after ${Math.ceil(retryDelay / 1000)}s`
    )
    await sleep(retryDelay)
    return fetch(url, options)
  }

  return res
}

async function formatResponseError(res, description) {
  const text = await res.text().catch(() => '')
  const details = formatRateLimitDetails(res.headers)
  return `${description} failed with ${res.status}${details ? ` (${details})` : ''}: ${text}`
}

function formatRateLimitDetails(headers) {
  const details = []
  const remaining = headers.get('x-ratelimit-remaining')
  const resource = headers.get('x-ratelimit-resource')
  const retryAfter = headers.get('retry-after')
  const reset = headers.get('x-ratelimit-reset')

  if (remaining !== null) {
    details.push(`remaining=${remaining}`)
  }
  if (resource) {
    details.push(`resource=${resource}`)
  }
  if (reset) {
    details.push(`reset=${formatRateLimitReset(reset)}`)
  }
  if (retryAfter) {
    details.push(`retry-after=${retryAfter}`)
  }

  return details.join(', ')
}

function formatRateLimitReset(value) {
  const timestamp = Number(value)
  if (!Number.isFinite(timestamp)) {
    return value
  }

  return `${value}/${new Date(timestamp * 1000).toISOString()}`
}

function getRetryAfterMs(headers) {
  const retryAfter = headers.get('retry-after')
  if (!retryAfter) {
    return null
  }

  const seconds = Number(retryAfter)
  if (Number.isFinite(seconds)) {
    return Math.max(0, seconds * 1000)
  }

  const retryAt = Date.parse(retryAfter)
  if (Number.isFinite(retryAt)) {
    return Math.max(0, retryAt - Date.now())
  }

  return null
}

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms))
}

async function mapLimit(items, limit, mapper) {
  const queue = [...items]
  const workers = Array.from(
    { length: Math.min(limit, queue.length) },
    async () => {
      while (queue.length > 0) {
        const item = queue.shift()
        await mapper(item)
      }
    }
  )

  await Promise.all(workers)
}

main().catch((err) => {
  console.error(err)
  process.exit(1)
})