pnpm

1
import assert from 'assert'
2
import path from 'path'
3
import util from 'util'
4
import { calcDepState, type DepsStateCache } from '@pnpm/calc-dep-state'
5
import { skippedOptionalDependencyLogger } from '@pnpm/core-loggers'
6
import { runPostinstallHooks } from '@pnpm/lifecycle'
7
import { linkBins, linkBinsOfPackages } from '@pnpm/link-bins'
8
import { logger } from '@pnpm/logger'
9
import { hardLinkDir } from '@pnpm/worker'
10
import { readPackageJsonFromDir, safeReadPackageJsonFromDir } from '@pnpm/read-package-json'
11
import { type StoreController } from '@pnpm/store-controller-types'
12
import { applyPatchToDir } from '@pnpm/patching.apply-patch'
13
import { type DependencyManifest } from '@pnpm/types'
14
import pDefer, { type DeferredPromise } from 'p-defer'
15
import pickBy from 'ramda/src/pickBy'
16
import runGroups from 'run-groups'
17
import { buildSequence, type DependenciesGraph, type DependenciesGraphNode } from './buildSequence'
18

19
export type { DepsStateCache }
20

21
export async function buildModules (
22
  depGraph: DependenciesGraph,
23
  rootDepPaths: string[],
24
  opts: {
25
    allowBuild?: (pkgName: string) => boolean
26
    childConcurrency?: number
27
    depsToBuild?: Set<string>
28
    depsStateCache: DepsStateCache
29
    extraBinPaths?: string[]
30
    extraNodePaths?: string[]
31
    extraEnv?: Record<string, string>
32
    ignoreScripts?: boolean
33
    lockfileDir: string
34
    optional: boolean
35
    preferSymlinkedExecutables?: boolean
36
    rawConfig: object
37
    unsafePerm: boolean
38
    userAgent: string
39
    scriptsPrependNodePath?: boolean | 'warn-only'
40
    scriptShell?: string
41
    shellEmulator?: boolean
42
    sideEffectsCacheWrite: boolean
43
    storeController: StoreController
44
    rootModulesDir: string
45
    hoistedLocations?: Record<string, string[]>
46
  }
47
): Promise<void> {
48
  const warn = (message: string) => {
49
    logger.warn({ message, prefix: opts.lockfileDir })
50
  }
51
  // postinstall hooks
52

53
  const buildDepOpts = {
54
    ...opts,
55
    builtHoistedDeps: opts.hoistedLocations ? {} : undefined,
56
    warn,
57
  }
58
  const chunks = buildSequence(depGraph, rootDepPaths)
59
  const ignoredPkgs = new Set<string>()
60
  const allowBuild = opts.allowBuild
61
    ? (pkgName: string) => {
62
      if (opts.allowBuild!(pkgName)) return true
63
      ignoredPkgs.add(pkgName)
64
      return false
65
    }
66
    : () => true
67
  const groups = chunks.map((chunk) => {
68
    chunk = chunk.filter((depPath) => {
69
      const node = depGraph[depPath]
70
      return (node.requiresBuild || node.patchFile != null) && !node.isBuilt
71
    })
72
    if (opts.depsToBuild != null) {
73
      chunk = chunk.filter((depPath) => opts.depsToBuild!.has(depPath))
74
    }
75

76
    return chunk.map((depPath: string) =>
77
      async () => {
78
        return buildDependency(depPath, depGraph, {
79
          ...buildDepOpts,
80
          ignoreScripts: Boolean(buildDepOpts.ignoreScripts) || !allowBuild(depGraph[depPath].name),
81
        })
82
      }
83
    )
84
  })
85
  await runGroups(opts.childConcurrency ?? 4, groups)
86
  if (ignoredPkgs.size > 0) {
87
    logger.info({
88
      message: `The following dependencies have build scripts that were ignored: ${Array.from(ignoredPkgs).sort().join(', ')}`,
89
      prefix: opts.lockfileDir,
90
    })
91
  }
92
}
93

94
async function buildDependency (
95
  depPath: string,
96
  depGraph: DependenciesGraph,
97
  opts: {
98
    extraBinPaths?: string[]
99
    extraNodePaths?: string[]
100
    extraEnv?: Record<string, string>
101
    depsStateCache: DepsStateCache
102
    ignoreScripts?: boolean
103
    lockfileDir: string
104
    optional: boolean
105
    preferSymlinkedExecutables?: boolean
106
    rawConfig: object
107
    rootModulesDir: string
108
    scriptsPrependNodePath?: boolean | 'warn-only'
109
    scriptShell?: string
110
    shellEmulator?: boolean
111
    sideEffectsCacheWrite: boolean
112
    storeController: StoreController
113
    unsafePerm: boolean
114
    hoistedLocations?: Record<string, string[]>
115
    builtHoistedDeps?: Record<string, DeferredPromise<void>>
116
    warn: (message: string) => void
117
  }
118
): Promise<void> {
119
  const depNode = depGraph[depPath]
120
  if (!depNode.filesIndexFile) return
121
  if (opts.builtHoistedDeps) {
122
    if (opts.builtHoistedDeps[depNode.depPath]) {
123
      await opts.builtHoistedDeps[depNode.depPath].promise
124
      return
125
    }
126
    opts.builtHoistedDeps[depNode.depPath] = pDefer()
127
  }
128
  try {
129
    await linkBinsOfDependencies(depNode, depGraph, opts)
130
    const isPatched = depNode.patchFile?.path != null
131
    if (isPatched) {
132
      applyPatchToDir({ patchedDir: depNode.dir, patchFilePath: depNode.patchFile!.path })
133
    }
134
    const hasSideEffects = !opts.ignoreScripts && await runPostinstallHooks({
135
      depPath,
136
      extraBinPaths: opts.extraBinPaths,
137
      extraEnv: opts.extraEnv,
138
      initCwd: opts.lockfileDir,
139
      optional: depNode.optional,
140
      pkgRoot: depNode.dir,
141
      rawConfig: opts.rawConfig,
142
      rootModulesDir: opts.rootModulesDir,
143
      scriptsPrependNodePath: opts.scriptsPrependNodePath,
144
      scriptShell: opts.scriptShell,
145
      shellEmulator: opts.shellEmulator,
146
      unsafePerm: opts.unsafePerm || false,
147
    })
148
    if ((isPatched || hasSideEffects) && opts.sideEffectsCacheWrite) {
149
      try {
150
        const sideEffectsCacheKey = calcDepState(depGraph, opts.depsStateCache, depPath, {
151
          patchFileHash: depNode.patchFile?.hash,
152
          isBuilt: hasSideEffects,
153
        })
154
        await opts.storeController.upload(depNode.dir, {
155
          sideEffectsCacheKey,
156
          filesIndexFile: depNode.filesIndexFile,
157
        })
158
      } catch (err: unknown) {
159
        assert(util.types.isNativeError(err))
160
        if (err && 'statusCode' in err && err.statusCode === 403) {
161
          logger.warn({
162
            message: `The store server disabled upload requests, could not upload ${depNode.dir}`,
163
            prefix: opts.lockfileDir,
164
          })
165
        } else {
166
          logger.warn({
167
            error: err,
168
            message: `An error occurred while uploading ${depNode.dir}`,
169
            prefix: opts.lockfileDir,
170
          })
171
        }
172
      }
173
    }
174
  } catch (err: unknown) {
175
    assert(util.types.isNativeError(err))
176
    if (depNode.optional) {
177
      // TODO: add parents field to the log
178
      const pkg = await readPackageJsonFromDir(path.join(depNode.dir)) as DependencyManifest
179
      skippedOptionalDependencyLogger.debug({
180
        details: err.toString(),
181
        package: {
182
          id: depNode.dir,
183
          name: pkg.name,
184
          version: pkg.version,
185
        },
186
        prefix: opts.lockfileDir,
187
        reason: 'build_failure',
188
      })
189
      return
190
    }
191
    throw err
192
  } finally {
193
    const hoistedLocationsOfDep = opts.hoistedLocations?.[depNode.depPath]
194
    if (hoistedLocationsOfDep) {
195
      // There is no need to build the same package in every location.
196
      // We just copy the built package to every location where it is present.
197
      const currentHoistedLocation = path.relative(opts.lockfileDir, depNode.dir)
198
      const nonBuiltHoistedDeps = hoistedLocationsOfDep?.filter((hoistedLocation) => hoistedLocation !== currentHoistedLocation)
199
      await hardLinkDir(depNode.dir, nonBuiltHoistedDeps)
200
    }
201
    if (opts.builtHoistedDeps) {
202
      opts.builtHoistedDeps[depNode.depPath].resolve()
203
    }
204
  }
205
}
206

207
export async function linkBinsOfDependencies (
208
  depNode: DependenciesGraphNode,
209
  depGraph: DependenciesGraph,
210
  opts: {
211
    extraNodePaths?: string[]
212
    optional: boolean
213
    preferSymlinkedExecutables?: boolean
214
    warn: (message: string) => void
215
  }
216
): Promise<void> {
217
  const childrenToLink: Record<string, string> = opts.optional
218
    ? depNode.children
219
    : pickBy((child, childAlias) => !depNode.optionalDependencies.has(childAlias), depNode.children)
220

221
  const binPath = path.join(depNode.dir, 'node_modules/.bin')
222

223
  const pkgNodes = [
224
    ...Object.entries(childrenToLink)
225
      .map(([alias, childDepPath]) => ({ alias, dep: depGraph[childDepPath] }))
226
      .filter(({ alias, dep }) => {
227
        if (!dep) {
228
          // TODO: Try to reproduce this issue with a test in @pnpm/core
229
          logger.debug({ message: `Failed to link bins of "${alias}" to "${binPath}". This is probably not an issue.` })
230
          return false
231
        }
232
        return dep.hasBin && dep.installable !== false
233
      })
234
      .map(({ dep }) => dep),
235
    depNode,
236
  ]
237
  const pkgs = await Promise.all(pkgNodes
238
    .map(async (dep) => ({
239
      location: dep.dir,
240
      manifest: await dep.fetchingBundledManifest?.() ?? (await safeReadPackageJsonFromDir(dep.dir) as DependencyManifest) ?? {},
241
    }))
242
  )
243

244
  await linkBinsOfPackages(pkgs, binPath, {
245
    extraNodePaths: opts.extraNodePaths,
246
    preferSymlinkedExecutables: opts.preferSymlinkedExecutables,
247
  })
248

249
  // link also the bundled dependencies` bins
250
  if (depNode.hasBundledDependencies) {
251
    const bundledModules = path.join(depNode.dir, 'node_modules')
252
    await linkBins(bundledModules, binPath, {
253
      extraNodePaths: opts.extraNodePaths,
254
      preferSymlinkedExecutables: opts.preferSymlinkedExecutables,
255
      warn: opts.warn,
256
    })
257
  }
258
}
259