pnpm

1
import path from 'path'
2
import { calcDepState, type DepsStateCache } from '@pnpm/calc-dep-state'
3
import {
4
  progressLogger,
5
  removalLogger,
6
  statsLogger,
7
} from '@pnpm/core-loggers'
8
import {
9
  type DepHierarchy,
10
  type DependenciesGraph,
11
} from '@pnpm/deps.graph-builder'
12
import { linkBins } from '@pnpm/link-bins'
13
import { logger } from '@pnpm/logger'
14
import {
15
  type PackageFilesResponse,
16
  type StoreController,
17
} from '@pnpm/store-controller-types'
18
import pLimit from 'p-limit'
19
import difference from 'ramda/src/difference'
20
import isEmpty from 'ramda/src/isEmpty'
21
import rimraf from '@zkochan/rimraf'
22

23
const limitLinking = pLimit(16)
24

25
export async function linkHoistedModules (
26
  storeController: StoreController,
27
  graph: DependenciesGraph,
28
  prevGraph: DependenciesGraph,
29
  hierarchy: DepHierarchy,
30
  opts: {
31
    depsStateCache: DepsStateCache
32
    disableRelinkLocalDirDeps?: boolean
33
    force: boolean
34
    ignoreScripts: boolean
35
    lockfileDir: string
36
    preferSymlinkedExecutables?: boolean
37
    sideEffectsCacheRead: boolean
38
  }
39
): Promise<void> {
40
  // TODO: remove nested node modules first
41
  const dirsToRemove = difference(
42
    Object.keys(prevGraph),
43
    Object.keys(graph)
44
  )
45
  statsLogger.debug({
46
    prefix: opts.lockfileDir,
47
    removed: dirsToRemove.length,
48
  })
49
  // We should avoid removing unnecessary directories while simultaneously adding new ones.
50
  // Doing so can sometimes lead to a race condition when linking commands to `node_modules/.bin`.
51
  await Promise.all(dirsToRemove.map((dir) => tryRemoveDir(dir)))
52
  await Promise.all(
53
    Object.entries(hierarchy)
54
      .map(([parentDir, depsHierarchy]) => {
55
        function warn (message: string) {
56
          logger.info({
57
            message,
58
            prefix: parentDir,
59
          })
60
        }
61
        return linkAllPkgsInOrder(storeController, graph, prevGraph, depsHierarchy, parentDir, {
62
          ...opts,
63
          warn,
64
        })
65
      })
66
  )
67
}
68

69
async function tryRemoveDir (dir: string): Promise<void> {
70
  removalLogger.debug(dir)
71
  try {
72
    await rimraf(dir)
73
  } catch (err: any) { // eslint-disable-line
74
    /* Just ignoring for now. Not even logging.
75
    logger.warn({
76
      error: err,
77
      message: `Failed to remove "${pathToRemove}"`,
78
      prefix: lockfileDir,
79
    })
80
    */
81
  }
82
}
83

84
async function linkAllPkgsInOrder (
85
  storeController: StoreController,
86
  graph: DependenciesGraph,
87
  prevGraph: DependenciesGraph,
88
  hierarchy: DepHierarchy,
89
  parentDir: string,
90
  opts: {
91
    depsStateCache: DepsStateCache
92
    disableRelinkLocalDirDeps?: boolean
93
    force: boolean
94
    ignoreScripts: boolean
95
    lockfileDir: string
96
    preferSymlinkedExecutables?: boolean
97
    sideEffectsCacheRead: boolean
98
    warn: (message: string) => void
99
  }
100
): Promise<void> {
101
  const _calcDepState = calcDepState.bind(null, graph, opts.depsStateCache)
102
  await Promise.all(
103
    Object.entries(hierarchy).map(async ([dir, deps]) => {
104
      const depNode = graph[dir]
105
      if (depNode.fetching) {
106
        let filesResponse!: PackageFilesResponse
107
        try {
108
          filesResponse = (await depNode.fetching()).files
109
        } catch (err: any) { // eslint-disable-line
110
          if (depNode.optional) return
111
          throw err
112
        }
113

114
        depNode.requiresBuild = filesResponse.requiresBuild
115
        let sideEffectsCacheKey: string | undefined
116
        if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
117
          sideEffectsCacheKey = _calcDepState(dir, {
118
            isBuilt: !opts.ignoreScripts && depNode.requiresBuild,
119
            patchFileHash: depNode.patchFile?.hash,
120
          })
121
        }
122
        // Limiting the concurrency here fixes an out of memory error.
123
        // It is not clear why it helps as importing is also limited inside fs.indexed-pkg-importer.
124
        // The out of memory error was reproduced on the teambit/bit repository with the "rootComponents" feature turned on
125
        await limitLinking(async () => {
126
          const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
127
            filesResponse,
128
            force: true,
129
            disableRelinkLocalDirDeps: opts.disableRelinkLocalDirDeps,
130
            keepModulesDir: true,
131
            requiresBuild: depNode.patchFile != null || depNode.requiresBuild,
132
            sideEffectsCacheKey,
133
          })
134
          if (importMethod) {
135
            progressLogger.debug({
136
              method: importMethod,
137
              requester: opts.lockfileDir,
138
              status: 'imported',
139
              to: depNode.dir,
140
            })
141
          }
142
          depNode.isBuilt = isBuilt
143
        })
144
      }
145
      return linkAllPkgsInOrder(storeController, graph, prevGraph, deps, dir, opts)
146
    })
147
  )
148
  const modulesDir = path.join(parentDir, 'node_modules')
149
  const binsDir = path.join(modulesDir, '.bin')
150
  await linkBins(modulesDir, binsDir, {
151
    allowExoticManifests: true,
152
    preferSymlinkedExecutables: opts.preferSymlinkedExecutables,
153
    warn: opts.warn,
154
  })
155
}
156