Edit File - File Manager

Edit File: load-actual.js

// mix-in implementing the loadActual method

const { relative, dirname, resolve, join, normalize } = require('node:path')

const rpj = require('read-package-json-fast')
const { readdirScoped } = require('@npmcli/fs')
const { walkUp } = require('walk-up-path')
const ancestorPath = require('common-ancestor-path')
const treeCheck = require('../tree-check.js')

const Shrinkwrap = require('../shrinkwrap.js')
const calcDepFlags = require('../calc-dep-flags.js')
const Node = require('../node.js')
const Link = require('../link.js')
const realpath = require('../realpath.js')

// public symbols
const _changePath = Symbol.for('_changePath')
const _setWorkspaces = Symbol.for('setWorkspaces')
const _rpcache = Symbol.for('realpathCache')
const _stcache = Symbol.for('statCache')

module.exports = cls => class ActualLoader extends cls {
  #actualTree
  // ensure when walking the tree that we don't call loadTree on the same
  // actual node more than one time.
  #actualTreeLoaded = new Set()
  #actualTreePromise

// cache of nodes when loading the actualTree, so that we avoid loaded the
  // same node multiple times when symlinks attack.
  #cache = new Map()
  #filter

// cache of link targets for setting fsParent links
  // We don't do fsParent as a magic getter/setter, because it'd be too costly
  // to keep up to date along the walk.
  // And, we know that it can ONLY be relevant when the node is a target of a
  // link, otherwise it'd be in a node_modules folder, so take advantage of
  // that to limit the scans later.
  #topNodes = new Set()
  #transplantFilter

constructor (options) {
    super(options)

// the tree of nodes on disk
    this.actualTree = options.actualTree

// caches for cached realpath calls
    const cwd = process.cwd()
    // assume that the cwd is real enough for our purposes
    this[_rpcache] = new Map([[cwd, cwd]])
    this[_stcache] = new Map()
  }

// public method
  // TODO remove options param in next semver major
  async loadActual (options = {}) {
    // In the past this.actualTree was set as a promise that eventually
    // resolved, and overwrite this.actualTree with the resolved value.  This
    // was a problem because virtually no other code expects this.actualTree to
    // be a promise.  Instead we only set it once resolved, and also return it
    // from the promise so that it is what's returned from this function when
    // awaited.
    if (this.actualTree) {
      return this.actualTree
    }
    if (!this.#actualTreePromise) {
      // allow the user to set options on the ctor as well.
      // XXX: deprecate separate method options objects.
      options = { ...this.options, ...options }

this.#actualTreePromise = this.#loadActual(options)
        .then(tree => {
          // reset all deps to extraneous prior to recalc
          if (!options.root) {
            for (const node of tree.inventory.values()) {
              node.extraneous = true
            }
          }

// only reset root flags if we're not re-rooting,
          // otherwise leave as-is
          calcDepFlags(tree, !options.root)
          this.actualTree = treeCheck(tree)
          return this.actualTree
        })
    }
    return this.#actualTreePromise
  }

// return the promise so that we don't ever have more than one going at the
  // same time.  This is so that buildIdealTree can default to the actualTree
  // if no shrinkwrap present, but reify() can still call buildIdealTree and
  // loadActual in parallel safely.

async #loadActual (options) {
    // mostly realpath to throw if the root doesn't exist
    const {
      global,
      filter = () => true,
      root = null,
      transplantFilter = () => true,
      ignoreMissing = false,
      forceActual = false,
    } = options
    this.#filter = filter
    this.#transplantFilter = transplantFilter

if (global) {
      const real = await realpath(this.path, this[_rpcache], this[_stcache])
      const params = {
        path: this.path,
        realpath: real,
        pkg: {},
        global,
        loadOverrides: true,
      }
      if (this.path === real) {
        this.#actualTree = this.#newNode(params)
      } else {
        this.#actualTree = await this.#newLink(params)
      }
    } else {
      // not in global mode, hidden lockfile is allowed, load root pkg too
      this.#actualTree = await this.#loadFSNode({
        path: this.path,
        real: await realpath(this.path, this[_rpcache], this[_stcache]),
        loadOverrides: true,
      })

this.#actualTree.assertRootOverrides()

// if forceActual is set, don't even try the hidden lockfile
      if (!forceActual) {
        // Note: hidden lockfile will be rejected if it's not the latest thing
        // in the folder, or if any of the entries in the hidden lockfile are
        // missing.
        const meta = await Shrinkwrap.load({
          path: this.#actualTree.path,
          hiddenLockfile: true,
          resolveOptions: this.options,
        })

if (meta.loadedFromDisk) {
          this.#actualTree.meta = meta
          // have to load on a new Arborist object, so we don't assign
          // the virtualTree on this one!  Also, the weird reference is because
          // we can't easily get a ref to Arborist in this module, without
          // creating a circular reference, since this class is a mixin used
          // to build up the Arborist class itself.
          await new this.constructor({ ...this.options }).loadVirtual({
            root: this.#actualTree,
          })
          await this[_setWorkspaces](this.#actualTree)

this.#transplant(root)
          return this.#actualTree
        }
      }

const meta = await Shrinkwrap.load({
        path: this.#actualTree.path,
        lockfileVersion: this.options.lockfileVersion,
        resolveOptions: this.options,
      })
      this.#actualTree.meta = meta
    }

await this.#loadFSTree(this.#actualTree)
    await this[_setWorkspaces](this.#actualTree)

// if there are workspace targets without Link nodes created, load
    // the targets, so that we know what they are.
    if (this.#actualTree.workspaces && this.#actualTree.workspaces.size) {
      const promises = []
      for (const path of this.#actualTree.workspaces.values()) {
        if (!this.#cache.has(path)) {
          // workspace overrides use the root overrides
          const p = this.#loadFSNode({ path, root: this.#actualTree, useRootOverrides: true })
            .then(node => this.#loadFSTree(node))
          promises.push(p)
        }
      }
      await Promise.all(promises)
    }

if (!ignoreMissing) {
      await this.#findMissingEdges()
    }

// try to find a node that is the parent in a fs tree sense, but not a
    // node_modules tree sense, of any link targets.  this allows us to
    // resolve deps that node will find, but a legacy npm view of the
    // world would not have noticed.
    for (const path of this.#topNodes) {
      const node = this.#cache.get(path)
      if (node && !node.parent && !node.fsParent) {
        for (const p of walkUp(dirname(path))) {
          if (this.#cache.has(p)) {
            node.fsParent = this.#cache.get(p)
            break
          }
        }
      }
    }

this.#transplant(root)

if (global) {
      // need to depend on the children, or else all of them
      // will end up being flagged as extraneous, since the
      // global root isn't a "real" project
      const tree = this.#actualTree
      const actualRoot = tree.isLink ? tree.target : tree
      const { dependencies = {} } = actualRoot.package
      for (const [name, kid] of actualRoot.children.entries()) {
        const def = kid.isLink ? `file:${kid.realpath.replace(/#/g, '%23')}` : '*'
        dependencies[name] = dependencies[name] || def
      }
      actualRoot.package = { ...actualRoot.package, dependencies }
    }
    return this.#actualTree
  }

#transplant (root) {
    if (!root || root === this.#actualTree) {
      return
    }

this.#actualTree[_changePath](root.path)
    for (const node of this.#actualTree.children.values()) {
      if (!this.#transplantFilter(node)) {
        node.root = null
      }
    }

root.replace(this.#actualTree)
    for (const node of this.#actualTree.fsChildren) {
      node.root = this.#transplantFilter(node) ? root : null
    }

this.#actualTree = root
  }

async #loadFSNode ({ path, parent, real, root, loadOverrides, useRootOverrides }) {
    if (!real) {
      try {
        real = await realpath(path, this[_rpcache], this[_stcache])
      } catch (error) {
        // if realpath fails, just provide a dummy error node
        return new Node({
          error,
          path,
          realpath: path,
          parent,
          root,
          loadOverrides,
        })
      }
    }

const cached = this.#cache.get(path)
    let node
    // missing edges get a dummy node, assign the parent and return it
    if (cached && !cached.dummy) {
      cached.parent = parent
      return cached
    } else {
      const params = {
        installLinks: this.installLinks,
        legacyPeerDeps: this.legacyPeerDeps,
        path,
        realpath: real,
        parent,
        root,
        loadOverrides,
      }

try {
        const pkg = await rpj(join(real, 'package.json'))
        params.pkg = pkg
        if (useRootOverrides && root.overrides) {
          params.overrides = root.overrides.getNodeRule({ name: pkg.name, version: pkg.version })
        }
      } catch (err) {
        params.error = err
      }

// soldier on if read-package-json raises an error, passing it to the
      // Node which will attach it to its errors array (Link passes it along to
      // its target node)
      if (normalize(path) === real) {
        node = this.#newNode(params)
      } else {
        node = await this.#newLink(params)
      }
    }
    this.#cache.set(path, node)
    return node
  }

#newNode (options) {
    // check it for an fsParent if it's a tree top.  there's a decent chance
    // it'll get parented later, making the fsParent scan a no-op, but better
    // safe than sorry, since it's cheap.
    const { parent, realpath } = options
    if (!parent) {
      this.#topNodes.add(realpath)
    }
    return new Node(options)
  }

async #newLink (options) {
    const { realpath } = options
    this.#topNodes.add(realpath)
    const target = this.#cache.get(realpath)
    const link = new Link({ ...options, target })

if (!target) {
      // Link set its target itself in this case
      this.#cache.set(realpath, link.target)
      // if a link target points at a node outside of the root tree's
      // node_modules hierarchy, then load that node as well.
      await this.#loadFSTree(link.target)
    }

return link
  }

async #loadFSTree (node) {
    const did = this.#actualTreeLoaded
    if (!node.isLink && !did.has(node.target.realpath)) {
      did.add(node.target.realpath)
      await this.#loadFSChildren(node.target)
      return Promise.all(
        [...node.target.children.entries()]
          .filter(([, kid]) => !did.has(kid.realpath))
          .map(([, kid]) => this.#loadFSTree(kid))
      )
    }
  }

// create child nodes for all the entries in node_modules
  // and attach them to the node as a parent
  async #loadFSChildren (node) {
    const nm = resolve(node.realpath, 'node_modules')
    try {
      const kids = await readdirScoped(nm).then(paths => paths.map(p => p.replace(/\\/g, '/')))
      return Promise.all(
        // ignore . dirs and retired scoped package folders
        kids.filter(kid => !/^(@[^/]+\/)?\./.test(kid))
          .filter(kid => this.#filter(node, kid))
          .map(kid => this.#loadFSNode({
            parent: node,
            path: resolve(nm, kid),
          })))
    } catch {
      // error in the readdir is not fatal, just means no kids
    }
  }

async #findMissingEdges () {
    // try to resolve any missing edges by walking up the directory tree,
    // checking for the package in each node_modules folder.  stop at the
    // root directory.
    // The tricky move here is that we load a "dummy" node for the folder
    // containing the node_modules folder, so that it can be assigned as
    // the fsParent.  It's a bad idea to *actually* load that full node,
    // because people sometimes develop in ~/projects/node_modules/...
    // so we'd end up loading a massive tree with lots of unrelated junk.
    const nmContents = new Map()
    const tree = this.#actualTree
    for (const node of tree.inventory.values()) {
      const ancestor = ancestorPath(node.realpath, this.path)

const depPromises = []
      for (const [name, edge] of node.edgesOut.entries()) {
        const notMissing = !edge.missing &&
          !(edge.to && (edge.to.dummy || edge.to.parent !== node))
        if (notMissing) {
          continue
        }

// start the walk from the dirname, because we would have found
        // the dep in the loadFSTree step already if it was local.
        for (const p of walkUp(dirname(node.realpath))) {
          // only walk as far as the nearest ancestor
          // this keeps us from going into completely unrelated
          // places when a project is just missing something, but
          // allows for finding the transitive deps of link targets.
          // ie, if it has to go up and back out to get to the path
          // from the nearest common ancestor, we've gone too far.
          if (ancestor && /^\.\.(?:[\\/]|$)/.test(relative(ancestor, p))) {
            break
          }

let entries
          if (!nmContents.has(p)) {
            entries = await readdirScoped(p + '/node_modules')
              .catch(() => []).then(paths => paths.map(p => p.replace(/\\/g, '/')))
            nmContents.set(p, entries)
          } else {
            entries = nmContents.get(p)
          }

if (!entries.includes(name)) {
            continue
          }

let d
          if (!this.#cache.has(p)) {
            d = new Node({ path: p, root: node.root, dummy: true })
            this.#cache.set(p, d)
          } else {
            d = this.#cache.get(p)
          }
          if (d.dummy) {
            // it's a placeholder, so likely would not have loaded this dep,
            // unless another dep in the tree also needs it.
            const depPath = normalize(`${p}/node_modules/${name}`)
            const cached = this.#cache.get(depPath)
            if (!cached || cached.dummy) {
              depPromises.push(this.#loadFSNode({
                path: depPath,
                root: node.root,
                parent: d,
              }).then(node => this.#loadFSTree(node)))
            }
          }
          break
        }
      }
      await Promise.all(depPromises)
    }
  }
}

Back to File Manager