wip: better dependency import mgmt

.yarn/sdks/typescript/package.json

@@ -1,6 +1,6 @@
 {
  "name": "typescript",
- "version": "3.9.9-pnpify",
+ "version": "4.2.3-pnpify",
  "main": "./lib/typescript.js",
  "type": "commonjs"
 }

jest.config.js

@@ -1,5 +1,5 @@
 module.exports = {
- roots: ['<rootDir>/tests'],
+ roots: ['<rootDir>/src', '<rootDir>/tests'],
  testMatch: [
  '**/__tests__/**/*.+(ts|tsx|js)',
  '**/?(*.)+(spec|test).+(ts|tsx|js)',

package.json

@@ -23,18 +23,19 @@
  "lint:fix": "run lint --fix"
  },
  "devDependencies": {
- "@babel/cli": "^7.13.0",
- "@babel/core": "^7.13.1",
- "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.0",
- "@babel/plugin-proposal-optional-chaining": "^7.13.0",
- "@babel/plugin-transform-runtime": "^7.13.7",
- "@babel/preset-env": "^7.13.5",
+ "@babel/cli": "^7.13.10",
+ "@babel/core": "^7.13.10",
+ "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8",
+ "@babel/plugin-proposal-optional-chaining": "^7.13.8",
+ "@babel/plugin-transform-runtime": "^7.13.10",
+ "@babel/preset-env": "^7.13.10",
  "@babel/preset-typescript": "^7.13.0",
  "@tophat/eslint-config": "^0.3.1",
- "@types/babel__traverse": "^7.11.0",
- "@types/jest": "^24.0.23",
+ "@types/babel__parser": "^7.1.1",
+ "@types/babel__traverse": "^7.11.1",
+ "@types/jest": "^26.0.20",
  "@types/minimatch": "^3.0.3",
- "@types/node": "^12.12.7",
+ "@types/node": "^14.0.0",
  "@types/yargs": "^16.0.0",
  "@typescript-eslint/eslint-plugin": "^2.7.0",
  "@typescript-eslint/parser": "^2.7.0",
@@ -48,23 +49,24 @@
  "eslint-plugin-jest": "^23.0.4",
  "eslint-plugin-prettier": "^3.1.1",
  "husky": "^5.1.1",
- "jest": "^24.9.0",
+ "jest": "^26.6.3",
  "lint-staged": "^10.5.4",
  "mock-fs": "^4.10.4",
  "packwatch": "^1.0.0",
  "prettier": "^2.2.1",
- "typescript": "^3.7.2"
+ "typescript": "^4.2.3"
  },
  "dependencies": {
- "@babel/parser": "^7.13.4",
+ "@babel/parser": "^7.13.12",
  "@babel/traverse": "^7.13.0",
+ "@babel/types": "^7.13.0",
  "@yarnpkg/cli": "^2.4.0",
  "@yarnpkg/core": "^2.4.0",
  "@yarnpkg/fslib": "^2.4.0",
  "@yarnpkg/plugin-npm": "^2.4.0",
  "@yarnpkg/plugin-pack": "^2.2.3",
  "chalk": "^4.1.0",
- "minimatch": "^3.0.4",
+ "fast-glob": "^3.2.5",
  "yargs": "^16.2.0"
  },
  "lint-staged": {

src/cli.ts

@@ -10,24 +10,39 @@ const argv = yargs(process.argv.slice(2))
  type: 'string',
  description: 'Project root',
  })
- .option('include', {
+ .option('includeFiles', {
  type: 'array',
  description: 'Paths to include in the analysis',
  })
- .option('exclude', {
+ .option('excludeFiles', {
  type: 'array',
  description: 'Paths to exclude from the analysis',
  })
+ .option('excludePackages', {
+ type: 'array',
+ description: 'Package names to exclude from the analysis',
+ })
+ .option('devFiles', {
+ type: 'array',
+ description: 'Paths to mark as dev files',
+ })
+ .option('skipRoot', {
+ type: 'boolean',
+ description: 'Whether to skip over root workspace',
+ })
  .option('fix', {
  type: 'boolean',
  description: 'Attempt to fix package.json based on analysis',
  }).argv
 
 validateDependencies({
  cwd: argv.cwd,
- include: argv.include as string[],
- exclude: argv.exclude as string[],
+ includeFiles: argv.includeFies as string[],
+ excludeFiles: argv.excludeFiles as string[],
+ devFiles: argv.devFiles as string[],
+ excludePackages: argv.excludePackages as string[],
  fix: argv.fix,
+ skipRoot: argv.skipRoot,
 }).catch((e) => {
  console.log(e)
  process.exit(1)

src/diffDependenciesAndImportsByWorkspace.ts

@@ -1,12 +1,21 @@
-import { structUtils } from '@yarnpkg/core'
+import { Workspace, structUtils } from '@yarnpkg/core'
+import fastGlob from 'fast-glob'
 
-import { Context, DiffReport, PackagesByWorkspaceMap } from './types'
+import {
+ AnalysisConfiguration,
+ Context,
+ DependenciesMap,
+ DiffReport,
+ ImportRecord,
+ ImportRecordsByWorkspaceMap,
+} from './types'
 
-export default function diffDependenciesAndImportsByWorkspace(
+export default async function diffDependenciesAndImportsByWorkspace(
  context: Context,
- dependenciesMap: PackagesByWorkspaceMap,
- importsMap: PackagesByWorkspaceMap,
-): DiffReport {
+ configuration: AnalysisConfiguration,
+ dependenciesMap: Map<Workspace, DependenciesMap>,
+ importsMap: ImportRecordsByWorkspaceMap,
+): Promise<DiffReport> {
  const { workspaces } = context.project
  const undeclaredDependenciesMap = new Map()
  const unusedDependenciesMap = new Map()
@@ -16,15 +25,21 @@ export default function diffDependenciesAndImportsByWorkspace(
  const workspaceIdent = structUtils.stringifyIdent(
  workspace.manifest.name,
  )
- const workspaceDependencies =
- dependenciesMap.get(workspace) ?? new Set()
- const workspaceImports = importsMap.get(workspace) ?? new Set()
 
- const undeclaredDependencies = getUndeclaredDependencies(
+ const workspaceDependencies = dependenciesMap.get(workspace)
+ const workspaceImports = importsMap.get(workspace)
+
+ if (!workspaceDependencies || !workspaceImports) continue
+
+ const undeclaredDependencies = await getUndeclaredDependencies(
+ configuration,
+ workspace,
  workspaceDependencies,
  workspaceImports,
  )
- const unusedDependencies = getUnusedDependencies(
+ const unusedDependencies = await getUnusedDependencies(
+ configuration,
+ workspace,
  workspaceDependencies,
  workspaceImports,
  )
@@ -39,28 +54,115 @@ export default function diffDependenciesAndImportsByWorkspace(
  }
 }
 
-function getUndeclaredDependencies(
- dependencies: Set<string>,
- imports: Set<string>,
-): Set<string> {
+async function getUndeclaredDependencies(
+ configuration: AnalysisConfiguration,
+ workspace: Workspace,
+ dependenciesMap: DependenciesMap,
+ imports: Set<ImportRecord>,
+): Promise<Set<string>> {
  const undeclaredDependencies: Set<string> = new Set()
+ const devFiles = await fastGlob(configuration.devFiles, {
+ cwd: workspace.cwd,
+ })
+ for (const { imported, importedFrom } of imports) {
+ const importedIdent = structUtils.parseIdent(imported)
+ const identHash = importedIdent.identHash
+
+ // Undeclared if not in dep or peerdep
+ // Allowed to be in dev if file matches dev glob.
+ // TODO: Add config option for dev allowed glob
 
- for (const importedPackage of imports) {
- if (!dependencies.has(importedPackage))
- undeclaredDependencies.add(importedPackage)
+ if (dependenciesMap.dependencies.has(identHash)) continue
+
+ if (
+ dependenciesMap.devDependencies.has(identHash) &&
+ devFiles.includes(importedFrom)
+ )
+ continue
+
+ if (
+ dependenciesMap.peerDependencies.has(identHash) &&
+ !devFiles.includes(importedFrom)
+ )
+ continue
+
+ undeclaredDependencies.add(imported)
  }
 
  return undeclaredDependencies
 }
 
-function getUnusedDependencies(
- dependencies: Set<string>,
- imports: Set<string>,
-): Set<string> {
+// TODO
+//function getMovableDependencies(): void {}
+
+async function getUnusedDependencies(
+ configuration: AnalysisConfiguration,
+ workspace: Workspace,
+ dependenciesMap: DependenciesMap,
+ imports: Set<ImportRecord>,
+): Promise<Set<string>> {
  const unusedDependencies: Set<string> = new Set()
 
- for (const dependency of dependencies) {
- if (!imports.has(dependency)) unusedDependencies.add(dependency)
+ const importsUsage = new Map<string, Set<string>>()
+ const devImportsUsage = new Map<string, Set<string>>()
+ const devFiles = await fastGlob(configuration.devFiles, {
+ cwd: workspace.cwd,
+ })
+ for (const { imported, importedFrom } of imports.values()) {
+ if (devFiles.includes(importedFrom)) {
+ const devSet = devImportsUsage.get(imported) ?? new Set<string>()
+ devImportsUsage.set(imported, devSet)
+ devSet.add(importedFrom)
+ } else {
+ const depSet = importsUsage.get(imported) ?? new Set<string>()
+ importsUsage.set(imported, depSet)
+ depSet.add(importedFrom)
+ }
  }
+
+ for (const dependencyDescriptor of dependenciesMap.dependencies.values()) {
+ const dependencyName = structUtils.stringifyIdent(dependencyDescriptor)
+
+ // Not considering ranges.
+ if (
+ dependenciesMap.transitivePeerDependencies.has(
+ dependencyDescriptor.identHash,
+ )
+ ) {
+ continue
+ }
+
+ if (
+ importsUsage.has(dependencyName) ||
+ dependenciesMap.binaries.has(dependencyDescriptor.identHash) ||
+ configuration.excludePackages(dependencyName)
+ ) {
+ continue
+ }
+ unusedDependencies.add(dependencyName)
+ }
+ for (const dependencyDescriptor of dependenciesMap.devDependencies.values()) {
+ const dependencyName = structUtils.stringifyIdent(dependencyDescriptor)
+
+ // should we do this for dev as well outside of root?
+ if (
+ dependenciesMap.transitivePeerDependencies.has(
+ dependencyDescriptor.identHash,
+ )
+ ) {
+ continue
+ }
+
+ // Unused if devSet[dependency] is empty
+ if (
+ devImportsUsage.has(dependencyName) ||
+ dependenciesMap.binaries.has(dependencyDescriptor.identHash) ||
+ configuration.excludePackages(dependencyName)
+ ) {
+ continue
+ }
+ unusedDependencies.add(dependencyName)
+ }
+
  return unusedDependencies
 }

src/fixWorkspaces.ts

@@ -2,7 +2,7 @@ import path from 'path'
 import { promises as fs } from 'fs'
 
 import { IdentHash, Manifest, Workspace, structUtils } from '@yarnpkg/core'
-import { PortablePath } from '@yarnpkg/fslib'
+import { PortablePath, npath } from '@yarnpkg/fslib'
 import { npmHttpUtils } from '@yarnpkg/plugin-npm'
 import chalk from 'chalk'
 
@@ -12,7 +12,9 @@ export default async function fixWorkspaces(
  context: Context,
  diffReport: DiffReport,
 ): Promise<void> {
- const workspaces = context.project.workspaces
+ const workspaces = [...context.workspaceCwds].map((cwd) =>
+ context.project.getWorkspaceByCwd(npath.toPortablePath(cwd)),
+ )
  const resolvedVersionsFromNodeModules = await maybeResolvePackageVersionsFromNodeModules(
  context,
  )