From c858f49c7f6a7ee5321765289a99060bf0d0f020 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Mon, 16 Dec 2024 16:45:31 +0100 Subject: [PATCH] Sustainable Kibana Architecture: Relocate script v4 (#204383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary * Auto-detect "upstream" and "origin" remotes (instead of assuming their names). * Allow relocating modules that are already in a "sustainable" folder. * Filter out modules that are in the correct locations. * Update the list of _modules to relocate_ to show only those modules that are actually moved. --------- Co-authored-by: Alejandro Fernández Haro --- packages/kbn-relocate/README.md | 5 +- packages/kbn-relocate/constants.ts | 27 ++++++--- packages/kbn-relocate/index.ts | 20 ++++--- packages/kbn-relocate/relocate.ts | 56 ++++++++++--------- .../{utils.exec.ts => utils/exec.ts} | 0 .../{utils.git.ts => utils/git.ts} | 22 +++++++- .../{utils.logging.ts => utils/logging.ts} | 6 +- .../{utils.relocate.ts => utils/relocate.ts} | 41 +++++++++++--- .../kbn-relocate/{ => utils}/transforms.ts | 2 +- 9 files changed, 120 insertions(+), 59 deletions(-) rename packages/kbn-relocate/{utils.exec.ts => utils/exec.ts} (100%) rename packages/kbn-relocate/{utils.git.ts => utils/git.ts} (81%) rename packages/kbn-relocate/{utils.logging.ts => utils/logging.ts} (96%) rename packages/kbn-relocate/{utils.relocate.ts => utils/relocate.ts} (84%) rename packages/kbn-relocate/{ => utils}/transforms.ts (97%) diff --git a/packages/kbn-relocate/README.md b/packages/kbn-relocate/README.md index 899a5e9b97970..06a04067a26a2 100644 --- a/packages/kbn-relocate/README.md +++ b/packages/kbn-relocate/README.md @@ -8,12 +8,9 @@ You must have `gh` CLI tool installed. You can install it by running: ```sh brew install gh +gh auth login ``` -You must have `elastic/kibana` remote configured under the name `upstream`. - -You must have a remote named `origin` pointing to your fork of the Kibana repo. - ## Usage First of all, you need to decide whether you want to contribute to an existing PR or to create a new one. Use the `--pr` flag to specify the PR you are trying to update: diff --git a/packages/kbn-relocate/constants.ts b/packages/kbn-relocate/constants.ts index 059c32004370e..5b5f4099b782b 100644 --- a/packages/kbn-relocate/constants.ts +++ b/packages/kbn-relocate/constants.ts @@ -13,13 +13,26 @@ export const BASE_FOLDER = process.cwd() + '/'; export const BASE_FOLDER_DEPTH = process.cwd().split('/').length; export const KIBANA_FOLDER = process.cwd().split('/').pop()!; export const EXCLUDED_MODULES = ['@kbn/core']; -export const TARGET_FOLDERS = [ - 'src/platform/plugins/', - 'src/platform/packages/', - 'x-pack/platform/plugins/', - 'x-pack/platform/packages/', - 'x-pack/solutions/', -]; +export const TARGET_FOLDERS: Record = { + 'platform:private': [ + 'src/platform/packages/private/', + 'src/platform/plugins/private/', + 'x-pack/platform/packages/private/', + 'x-pack/platform/plugins/private/', + ], + 'platform:shared': [ + 'src/platform/packages/shared/', + 'src/platform/plugins/shared/', + 'x-pack/platform/packages/shared/', + 'x-pack/platform/plugins/shared/', + ], + 'observability:private': [ + 'x-pack/solutions/observability/packages/', + 'x-pack/solutions/observability/plugins/', + ], + 'search:private': ['x-pack/solutions/search/packages/', 'x-pack/solutions/search/plugins/'], + 'security:private': ['x-pack/solutions/security/packages/', 'x-pack/solutions/security/plugins/'], +}; export const EXTENSIONS = [ 'eslintignore', 'gitignore', diff --git a/packages/kbn-relocate/index.ts b/packages/kbn-relocate/index.ts index cddaa307ab7b6..12f89a29275e6 100644 --- a/packages/kbn-relocate/index.ts +++ b/packages/kbn-relocate/index.ts @@ -46,15 +46,17 @@ export const runKbnRelocateCli = () => { await findAndMoveModule(flags.moveOnly, log); } else { const { pr, team, path, include, exclude, baseBranch } = flags; - await findAndRelocateModules({ - prNumber: toOptString('prNumber', pr), - baseBranch: toOptString('baseBranch', baseBranch, 'main')!, - teams: toStringArray(team), - paths: toStringArray(path), - included: toStringArray(include), - excluded: toStringArray(exclude), - log, - }); + await findAndRelocateModules( + { + prNumber: toOptString('prNumber', pr), + baseBranch: toOptString('baseBranch', baseBranch, 'main')!, + teams: toStringArray(team), + paths: toStringArray(path), + included: toStringArray(include), + excluded: toStringArray(exclude), + }, + log + ); } }, { diff --git a/packages/kbn-relocate/relocate.ts b/packages/kbn-relocate/relocate.ts index dde0c83fc096e..fe2537ddeb040 100644 --- a/packages/kbn-relocate/relocate.ts +++ b/packages/kbn-relocate/relocate.ts @@ -16,22 +16,17 @@ import type { ToolingLog } from '@kbn/tooling-log'; import { getPackages } from '@kbn/repo-packages'; import { REPO_ROOT } from '@kbn/repo-info'; import type { Package } from './types'; -import { - DESCRIPTION, - EXCLUDED_MODULES, - KIBANA_FOLDER, - NEW_BRANCH, - TARGET_FOLDERS, -} from './constants'; +import { DESCRIPTION, EXCLUDED_MODULES, KIBANA_FOLDER, NEW_BRANCH } from './constants'; import { belongsTo, calculateModuleTargetFolder, + isInTargetFolder, replaceReferences, replaceRelativePaths, -} from './utils.relocate'; -import { safeExec } from './utils.exec'; -import { relocatePlan, relocateSummary } from './utils.logging'; -import { checkoutBranch, checkoutResetPr } from './utils.git'; +} from './utils/relocate'; +import { safeExec } from './utils/exec'; +import { relocatePlan, relocateSummary } from './utils/logging'; +import { checkoutBranch, checkoutResetPr, findGithubLogin, findRemoteName } from './utils/git'; const moveModule = async (module: Package, log: ToolingLog) => { const destination = calculateModuleTargetFolder(module); @@ -52,11 +47,6 @@ const relocateModules = async (toMove: Package[], log: ToolingLog): Promise module.directory.includes(folder))) { - log.warning(`The module ${module.id} is already in a "sustainable" folder. Skipping`); - // skip modules that are already moved - continue; - } log.info(''); log.info('--------------------------------------------------------------------------------'); log.info(`\t${module.id} (${i + 1} of ${toMove.length})`); @@ -93,10 +83,9 @@ export interface RelocateModulesParams { paths: string[]; included: string[]; excluded: string[]; - log: ToolingLog; } -const findModules = ({ teams, paths, included, excluded }: FindModulesParams) => { +const findModules = ({ teams, paths, included, excluded }: FindModulesParams, log: ToolingLog) => { // get all modules const modules = getPackages(REPO_ROOT); @@ -123,13 +112,14 @@ const findModules = ({ teams, paths, included, excluded }: FindModulesParams) => paths.some((path) => module.directory.includes(path)) ) // the module is not explicitly excluded - .filter(({ id }) => !excluded.includes(id)), - 'id' + .filter(({ id }) => !excluded.includes(id)) + // exclude modules that are in the correct folder + .filter((module) => !isInTargetFolder(module, log)) ); }; export const findAndMoveModule = async (moduleId: string, log: ToolingLog) => { - const modules = findModules({ teams: [], paths: [], included: [moduleId], excluded: [] }); + const modules = findModules({ teams: [], paths: [], included: [moduleId], excluded: [] }, log); if (!modules.length) { log.warning(`Cannot move ${moduleId}, either not found or not allowed!`); } else { @@ -137,10 +127,24 @@ export const findAndMoveModule = async (moduleId: string, log: ToolingLog) => { } }; -export const findAndRelocateModules = async (params: RelocateModulesParams) => { - const { prNumber, log, baseBranch, ...findParams } = params; +export const findAndRelocateModules = async (params: RelocateModulesParams, log: ToolingLog) => { + const upstream = await findRemoteName('elastic/kibana'); + if (!upstream) { + log.error( + 'This repository does not have a remote pointing to the elastic/kibana repository. Aborting' + ); + return; + } + + const origin = await findRemoteName(`${await findGithubLogin()}/kibana`); + if (!origin) { + log.error('This repository does not have a remote pointing to your Kibana fork. Aborting'); + return; + } + + const { prNumber, baseBranch, ...findParams } = params; - const toMove = findModules(findParams); + const toMove = findModules(findParams, log); if (!toMove.length) { log.info( `No packages match the specified filters. Please tune your '--path' and/or '--team' and/or '--include' flags` @@ -164,7 +168,7 @@ export const findAndRelocateModules = async (params: RelocateModulesParams) => { await safeExec(`git restore --staged .`); await safeExec(`git restore .`); await safeExec(`git clean -f -d`); - await safeExec(`git checkout ${baseBranch} && git pull upstream ${baseBranch}`); + await safeExec(`git checkout ${baseBranch} && git pull ${upstream} ${baseBranch}`); if (prNumber) { // checkout existing PR, reset all commits, rebase from baseBranch @@ -204,7 +208,7 @@ export const findAndRelocateModules = async (params: RelocateModulesParams) => { const pushCmd = prNumber ? `git push --force-with-lease` - : `git push --set-upstream origin ${NEW_BRANCH}`; + : `git push --set-upstream ${origin} ${NEW_BRANCH}`; if (!res2.pushBranch) { log.info(`Remember to push changes with "${pushCmd}"`); diff --git a/packages/kbn-relocate/utils.exec.ts b/packages/kbn-relocate/utils/exec.ts similarity index 100% rename from packages/kbn-relocate/utils.exec.ts rename to packages/kbn-relocate/utils/exec.ts diff --git a/packages/kbn-relocate/utils.git.ts b/packages/kbn-relocate/utils/git.ts similarity index 81% rename from packages/kbn-relocate/utils.git.ts rename to packages/kbn-relocate/utils/git.ts index 4f002772528fd..f2e529bee6d0f 100644 --- a/packages/kbn-relocate/utils.git.ts +++ b/packages/kbn-relocate/utils/git.ts @@ -8,8 +8,26 @@ */ import inquirer from 'inquirer'; -import type { Commit, PullRequest } from './types'; -import { safeExec } from './utils.exec'; +import type { Commit, PullRequest } from '../types'; +import { safeExec } from './exec'; + +export const findRemoteName = async (repo: string) => { + const res = await safeExec('git remote -v'); + const remotes = res.stdout.split('\n').map((line) => line.split(/\t| /).filter(Boolean)); + return remotes.find(([_, url]) => url.includes(`github.com/${repo}`))?.[0]; +}; + +export const findGithubLogin = async () => { + const res = await safeExec('gh auth status'); + // e.g. ✓ Logged in to github.com account gsoldevila (/Users/gsoldevila/.config/gh/hosts.yml) + const loginLine = res.stdout + .split('\n') + .find((line) => line.includes('Logged in')) + ?.split(/\t| /) + .filter(Boolean); + + return loginLine?.[loginLine?.findIndex((fragment) => fragment === 'account') + 1]; +}; export const findPr = async (number: string): Promise => { const res = await safeExec(`gh pr view ${number} --json commits,headRefName`); diff --git a/packages/kbn-relocate/utils.logging.ts b/packages/kbn-relocate/utils/logging.ts similarity index 96% rename from packages/kbn-relocate/utils.logging.ts rename to packages/kbn-relocate/utils/logging.ts index 5b290292dd75e..742610dfe1de6 100644 --- a/packages/kbn-relocate/utils.logging.ts +++ b/packages/kbn-relocate/utils/logging.ts @@ -10,8 +10,8 @@ import type { ToolingLog } from '@kbn/tooling-log'; import { appendFileSync, writeFileSync } from 'fs'; import dedent from 'dedent'; -import type { Package } from './types'; -import { calculateModuleTargetFolder } from './utils.relocate'; +import type { Package } from '../types'; +import { calculateModuleTargetFolder } from './relocate'; import { BASE_FOLDER, DESCRIPTION, @@ -19,7 +19,7 @@ import { SCRIPT_ERRORS, UPDATED_REFERENCES, UPDATED_RELATIVE_PATHS, -} from './constants'; +} from '../constants'; export const relocatePlan = (modules: Package[], log: ToolingLog) => { const plugins = modules.filter((module) => module.manifest.type === 'plugin'); diff --git a/packages/kbn-relocate/utils.relocate.ts b/packages/kbn-relocate/utils/relocate.ts similarity index 84% rename from packages/kbn-relocate/utils.relocate.ts rename to packages/kbn-relocate/utils/relocate.ts index 15121fefd344a..2f2b6a78379e6 100644 --- a/packages/kbn-relocate/utils.relocate.ts +++ b/packages/kbn-relocate/utils/relocate.ts @@ -10,7 +10,7 @@ import { join } from 'path'; import type { ToolingLog } from '@kbn/tooling-log'; import { orderBy } from 'lodash'; -import type { Package } from './types'; +import type { Package } from '../types'; import { applyTransforms } from './transforms'; import { BASE_FOLDER, @@ -22,8 +22,8 @@ import { TARGET_FOLDERS, UPDATED_REFERENCES, UPDATED_RELATIVE_PATHS, -} from './constants'; -import { quietExec, safeExec } from './utils.exec'; +} from '../constants'; +import { quietExec, safeExec } from './exec'; export const belongsTo = (module: Package, owner: string): boolean => { return Array.from(module.manifest.owner)[0] === owner; @@ -40,11 +40,18 @@ export const calculateModuleTargetFolder = (module: Package): string => { const isPlugin = module.manifest.type === 'plugin'; const fullPath = join(BASE_FOLDER, module.directory); let moduleDelimiter = isPlugin ? '/plugins/' : '/packages/'; - if (TARGET_FOLDERS.some((folder) => module.directory.includes(folder)) && group === 'platform') { - // if a platform module has already been relocated, strip the /private/ or /shared/ part too - moduleDelimiter += `${module.visibility}/`; + + // for platform modules that are in a sustainable folder, strip the /private/ or /shared/ part too + if (module.directory.includes(`${moduleDelimiter}private/`)) { + moduleDelimiter += 'private/'; + } else if (module.directory.includes(`${moduleDelimiter}shared/`)) { + moduleDelimiter += 'shared/'; } - const moduleFolder = fullPath.split(moduleDelimiter).pop()!; + + const chunks = fullPath.split(moduleDelimiter); + chunks.shift(); // remove the base path up to '/packages/' or '/plugins/' + const moduleFolder = chunks.join(moduleDelimiter); // in case there's an extra /packages/ or /plugins/ folder + let path: string; if (group === 'platform') { @@ -79,6 +86,26 @@ export const calculateModuleTargetFolder = (module: Package): string => { return applyTransforms(module, path); }; +export const isInTargetFolder = (module: Package, log: ToolingLog): boolean => { + if (!module.group || !module.visibility) { + log.warning(`The module '${module.id}' is missing the group/visibility information`); + return true; + } + + const baseTargetFolders = TARGET_FOLDERS[`${module.group}:${module.visibility}`]; + const baseTargetFolder = baseTargetFolders.find((candidate) => { + return module.directory.includes(candidate); + }); + if (baseTargetFolder) { + log.info( + `The module ${module.id} is already in the correct folder: '${baseTargetFolder}'. Skipping` + ); + return true; + } + + return false; +}; + export const replaceReferences = async (module: Package, destination: string, log: ToolingLog) => { const dir = module.directory; const source = diff --git a/packages/kbn-relocate/transforms.ts b/packages/kbn-relocate/utils/transforms.ts similarity index 97% rename from packages/kbn-relocate/transforms.ts rename to packages/kbn-relocate/utils/transforms.ts index 72f57a24daa00..ed584abeb55ab 100644 --- a/packages/kbn-relocate/transforms.ts +++ b/packages/kbn-relocate/utils/transforms.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import type { Package } from './types'; +import type { Package } from '../types'; type TransformFunction = (param: string) => string; const TRANSFORMS: Record = {