From 4ec203601e4e59fc2313bf7622e1b21b81850676 Mon Sep 17 00:00:00 2001 From: ANGkeith Date: Thu, 3 Oct 2024 10:02:39 +0800 Subject: [PATCH] feat: add support for absolute path for options that makes sense --- src/argv.ts | 38 +++++++--- src/commander.ts | 10 +-- src/handler.ts | 37 +++------- src/index.ts | 10 +-- src/job.ts | 36 +++++----- src/parser-includes.ts | 26 +++---- src/parser.ts | 6 +- src/state.ts | 8 +-- src/utils.ts | 4 +- src/variables-from-files.ts | 3 +- tests/cases.test.ts | 4 +- ...ration.artifacts-after-afterscript.test.ts | 8 +-- .../somefolder/file-var | 0 .../integration.custom-home.test.ts | 15 ++-- .../custom-state-dir/.gitlab-ci.yml | 4 ++ .../integration.custom-state-dir.test.ts | 70 +++++++++++++++++++ .../integration.variable-order.test.ts | 2 +- 17 files changed, 179 insertions(+), 102 deletions(-) rename tests/test-cases/custom-home/.home/{ => .gitlab-ci-local}/somefolder/file-var (100%) create mode 100644 tests/test-cases/custom-state-dir/.gitlab-ci.yml create mode 100644 tests/test-cases/custom-state-dir/integration.custom-state-dir.test.ts diff --git a/src/argv.ts b/src/argv.ts index 931c79423..a29be2a67 100644 --- a/src/argv.ts +++ b/src/argv.ts @@ -21,6 +21,14 @@ async function gitRootPath () { return stdout; } +const generateGitIgnore = (stateDir: string) => { + const gitIgnoreFilePath = path.join(stateDir, ".gitignore"); + const gitIgnoreContent = "*\n!.gitignore\n"; + if (!fs.existsSync(gitIgnoreFilePath)) { + fs.outputFileSync(gitIgnoreFilePath, gitIgnoreContent); + } +}; + export class Argv { private map: Map = new Map(); @@ -39,7 +47,7 @@ export class Argv { const argv = new Argv(args, writeStreams); await argv.fallbackCwd(args); - argv.injectDotenv(`${argv.home}/.gitlab-ci-local/.env`, args); + argv.injectDotenv(`${argv.home}/.env`, args); argv.injectDotenv(`${argv.cwd}/.gitlab-ci-local-env`, args); if (!argv.shellExecutorNoImage && argv.shellIsolation) { @@ -75,23 +83,37 @@ export class Argv { get cwd (): string { let cwd = this.map.get("cwd") ?? "."; assert(typeof cwd != "object", "--cwd option cannot be an array"); - assert(!path.isAbsolute(cwd), "Please use relative path for the --cwd option"); - cwd = path.normalize(`${process.cwd()}/${cwd}`); - cwd = cwd.replace(/\/$/, ""); - assert(fs.pathExistsSync(cwd), `${cwd} is not a directory`); + if (!path.isAbsolute(cwd)) { + cwd = path.resolve(`${process.cwd()}/${cwd}`); + } + assert(fs.pathExistsSync(cwd), `--cwd (${cwd}) is not a directory`); return cwd; } get file (): string { - return this.map.get("file") ?? ".gitlab-ci.yml"; + let file = this.map.get("file") ?? ".gitlab-ci.yml"; + if (!path.isAbsolute(file)) { + file = `${this.cwd}/${file}`; + } + assert(fs.existsSync(`${file}`), `--file (${file}) could not be found`); + return file; } get stateDir (): string { - return (this.map.get("stateDir") ?? ".gitlab-ci-local").replace(/\/$/, ""); + let stateDir = this.map.get("stateDir") ?? ".gitlab-ci-local"; + if (path.isAbsolute(stateDir)) { + // autogenerate uniqueStateDir + return `${stateDir}/${this.cwd.replaceAll("/", ".")}`; + } + stateDir = `${this.cwd}/${stateDir}`; + generateGitIgnore(stateDir); + return stateDir; } get home (): string { - return (this.map.get("home") ?? process.env.HOME ?? "").replace(/\/$/, ""); + const home = (this.map.get("home") ?? `${process.env.HOME}/.gitlab-ci-local}`).replace(/\/$/, ""); + assert(path.isAbsolute(home), `--home (${home}) must be a absolute path`); + return home; } get volume (): string[] { diff --git a/src/commander.ts b/src/commander.ts index 80ce74d97..df2306c4e 100644 --- a/src/commander.ts +++ b/src/commander.ts @@ -19,7 +19,6 @@ export class Commander { potentialStarters = potentialStarters.filter(j => j.when !== "manual" || argv.manual.includes(j.name)); await Executor.runLoop(argv, jobs, stages, potentialStarters); await Commander.printReport({ - cwd: argv.cwd, showTimestamps: argv.showTimestamps, stateDir: argv.stateDir, writeStreams: writeStreams, @@ -39,7 +38,6 @@ export class Commander { potentialStarters = potentialStarters.filter(j => j.stage === argv.stage); await Executor.runLoop(argv, jobs, stages, potentialStarters); await Commander.printReport({ - cwd: argv.cwd, showTimestamps: argv.showTimestamps, stateDir: argv.stateDir, writeStreams: writeStreams, @@ -78,7 +76,6 @@ export class Commander { await Executor.runLoop(argv, Array.from(jobSet), stages, starters); await Commander.printReport({ - cwd: argv.cwd, showTimestamps: argv.showTimestamps, stateDir: argv.stateDir, writeStreams: writeStreams, @@ -88,8 +85,7 @@ export class Commander { }); } - static async printReport ({cwd, stateDir, showTimestamps, writeStreams, jobs, stages, jobNamePad}: { - cwd: string; + static async printReport ({stateDir, showTimestamps, writeStreams, jobs, stages, jobNamePad}: { showTimestamps: boolean; stateDir: string; writeStreams: WriteStreams; @@ -149,7 +145,7 @@ export class Commander { const namePad = name.padEnd(jobNamePad); const safeName = Utils.safeDockerString(name); writeStreams.stdout(chalk`{black.bgYellowBright WARN }${renderDuration(prettyDuration)} {blueBright ${namePad}} pre_script\n`); - const outputLog = await fs.readFile(`${cwd}/${stateDir}/output/${safeName}.log`, "utf8"); + const outputLog = await fs.readFile(`${stateDir}/output/${safeName}.log`, "utf8"); for (const line of outputLog.split(/\r?\n/).filter(j => !j.includes("[32m$ ")).filter(j => j !== "").slice(-3)) { writeStreams.stdout(chalk` {yellow >} ${line}\n`); } @@ -170,7 +166,7 @@ export class Commander { const namePad = name.padEnd(jobNamePad); const safeName = Utils.safeDockerString(name); writeStreams.stdout(chalk`{black.bgRed FAIL }${renderDuration(prettyDuration)} {blueBright ${namePad}}\n`); - const outputLog = await fs.readFile(`${cwd}/${stateDir}/output/${safeName}.log`, "utf8"); + const outputLog = await fs.readFile(`${stateDir}/output/${safeName}.log`, "utf8"); for (const line of outputLog.split(/\r?\n/).filter(j => !j.includes("[32m$ ")).filter(j => j !== "").slice(-3)) { writeStreams.stdout(chalk` {red >} ${line}\n`); } diff --git a/src/handler.ts b/src/handler.ts index a18534b2e..a8ccdd0a3 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -1,6 +1,5 @@ import * as yaml from "js-yaml"; import chalk from "chalk"; -import path from "path"; import * as fs from "fs-extra"; import * as yargs from "yargs"; import {Commander} from "./commander"; @@ -13,19 +12,10 @@ import {Utils} from "./utils"; import {Argv} from "./argv"; import assert from "assert"; -const generateGitIgnore = (cwd: string, stateDir: string) => { - const gitIgnoreFilePath = `${cwd}/${stateDir}/.gitignore`; - const gitIgnoreContent = "*\n!.gitignore\n"; - if (!fs.existsSync(gitIgnoreFilePath)) { - fs.outputFileSync(gitIgnoreFilePath, gitIgnoreContent); - } -}; - export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[] = []) { const argv = await Argv.build(args, writeStreams); const cwd = argv.cwd; const stateDir = argv.stateDir; - const file = argv.file; let parser: Parser | null = null; if (argv.completion) { @@ -33,15 +23,13 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ return []; } - assert(fs.existsSync(`${cwd}/${file}`), `${path.resolve(cwd)}/${file} could not be found`); - if (argv.fetchIncludes) { await Parser.create(argv, writeStreams, 0, jobs); return []; } if (argv.preview) { - const pipelineIid = await state.getPipelineIid(cwd, stateDir); + const pipelineIid = await state.getPipelineIid(stateDir); parser = await Parser.create(argv, writeStreams, pipelineIid, jobs, false); const gitlabData = parser.gitlabData; for (const jobName of Object.keys(gitlabData)) { @@ -55,26 +43,25 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ } writeStreams.stdout(`---\n${yaml.dump(gitlabData, {lineWidth: 160})}`); } else if (argv.list || argv.listAll) { - const pipelineIid = await state.getPipelineIid(cwd, stateDir); + const pipelineIid = await state.getPipelineIid(stateDir); parser = await Parser.create(argv, writeStreams, pipelineIid, jobs); Commander.runList(parser, writeStreams, argv.listAll); } else if (argv.listJson) { - const pipelineIid = await state.getPipelineIid(cwd, stateDir); + const pipelineIid = await state.getPipelineIid(stateDir); parser = await Parser.create(argv, writeStreams, pipelineIid, jobs); Commander.runJson(parser, writeStreams); } else if (argv.listCsv || argv.listCsvAll) { - const pipelineIid = await state.getPipelineIid(cwd, stateDir); + const pipelineIid = await state.getPipelineIid(stateDir); parser = await Parser.create(argv, writeStreams, pipelineIid, jobs); Commander.runCsv(parser, writeStreams, argv.listCsvAll); } else if (argv.job.length > 0) { assert(argv.stage === null, "You cannot use --stage when starting individual jobs"); - generateGitIgnore(cwd, stateDir); const time = process.hrtime(); if (argv.needs || argv.onlyNeeds) { - await fs.remove(`${cwd}/${stateDir}/artifacts`); - await state.incrementPipelineIid(cwd, stateDir); + await fs.remove(`${stateDir}/artifacts`); + await state.incrementPipelineIid(stateDir); } - const pipelineIid = await state.getPipelineIid(cwd, stateDir); + const pipelineIid = await state.getPipelineIid(stateDir); parser = await Parser.create(argv, writeStreams, pipelineIid, jobs); await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker"); await Commander.runJobs(argv, parser, writeStreams); @@ -82,19 +69,17 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`); } } else if (argv.stage) { - generateGitIgnore(cwd, stateDir); const time = process.hrtime(); - const pipelineIid = await state.getPipelineIid(cwd, stateDir); + const pipelineIid = await state.getPipelineIid(stateDir); parser = await Parser.create(argv, writeStreams, pipelineIid, jobs); await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker"); await Commander.runJobsInStage(argv, parser, writeStreams); writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`); } else { - generateGitIgnore(cwd, stateDir); const time = process.hrtime(); - await fs.remove(`${cwd}/${stateDir}/artifacts`); - await state.incrementPipelineIid(cwd, stateDir); - const pipelineIid = await state.getPipelineIid(cwd, stateDir); + await fs.remove(`${stateDir}/artifacts`); + await state.incrementPipelineIid(stateDir); + const pipelineIid = await state.getPipelineIid(stateDir); parser = await Parser.create(argv, writeStreams, pipelineIid, jobs); await Utils.rsyncTrackedFiles(cwd, stateDir, ".docker"); await Commander.runPipeline(argv, parser, writeStreams); diff --git a/src/index.ts b/src/index.ts index 5266b3966..e0d8175e0 100755 --- a/src/index.ts +++ b/src/index.ts @@ -106,7 +106,7 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs)); }) .option("cwd", { type: "string", - description: "Path to a current working directory", + description: "Path to the current working directory of the gitlab-ci-local executor", requiresArg: true, }) .option("completion", { @@ -146,17 +146,17 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs)); }) .option("state-dir", { type: "string", - description: "Location of the .gitlab-ci-local state dir, relative to cwd, eg. (symfony/.gitlab-ci-local/)", + description: "Location of the .gitlab-ci-local state dir", requiresArg: false, }) .option("file", { type: "string", - description: "Location of the .gitlab-ci.yml, relative to cwd, eg. (gitlab/.gitlab-ci.yml)", + description: "Location of the .gitlab-ci.yml", requiresArg: false, }) .option("home", { type: "string", - description: "Location of the HOME .gitlab-ci-local folder ($HOME/.gitlab-ci-local/variables.yml)", + description: "Location of the HOME(gcl global config) [default: $HOME/.gitlab-ci-local]", requiresArg: false, }) .option("shell-isolation", { @@ -275,7 +275,7 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs)); completionFilter(); } else { Argv.build({...yargsArgv, autoCompleting: true}) - .then(argv => state.getPipelineIid(argv.cwd, argv.stateDir).then(pipelineIid => ({argv, pipelineIid}))) + .then(argv => state.getPipelineIid(argv.stateDir).then(pipelineIid => ({argv, pipelineIid}))) .then(({argv, pipelineIid}) => Parser.create(argv, new WriteStreamsMock(), pipelineIid, [])) .then((parser) => { const jobNames = [...parser.jobs.values()].filter((j) => j.when != "never").map((j) => j.name); diff --git a/src/job.ts b/src/job.ts index 899bc707b..4c582c861 100644 --- a/src/job.ts +++ b/src/job.ts @@ -160,7 +160,7 @@ export class Job { if (this.jobData["image"]) { ciProjectDir = CI_PROJECT_DIR; } else if (argv.shellIsolation) { - ciProjectDir = `${cwd}/${stateDir}/builds/${this.safeJobName}`; + ciProjectDir = `${stateDir}/builds/${this.safeJobName}`; } predefinedVariables["CI_JOB_ID"] = `${this.jobId}`; @@ -463,7 +463,7 @@ export class Job { const imageName = this.imageName(expanded); const safeJobName = this.safeJobName; - const outputLogFilePath = `${argv.cwd}/${argv.stateDir}/output/${safeJobName}.log`; + const outputLogFilePath = `${argv.stateDir}/output/${safeJobName}.log`; await fs.ensureFile(outputLogFilePath); await fs.truncate(outputLogFilePath); @@ -527,7 +527,7 @@ export class Job { const serviceName = service.name; await this.pullImage(writeStreams, serviceName); const serviceContainerId = await this.startService(writeStreams, Utils.expandVariables({...expanded, ...service.variables}), service); - const serviceContainerLogFile = `${argv.cwd}/${argv.stateDir}/services-output/${this.safeJobName}/${serviceName}-${serviceIndex}.log`; + const serviceContainerLogFile = `${argv.stateDir}/services-output/${this.safeJobName}/${serviceName}-${serviceIndex}.log`; await this.serviceHealthCheck(writeStreams, service, serviceIndex, serviceContainerLogFile); const {stdout, stderr} = await Utils.spawn([this.argv.containerExecutable, "logs", serviceContainerId]); await fs.ensureFile(serviceContainerLogFile); @@ -656,7 +656,7 @@ export class Job { const cwd = this.argv.cwd; const stateDir = this.argv.stateDir; const safeJobName = this.safeJobName; - const outputFilesPath = `${cwd}/${stateDir}/output/${safeJobName}.log`; + const outputFilesPath = `${stateDir}/output/${safeJobName}.log`; const buildVolumeName = this.buildVolumeName; const tmpVolumeName = this.tmpVolumeName; const imageName = this.imageName(expanded); @@ -839,7 +839,7 @@ export class Job { cmd += "exit 0\n"; - const jobScriptFile = `${cwd}/${stateDir}/scripts/${safeJobName}_${this.jobId}`; + const jobScriptFile = `${stateDir}/scripts/${safeJobName}_${this.jobId}`; await fs.outputFile(jobScriptFile, cmd, "utf-8"); await fs.chmod(jobScriptFile, "0755"); this._filesToRm.push(jobScriptFile); @@ -877,7 +877,7 @@ export class Job { if (imageName) { cp.stdin?.end("/gcl-cmd"); } else { - cp.stdin?.end(`./${stateDir}/scripts/${safeJobName}_${this.jobId}`); + cp.stdin?.end(`${stateDir}/scripts/${safeJobName}_${this.jobId}`); } }); } @@ -936,7 +936,6 @@ export class Job { } private async initProducerReportsDotenvVariables (writeStreams: WriteStreams, expanded: {[key: string]: string}) { - const cwd = this.argv.cwd; const stateDir = this.argv.stateDir; const producers = this.producers; let producerReportsEnvs = {}; @@ -945,7 +944,7 @@ export class Job { if (producerDotenv === null) continue; const safeProducerName = Utils.safeDockerString(producer.name); - const dotenvFolder = `${cwd}/${stateDir}/artifacts/${safeProducerName}/.gitlab-ci-reports/dotenv/`; + const dotenvFolder = `${stateDir}/artifacts/${safeProducerName}/.gitlab-ci-reports/dotenv/`; if (await fs.pathExists(dotenvFolder)) { const dotenvFiles = (await Utils.spawn(["find", ".", "-type", "f"], dotenvFolder)).stdout.split("\n"); for (const dotenvFile of dotenvFiles) { @@ -973,7 +972,7 @@ export class Job { const time = process.hrtime(); const cacheName = await this.getUniqueCacheName(cwd, expanded, c.key); - const cacheFolder = `${cwd}/${stateDir}/cache/${cacheName}`; + const cacheFolder = `${stateDir}/cache/${cacheName}`; if (!await fs.pathExists(cacheFolder)) { continue; } @@ -989,13 +988,12 @@ export class Job { private async copyArtifactsIn (writeStreams: WriteStreams) { if ((!this.imageName(this._variables) && !this.argv.shellIsolation) || (this.producers ?? []).length === 0) return; - const cwd = this.argv.cwd; const stateDir = this.argv.stateDir; const time = process.hrtime(); const promises = []; for (const producer of this.producers ?? []) { const producerSafeName = Utils.safeDockerString(producer.name); - const artifactFolder = `${cwd}/${stateDir}/artifacts/${producerSafeName}`; + const artifactFolder = `${stateDir}/artifacts/${producerSafeName}`; if (!await fs.pathExists(artifactFolder)) { await fs.mkdirp(artifactFolder); } @@ -1015,7 +1013,7 @@ export class Job { copyIn (source: string) { const safeJobName = this.safeJobName; if (!this.imageName(this._variables) && this.argv.shellIsolation) { - return Utils.spawn(["rsync", "-a", `${source}/.`, `${this.argv.cwd}/${this.argv.stateDir}/builds/${safeJobName}`]); + return Utils.spawn(["rsync", "-a", `${source}/.`, `${this.argv.stateDir}/builds/${safeJobName}`]); } return Utils.spawn([this.argv.containerExecutable, "cp", `${source}/.`, `${this._containerId}:/gcl-builds`]); } @@ -1045,7 +1043,7 @@ export class Job { }); endTime = process.hrtime(time); - const readdir = await fs.readdir(`${this.argv.cwd}/${stateDir}/cache/${cacheName}`); + const readdir = await fs.readdir(`${stateDir}/cache/${cacheName}`); if (readdir.length === 0) { writeStreams.stdout(chalk`${this.formattedJobName} {yellow !! no cache was copied for ${path} !!}\n`); } else { @@ -1099,13 +1097,13 @@ export class Job { if (reportDotenvs != null) { reportDotenvs.forEach(async (reportDotenv) => { - if (!await fs.pathExists(`${cwd}/${stateDir}/artifacts/${safeJobName}/.gitlab-ci-reports/dotenv/${reportDotenv}`)) { + if (!await fs.pathExists(`${stateDir}/artifacts/${safeJobName}/.gitlab-ci-reports/dotenv/${reportDotenv}`)) { writeStreams.stderr(chalk`${this.formattedJobName} {yellow artifact reports dotenv '${reportDotenv}' could not be found}\n`); } }); } - const readdir = await fs.readdir(`${cwd}/${stateDir}/artifacts/${safeJobName}`); + const readdir = await fs.readdir(`${stateDir}/artifacts/${safeJobName}`); if (readdir.length === 0) { writeStreams.stdout(chalk`${this.formattedJobName} {yellow !! no artifacts was copied !!}\n`); } else { @@ -1114,9 +1112,9 @@ export class Job { if (this.artifactsToSource && (this.argv.shellIsolation || this.imageName(expanded))) { time = process.hrtime(); - await Utils.spawn(["rsync", "--exclude=/.gitlab-ci-reports/", "-a", `${cwd}/${stateDir}/artifacts/${safeJobName}/.`, cwd]); + await Utils.spawn(["rsync", "--exclude=/.gitlab-ci-reports/", "-a", `${stateDir}/artifacts/${safeJobName}/.`, cwd]); if (reportDotenv != null) { - await Utils.spawn(["rsync", "-a", `${cwd}/${stateDir}/artifacts/${safeJobName}/.gitlab-ci-reports/dotenv/.`, cwd]); + await Utils.spawn(["rsync", "-a", `${stateDir}/artifacts/${safeJobName}/.gitlab-ci-reports/dotenv/.`, cwd]); } endTime = process.hrtime(time); writeStreams.stdout(chalk`${this.formattedJobName} {magentaBright copied artifacts to cwd} in {magenta ${prettyHrtime(endTime)}}\n`); @@ -1128,7 +1126,7 @@ export class Job { const buildVolumeName = this.buildVolumeName; const cwd = this.argv.cwd; - await fs.mkdirp(`${cwd}/${stateDir}/${type}`); + await fs.mkdirp(`${stateDir}/${type}`); if (this.imageName(this._variables)) { const {stdout: containerId} = await Utils.bash(`${this.argv.containerExecutable} create -i ${dockerCmdExtras.join(" ")} -v ${buildVolumeName}:/gcl-builds/ -w /gcl-builds docker.io/firecow/gitlab-ci-local-util bash -c "${cmd}"`, cwd); @@ -1136,7 +1134,7 @@ export class Job { await Utils.spawn([this.argv.containerExecutable, "start", containerId, "--attach"]); await Utils.spawn([this.argv.containerExecutable, "cp", `${containerId}:/${type}/.`, `${stateDir}/${type}/.`], cwd); } else if (this.argv.shellIsolation) { - await Utils.bash(`bash -eo pipefail -c "${cmd}"`, `${cwd}/${stateDir}/builds/${safeJobName}`); + await Utils.bash(`bash -eo pipefail -c "${cmd}"`, `${stateDir}/builds/${safeJobName}`); } else if (!this.argv.shellIsolation) { await Utils.bash(`bash -eo pipefail -c "${cmd}"`, `${cwd}`); } diff --git a/src/parser-includes.ts b/src/parser-includes.ts index f43f41579..256208a07 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -87,7 +87,7 @@ export class ParserIncludes { } else if (value["project"]) { for (const fileValue of Array.isArray(value["file"]) ? value["file"] : [value["file"]]) { const fileDoc = await Parser.loadYaml( - `${cwd}/${stateDir}/includes/${gitData.remote.host}/${value["project"]}/${value["ref"] || "HEAD"}/${fileValue}` + `${stateDir}/includes/${gitData.remote.host}/${value["project"]}/${value["ref"] || "HEAD"}/${fileValue}` , {inputs: value.inputs || {}} , expandVariables); // Expand local includes inside a "project"-like include @@ -143,13 +143,13 @@ export class ParserIncludes { const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]); const fsUrl = Utils.fsUrl(`https://${domain}/${project}/-/raw/${ref}/${file}`); const fileDoc = await Parser.loadYaml( - `${cwd}/${stateDir}/includes/${fsUrl}`, {inputs: value.inputs || {}}, expandVariables + `${stateDir}/includes/${fsUrl}`, {inputs: value.inputs || {}}, expandVariables ); includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } else if (value["remote"]) { const fsUrl = Utils.fsUrl(value["remote"]); const fileDoc = await Parser.loadYaml( - `${cwd}/${stateDir}/includes/${fsUrl}`, {inputs: value.inputs || {}}, expandVariables + `${stateDir}/includes/${fsUrl}`, {inputs: value.inputs || {}}, expandVariables ); includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } else { @@ -217,7 +217,7 @@ export class ParserIncludes { static async downloadIncludeRemote (cwd: string, stateDir: string, url: string, fetchIncludes: boolean): Promise { const fsUrl = Utils.fsUrl(url); try { - const target = `${cwd}/${stateDir}/includes/${fsUrl}`; + const target = `${stateDir}/includes/${fsUrl}`; if (await fs.pathExists(target) && !fetchIncludes) return; const res = await axios.get(url); await fs.outputFile(target, res.data); @@ -231,25 +231,25 @@ export class ParserIncludes { const normalizedFile = file.replace(/^\/+/, ""); try { const target = `${stateDir}/includes/${remote.host}/${project}/${ref}`; - if (await fs.pathExists(`${cwd}/${target}/${normalizedFile}`) && !fetchIncludes) return; + if (await fs.pathExists(`${target}/${normalizedFile}`) && !fetchIncludes) return; if (remote.schema.startsWith("http")) { const ext = "tmp-" + Math.random(); - await fs.mkdirp(path.dirname(`${cwd}/${target}/${normalizedFile}`)); + await fs.mkdirp(path.dirname(`${target}/${normalizedFile}`)); await Utils.bash(` - cd ${cwd}/${stateDir} \\ + cd ${stateDir} \\ && git clone --branch "${ref}" -n --depth=1 --filter=tree:0 \\ ${remote.schema}://${remote.host}/${project}.git \\ - ${cwd}/${target}.${ext} \\ - && cd ${cwd}/${target}.${ext} \\ + ${target}.${ext} \\ + && cd ${target}.${ext} \\ && git sparse-checkout set --no-cone ${normalizedFile} \\ && git checkout \\ - && cd ${cwd}/${stateDir} \\ - && cp ${cwd}/${target}.${ext}/${normalizedFile} \\ - ${cwd}/${target}/${normalizedFile} + && cd ${stateDir} \\ + && cp ${target}.${ext}/${normalizedFile} \\ + ${target}/${normalizedFile} `, cwd); } else { - await fs.mkdirp(`${cwd}/${target}`); + await fs.mkdirp(`${target}`); await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${normalizedFile} | tar -f - -xC ${target}/`, cwd); } } catch (e) { diff --git a/src/parser.ts b/src/parser.ts index 02f67f8ac..1203ecb81 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -67,8 +67,8 @@ export class Parser { } const parsingTime = process.hrtime(time); - const pathToExpandedGitLabCi = path.join(argv.cwd, argv.stateDir, "expanded-gitlab-ci.yml"); - fs.mkdirpSync(path.join(argv.cwd, argv.stateDir)); + const pathToExpandedGitLabCi = path.join(argv.stateDir, "expanded-gitlab-ci.yml"); + fs.mkdirpSync(argv.stateDir); fs.writeFileSync(pathToExpandedGitLabCi, yaml.dump(parser.gitlabData)); writeStreams.stderr(chalk`{grey parsing and downloads finished in ${prettyHrtime(parsingTime)}.}\n`); @@ -104,7 +104,7 @@ export class Parser { const expanded = Utils.expandVariables(variables); let yamlDataList: any[] = [{stages: [".pre", "build", "test", "deploy", ".post"]}]; - const gitlabCiData = await Parser.loadYaml(`${cwd}/${file}`, {}, this.expandVariables); + const gitlabCiData = await Parser.loadYaml(file, {}, this.expandVariables); yamlDataList = yamlDataList.concat(await ParserIncludes.init(gitlabCiData, {cwd, stateDir, writeStreams, gitData, fetchIncludes, variables: expanded, expandVariables: this.expandVariables, maximumIncludes: argv.maximumIncludes})); ParserIncludes.resetCount(); diff --git a/src/state.ts b/src/state.ts index 9ca315bcb..000d646f2 100644 --- a/src/state.ts +++ b/src/state.ts @@ -9,15 +9,15 @@ const loadStateYML = async (stateFile: string): Promise => { return yaml.load(stateFileContent) || {}; }; -const getPipelineIid = async (cwd: string, stateDir: string) => { - const stateFile = `${cwd}/${stateDir}/state.yml`; +const getPipelineIid = async (stateDir: string) => { + const stateFile = `${stateDir}/state.yml`; const ymlData = await loadStateYML(stateFile); return ymlData["pipelineIid"] ? ymlData["pipelineIid"] : 0; }; -const incrementPipelineIid = async (cwd: string, stateDir: string) => { - const stateFile = `${cwd}/${stateDir}/state.yml`; +const incrementPipelineIid = async (stateDir: string) => { + const stateFile = `${stateDir}/state.yml`; const ymlData = await loadStateYML(stateFile); ymlData["pipelineIid"] = ymlData["pipelineIid"] != null ? ymlData["pipelineIid"] + 1 : 0; diff --git a/src/utils.ts b/src/utils.ts index eff62d774..26fcde593 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -68,7 +68,7 @@ export class Utils { } static async getCoveragePercent (cwd: string, stateDir: string, coverageRegex: string, jobName: string) { - const content = await fs.readFile(`${cwd}/${stateDir}/output/${jobName}.log`, "utf8"); + const content = await fs.readFile(`${stateDir}/output/${jobName}.log`, "utf8"); const regex = new RegExp(coverageRegex.replace(/^\//, "").replace(/\/$/, ""), "gm"); const matches = Array.from(content.matchAll(regex)); @@ -300,7 +300,7 @@ ${evalStr} static async rsyncTrackedFiles (cwd: string, stateDir: string, target: string): Promise<{hrdeltatime: [number, number]}> { const time = process.hrtime(); - await fs.mkdirp(`${cwd}/${stateDir}/builds/${target}`); + await fs.mkdirp(`${stateDir}/builds/${target}`); await Utils.bash(`rsync -a --delete-excluded --delete --exclude-from=<(git ls-files -o --directory | awk '{print "/"$0}') --exclude ${stateDir}/ ./ ${stateDir}/builds/${target}/`, cwd); return {hrdeltatime: process.hrtime(time)}; } diff --git a/src/variables-from-files.ts b/src/variables-from-files.ts index cc5d472ca..6a26f9a9f 100644 --- a/src/variables-from-files.ts +++ b/src/variables-from-files.ts @@ -22,11 +22,10 @@ export class VariablesFromFiles { static async init (argv: Argv, writeStreams: WriteStreams, gitData: GitData): Promise<{[name: string]: CICDVariable}> { const cwd = argv.cwd; - const stateDir = argv.stateDir; const homeDir = argv.home; const remoteVariables = argv.remoteVariables; const autoCompleting = argv.autoCompleting; - const homeVariablesFile = `${homeDir}/${stateDir}/variables.yml`; + const homeVariablesFile = `${homeDir}/variables.yml`; const variables: {[name: string]: CICDVariable} = {}; let remoteFileData: any = {}; let homeFileData: any = {}; diff --git a/tests/cases.test.ts b/tests/cases.test.ts index e548b2105..88b5afe39 100644 --- a/tests/cases.test.ts +++ b/tests/cases.test.ts @@ -33,7 +33,7 @@ test("something/unknown-directory (non-existing dir)", async () => { expect(true).toBe(false); } catch (e) { assert(e instanceof AssertionError, "e is not instanceof AssertionError"); - expect(e.message).toBe(chalk`${process.cwd()}/something/unknown-directory is not a directory`); + expect(e.message).toBe(chalk`--cwd (${process.cwd()}/something/unknown-directory) is not a directory`); } }); @@ -46,7 +46,7 @@ test("docs (no .gitlab-ci.yml)", async () => { expect(true).toBe(false); } catch (e) { assert(e instanceof AssertionError, "e is not instanceof AssertionError"); - expect(e.message).toBe(chalk`${process.cwd()}/docs/.gitlab-ci.yml could not be found`); + expect(e.message).toBe(chalk`--file (${process.cwd()}/docs/.gitlab-ci.yml) could not be found`); } }); diff --git a/tests/test-cases/argv-cwd/integration.artifacts-after-afterscript.test.ts b/tests/test-cases/argv-cwd/integration.artifacts-after-afterscript.test.ts index db7f7e03b..0e955e340 100644 --- a/tests/test-cases/argv-cwd/integration.artifacts-after-afterscript.test.ts +++ b/tests/test-cases/argv-cwd/integration.artifacts-after-afterscript.test.ts @@ -72,7 +72,7 @@ job: }); test("won't fallback if not inside git repository", async () => { - const {stdout: tmpDir} = await Utils.bash("mktemp -d"); + const {stdout: tmpDir} = await Utils.bash("realpath $(mktemp -d)"); process.chdir(tmpDir); try { @@ -82,7 +82,7 @@ job: }, writeStreams); } catch (e: any) { assert(e instanceof AssertionError, "e is not instanceof AssertionError"); - expect(e.message).toContain(chalk`${tmpDir}/.gitlab-ci.yml could not be found`); + expect(e.message).toContain(chalk`--file (${tmpDir}/.gitlab-ci.yml) could not be found`); } }); @@ -98,7 +98,7 @@ job: }, writeStreams); } catch (e: any) { assert(e instanceof AssertionError, "e is not instanceof AssertionError"); - expect(e.message).toEqual(chalk`${originalDir}/${currentRelativeDir}/.gitlab-ci.yml could not be found`); + expect(e.message).toEqual(chalk`--file (${originalDir}/${currentRelativeDir}/.gitlab-ci.yml) could not be found`); } }); @@ -114,7 +114,7 @@ job: }, writeStreams); } catch (e: any) { assert(e instanceof AssertionError, "e is not instanceof AssertionError"); - expect(e.message).toEqual(chalk`${originalDir}/${currentRelativeDir}/.gitlab-ci.yml could not be found`); + expect(e.message).toEqual(chalk`--file (${originalDir}/${currentRelativeDir}/.gitlab-ci.yml) could not be found`); } }); diff --git a/tests/test-cases/custom-home/.home/somefolder/file-var b/tests/test-cases/custom-home/.home/.gitlab-ci-local/somefolder/file-var similarity index 100% rename from tests/test-cases/custom-home/.home/somefolder/file-var rename to tests/test-cases/custom-home/.home/.gitlab-ci-local/somefolder/file-var diff --git a/tests/test-cases/custom-home/integration.custom-home.test.ts b/tests/test-cases/custom-home/integration.custom-home.test.ts index 43dd0f1ce..bf7943b33 100644 --- a/tests/test-cases/custom-home/integration.custom-home.test.ts +++ b/tests/test-cases/custom-home/integration.custom-home.test.ts @@ -12,12 +12,15 @@ beforeAll(() => { initSpawnSpy([...WhenStatics.all, spyGitRemote]); }); +const home = `${process.cwd()}/tests/test-cases/custom-home/.home/.gitlab-ci-local`; +const homeNormalizeKey = `${process.cwd()}/tests/test-cases/custom-home/.home-normalize-key/.gitlab-ci-local`; + test("custom-home ", async () => { const writeStreams = new WriteStreamsMock(); await handler({ cwd: "tests/test-cases/custom-home", job: ["test-staging"], - home: "tests/test-cases/custom-home/.home", + home: home, }, writeStreams); const expected = [ @@ -37,7 +40,7 @@ test("custom-home ", async () => { await handler({ cwd: "tests/test-cases/custom-home", job: ["test-production"], - home: "tests/test-cases/custom-home/.home", + home: home, }, writeStreams); const expected = [ @@ -53,7 +56,7 @@ test("custom-home ", async () => { await handler({ cwd: "tests/test-cases/custom-home", job: ["test-image"], - home: "tests/test-cases/custom-home/.home", + home: home, }, writeStreams); const expected = [ @@ -69,7 +72,7 @@ test("custom-home ", async () => { await handler({ cwd: "tests/test-cases/custom-home", job: ["test-normalize-key"], - home: "tests/test-cases/custom-home/.home-normalize-key", + home: homeNormalizeKey, }, writeStreams); const expected = [ @@ -84,7 +87,7 @@ test("custom-home ", async () => { await handler({ cwd: "tests/test-cases/custom-home", job: ["test-predefined-overwrite"], - home: "tests/test-cases/custom-home/.home", + home: home, }, writeStreams); const expected = [ @@ -99,7 +102,7 @@ test("custom-home ", async () => { await handler({ cwd: "tests/test-cases/custom-home", job: ["build-job"], - home: "tests/test-cases/custom-home/.home", + home: home, }, writeStreams); const expected = [ diff --git a/tests/test-cases/custom-state-dir/.gitlab-ci.yml b/tests/test-cases/custom-state-dir/.gitlab-ci.yml new file mode 100644 index 000000000..b7f55c83d --- /dev/null +++ b/tests/test-cases/custom-state-dir/.gitlab-ci.yml @@ -0,0 +1,4 @@ +--- +job: + script: + - echo "Test something" diff --git a/tests/test-cases/custom-state-dir/integration.custom-state-dir.test.ts b/tests/test-cases/custom-state-dir/integration.custom-state-dir.test.ts new file mode 100644 index 000000000..5d957a985 --- /dev/null +++ b/tests/test-cases/custom-state-dir/integration.custom-state-dir.test.ts @@ -0,0 +1,70 @@ +import {WriteStreamsMock} from "../../../src/write-streams"; +import {handler} from "../../../src/handler"; +import chalk from "chalk"; +import fs from "fs-extra"; +import {initSpawnSpy} from "../../mocks/utils.mock"; +import {WhenStatics} from "../../mocks/when-statics"; + +beforeAll(() => { + initSpawnSpy(WhenStatics.all); +}); + +const cwd = `${process.cwd()}/tests/test-cases/custom-state-dir`; + +describe("--state-dir ", () => { + const customHomeDir = `${process.cwd()}/tests/test-cases/custom-state-dir/custom-gitlab-ci-local`; + afterAll(() => { + fs.rmSync(customHomeDir, {recursive: true}); + }); + + test("", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: cwd, + stateDir: customHomeDir, + }, writeStreams); + + const expected = [ + chalk`{blueBright job} {greenBright >} Test something`, + ]; + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); + + const uniqueStateDir = cwd.replaceAll("/", "."); + + expect(fs.pathExistsSync(`${customHomeDir}/${uniqueStateDir}`)).toEqual(true); + expect(fs.pathExistsSync(`${customHomeDir}/${uniqueStateDir}/builds`)).toEqual(true); + expect(fs.pathExistsSync(`${customHomeDir}/${uniqueStateDir}/output`)).toEqual(true); + expect(fs.pathExistsSync(`${customHomeDir}/${uniqueStateDir}/scripts`)).toEqual(true); + expect(fs.pathExistsSync(`${customHomeDir}/${uniqueStateDir}/expanded-gitlab-ci.yml`)).toEqual(true); + expect(fs.pathExistsSync(`${customHomeDir}/${uniqueStateDir}/state.yml`)).toEqual(true); + + expect(fs.pathExistsSync(`${customHomeDir}/${uniqueStateDir}/.gitignore`)).toEqual(false); + }); +}); + +describe("--state-dir ", () => { + const customHomeDir = "relative-custom-gitlab-ci-local"; + afterAll(() => { + fs.rmSync(`${cwd}/${customHomeDir}`, {recursive: true}); + }); + + test("", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: cwd, + stateDir: customHomeDir, + }, writeStreams); + + const expected = [ + chalk`{blueBright job} {greenBright >} Test something`, + ]; + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); + + expect(fs.pathExistsSync(`${cwd}/${customHomeDir}/builds`)).toEqual(true); + expect(fs.pathExistsSync(`${cwd}/${customHomeDir}/output`)).toEqual(true); + expect(fs.pathExistsSync(`${cwd}/${customHomeDir}/scripts`)).toEqual(true); + expect(fs.pathExistsSync(`${cwd}/${customHomeDir}/expanded-gitlab-ci.yml`)).toEqual(true); + expect(fs.pathExistsSync(`${cwd}/${customHomeDir}/state.yml`)).toEqual(true); + expect(fs.pathExistsSync(`${cwd}/${customHomeDir}/.gitignore`)).toEqual(true); + }); +}); diff --git a/tests/test-cases/variable-order/integration.variable-order.test.ts b/tests/test-cases/variable-order/integration.variable-order.test.ts index b8cc8fe21..959c6663a 100644 --- a/tests/test-cases/variable-order/integration.variable-order.test.ts +++ b/tests/test-cases/variable-order/integration.variable-order.test.ts @@ -14,7 +14,7 @@ test("variable-order --needs", async () => { cwd: "tests/test-cases/variable-order", job: ["test-job"], variable: ["PROJECT_VARIABLE=project-value"], - home: "tests/test-cases/variable-order/.home", + home: `${process.cwd()}/tests/test-cases/variable-order/.home/.gitlab-ci-local`, }, writeStreams); const expected = [