diff --git a/.github/workflows/integrationTests.yml b/.github/workflows/integrationTests.yml index ccf86093f..985e58fd0 100644 --- a/.github/workflows/integrationTests.yml +++ b/.github/workflows/integrationTests.yml @@ -20,9 +20,16 @@ jobs: if: ${{github.event_name != 'push' }} with: labels: 'safe to test' + - name: Checkout PR head branch + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + - name: Print current branch + run: git branch --show-current Distribution: needs: Pretest - name: Distribution (${{ matrix.os }}) + name: Distribution (${{ matrix.os }}) - check! strategy: fail-fast: false matrix: @@ -272,11 +279,13 @@ jobs: npm: needs: Pretest - name: npm (${{ matrix.os }}) + name: npm (${{ matrix.os }}) - node ${{ matrix.node }} strategy: fail-fast: false matrix: os: [ ubuntu-latest, macos-13, windows-latest ] + # Testing both npm < 8.19 and npm >= 8.19 + node: [ 15, 20 ] runs-on: ${{ matrix.os }} steps: - name: Checkout code @@ -287,9 +296,11 @@ jobs: - name: Install npm uses: actions/setup-node@v3 with: - node-version: "15" + node-version: ${{ matrix.node }} + - name: Config list run: npm config ls -l + - name: Install Java uses: actions/setup-java@v3 with: diff --git a/.gitignore b/.gitignore index af526146c..5cef1fcc1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,9 @@ atlassian-*.xml /itest/src/test/resources/org/jfrog/build/cache /build-info-extractor/src/test/resources/artifactory-bi.properties local.properties -target \ No newline at end of file +*.class +/*/bin/test/org/jfrog/build +/*/bin/test/*/snapshots/*.xml +/*/bin/test/*/settings/build-info*.json +/*/bin/test/*/pipLog.txt +target diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java index 3a30f4ddf..9de094cd6 100644 --- a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java +++ b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java @@ -131,4 +131,4 @@ private CommandResults runCommand(File workingDirectory, String[] args, List commandArgs) throws I npmrcBuilder.append("proxy = ").append(this.npmProxy).append("\n"); } + // Update Auth property for newer npm versions + handleNpmCompatibility(npmAuth, workingDir); + // Save npm auth npmAuth.forEach((key, value) -> npmrcBuilder.append(key).append("=").append(value).append("\n")); - // Write npmrc file try (FileWriter fileWriter = new FileWriter(npmrcPath.toFile()); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { @@ -200,6 +203,47 @@ private void createTempNpmrc(Path workingDir, List commandArgs) throws I } } + /** + * Handles when Npm is at least version 8.19 the Auth related settings needing to be scoped to a specific registry. + * Results in transforming: + * + * old NPMRC of : + * + * registry=http://NO-NO-Repo/ + * _auth={{AuthString}} + * + * + * into the new NPMRC: + * + * registry=http://NO-NO-Repo/ + * //NO-NO-Repo/:_auth={{AuthString}} + * + */ + private void handleNpmCompatibility(Properties npmAuth, Path workingDir) throws IOException, InterruptedException{ + Version npmVersion = new Version(this.npmDriver.version(workingDir.toFile())); + if(npmVersion.isAtLeast(new Version("8.19"))){ + logger.debug("NPM version at least 8.19"); + try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) { + String newAuthKey = artifactoryManager.getUrl(); + if (!StringUtils.endsWith(newAuthKey, "/")) { + newAuthKey += "/"; + } + newAuthKey += "api/npm/:"; + newAuthKey = newAuthKey.replaceAll("^http(s)?:",""); + + String[] checkList = { "_auth","_authToken","username","_password", "email", "certfile", "keyfile"}; + for(String propKey: checkList){ + String prop = npmAuth.getProperty(propKey); + if(prop != null){ + logger.debug("Found "+ propKey +", replacing with " + newAuthKey + propKey); + npmAuth.setProperty(newAuthKey+propKey, prop); + npmAuth.remove(propKey); + } + } + } + } + } + /** * Adds an array-value config to a StringBuilder of .npmrc file, in the following format: * key[] = value diff --git a/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java b/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java index d90761197..b5ce5fb97 100644 --- a/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java +++ b/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java @@ -1,26 +1,70 @@ package org.jfrog.build.extractor.npm.extractor; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.jfrog.build.IntegrationTestsBase; import org.jfrog.build.extractor.builder.BuildInfoBuilder; import org.jfrog.build.extractor.builder.DependencyBuilder; import org.jfrog.build.extractor.builder.ModuleBuilder; import org.jfrog.build.extractor.ci.BuildInfo; import org.jfrog.build.extractor.ci.Dependency; import org.jfrog.build.extractor.ci.Module; +import org.jfrog.build.extractor.npm.NpmDriver; +import org.jfrog.build.extractor.npm.types.NpmProject; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.testng.Assert.*; import static org.jfrog.build.extractor.npm.extractor.NpmBuildInfoExtractor.getDependenciesMapFromBuild; -import static org.testng.Assert.assertEquals; @Test -public class NpmBuildInfoExtractorTest { +public class NpmBuildInfoExtractorTest extends IntegrationTestsBase { + + private static final String TEST_SPACE = "npm_test_space"; + private static final File tempWorkspace = new File(System.getProperty("java.io.tmpdir"), TEST_SPACE); + private static final Path PROJECTS_ROOT = Paths.get(".").toAbsolutePath().normalize().resolve(Paths.get("src", "test", "resources", "org", "jfrog", "build", "extractor")); + + + private static final String NPM_LOCAL_REPO = "build-info-tests-npm-local"; + private static final Set DEV_SCOPE = Stream.of("dev").collect(Collectors.toSet()); + private static final Set PROD_SCOPE = Stream.of("prod").collect(Collectors.toSet()); + private static final Set DEV_PROD_SCOPE = Stream.of("prod", "dev").collect(Collectors.toSet()); + + public NpmBuildInfoExtractorTest() { + localRepo1 = getKeyWithTimestamp(NPM_LOCAL_REPO); + remoteRepo = ""; + virtualRepo = ""; + } + @AfterMethod + protected void cleanup() throws IOException { + FileUtils.deleteDirectory(tempWorkspace); + } + + @BeforeMethod + protected void init2() throws IOException { + FileUtils.forceMkdir(tempWorkspace); + } @DataProvider private Object[][] getDependenciesMapFromBuildProvider() { return new Object[][]{ @@ -122,4 +166,126 @@ private Module createTestModule(String id, List dependencies) { new DependencyBuilder().id("mod2dep1:2.1.0").sha1("sha1-mod2dep1").md5("md5-mod2dep1").build(), new DependencyBuilder().id("mod2dep2:2.2.0").sha1("sha1-mod2dep2").md5("md5-mod2dep2").build() }; + + private enum Project { + // Dependencies + ASGARD("asgard", "jfrog-asgard", "jfrog-asgard", "2.0.0", "a1fc28aa8733a161fa92d03379b71468d19292cd", "2fb7c420d2119831bc38559138d3444e"), + MIDGARD("midgard", "jfrog-midgard", "jfrog-midgard", "1.0.0", "547b8c7bb019863cc26438ef36e9b2d33668a626", "82f1558593727a7c89fb0b91859dab26"), + ALFHEIM("alfheim", "jfrog-alfheim", "jfrog-alfheim", "3.5.2", "f5592b523d2693649a94bbc2377cc653607a4053", "93e19985bb1c7c815abef052b67be244"), + SVARTALFHEIM("svartalfheim", "jfrog-svartalfheim", "jfrog-svartalfheim", "0.5.0", "473a5e001c67d716b8c9993245bd0ba2010c7374", "b1678118e32908b8e57f26fef1a23473"), + + // Test projects + A("a", "NpmExtractorTest Project A", "package-name1", "v0.0.1", "", ""), + B("b", "NpmExtractorTest-Project-B", "package-name2", "0.0.2", "", ""), + C("c", "NpmExtractorTestProjectC", "package-name3", "=0.0.3", "", ""); + + private final File projectOrigin; + private final String targetDir; + private final String name; + private final String version; + private final String sha1; + private final String md5; + + Project(String sourceDir, String targetDir, String name, String version, String sha1, String md5) { + this.projectOrigin = PROJECTS_ROOT.resolve(sourceDir).toFile(); + this.targetDir = targetDir; + this.name = name; + this.version = version; + this.sha1 = sha1; + this.md5 = md5; + } + + private String getModuleId() { + return String.format("%s:%s", name, version); + } + + private String getPackedFileName() { + return String.format("%s-%s.tgz", name, version); + } + + private String getDependencyId() { + return String.format("%s:%s", name, version); + } + + private Dependency toDependency(String[][] requestedBy, Set scope) { + return new DependencyBuilder().id(getDependencyId()) + .sha1(sha1) + .md5(md5) + .scopes(scope) + .requestedBy(requestedBy) + .build(); + } + + private String getRemotePath() { + return String.format("%s/-", name); + } + + private String getTargetPath() { + return String.format("%s/%s", getRemotePath(), getPackedFileName()); + } + } + + + @DataProvider + private Object[][] npmCiProvider() { + Dependency[] expectedDepsStep1 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"package-name1:v0.0.1"}}, PROD_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name1:v0.0.1"}}, PROD_SCOPE)}; + Dependency[] expectedDepsStep2 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "@jscope/package-name2:0.0.2"}}, DEV_SCOPE), Project.MIDGARD.toDependency(new String[][]{{"@jscope/package-name2:0.0.2"}}, DEV_SCOPE), Project.ALFHEIM.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "@jscope/package-name2:0.0.2"}}, DEV_SCOPE)}; + Dependency[] expectedDepsStep3 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "package-name3:=0.0.3"}, {"package-name3:=0.0.3"}}, DEV_PROD_SCOPE), Project.MIDGARD.toDependency(new String[][]{{"package-name3:=0.0.3"}}, DEV_SCOPE), Project.ALFHEIM.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "package-name3:=0.0.3"}}, DEV_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE)}; + Dependency[] expectedDepsStep4 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE)}; + return new Object[][]{ + {Project.A, expectedDepsStep1, "", true}, + {Project.B, expectedDepsStep2, "", true}, + {Project.B, new Dependency[]{}, "--production", false}, + {Project.C, expectedDepsStep3, "", true}, + {Project.C, expectedDepsStep4, "--only=production", true} + }; + } + + @SuppressWarnings("unused") + @Test(dataProvider = "npmCiProvider") + public void npmCiTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath) { + runNpmTest(project, expectedDependencies, args, packageJsonPath, true); + } + + private void runNpmTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath, boolean isNpmCi) { + args += " --verbose --no-audit"; + Path projectDir = null; + try { + // Prepare. + projectDir = createProjectDir(project); + Path path = packageJsonPath ? projectDir.resolve("package.json") : projectDir; + if (isNpmCi) { + // Run npm install to generate package-lock.json file. + new NpmInstallCi(artifactoryManagerBuilder, localRepo1, args, log, path, null, null, null, false, null).execute(); + } + + NpmDriver driver = new NpmDriver(null); + List commandArgs = StringUtils.isBlank(args) ? new ArrayList<>() : Arrays.asList(args.trim().split("\\s+")); + NpmProject proj = new NpmProject(commandArgs, localRepo1, path, isNpmCi); + // Execute command. + + NpmBuildInfoExtractor buildExtractor = new NpmBuildInfoExtractor(artifactoryManagerBuilder, driver, log, null, null,null); + BuildInfo buildInfo = buildExtractor.extract(proj); + + // Validate. + assertEquals(buildInfo.getModules().size(), 1); + Module module = buildInfo.getModules().get(0); + assertEquals(module.getType(), "npm"); + assertEquals(module.getId(), project.getModuleId()); + assertEqualsNoOrder(module.getDependencies().toArray(), expectedDependencies); + } catch (Exception e) { + fail(ExceptionUtils.getStackTrace(e)); + } finally { + if (projectDir != null) { + FileUtils.deleteQuietly(projectDir.toFile()); + } + } + } + + private Path createProjectDir(Project project) throws IOException { + File projectDir = Files.createTempDirectory(project.targetDir).toFile(); + FileUtils.copyDirectory(project.projectOrigin, projectDir); + return projectDir.toPath(); + } + }