Pr context

.github/workflows/integrationTests.yml

@@ -20,9 +20,16 @@ jobs:
  if: ${{github.event_name != 'push' }}
  with:
  labels: 'safe to test'
+ - name: Checkout PR head branch
+ uses: actions/checkout@v3
+ with:
+ ref: ${{ github.event.pull_request.head.ref }}
+ repository: ${{ github.event.pull_request.head.repo.full_name }}
+ - name: Print current branch
+ run: git branch --show-current
  Distribution:
  needs: Pretest
- name: Distribution (${{ matrix.os }})
+ name: Distribution (${{ matrix.os }}) - check!
  strategy:
  fail-fast: false
  matrix:
@@ -272,11 +279,13 @@ jobs:
 
  npm:
  needs: Pretest
- name: npm (${{ matrix.os }})
+ name: npm (${{ matrix.os }}) - node ${{ matrix.node }}
  strategy:
  fail-fast: false
  matrix:
  os: [ ubuntu-latest, macos-13, windows-latest ]
+ # Testing both npm < 8.19 and npm >= 8.19
+ node: [ 15, 20 ]
  runs-on: ${{ matrix.os }}
  steps:
  - name: Checkout code
@@ -287,9 +296,11 @@ jobs:
  - name: Install npm
  uses: actions/setup-node@v3
  with:
- node-version: "15"
+ node-version: ${{ matrix.node }}
+
  - name: Config list
  run: npm config ls -l
+
  - name: Install Java
  uses: actions/setup-java@v3
  with:

.gitignore

@@ -18,4 +18,9 @@ atlassian-*.xml
 /itest/src/test/resources/org/jfrog/build/cache
 /build-info-extractor/src/test/resources/artifactory-bi.properties
 local.properties
-target
+*.class
+/*/bin/test/org/jfrog/build
+/*/bin/test/*/snapshots/*.xml
+/*/bin/test/*/settings/build-info*.json
+/*/bin/test/*/pipLog.txt
+target

build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java

@@ -131,4 +131,4 @@ private CommandResults runCommand(File workingDirectory, String[] args, List<Str
 
  return npmCommandRes;
  }
-}
+}

build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java

@@ -5,6 +5,7 @@
 import org.jfrog.build.api.builder.ModuleType;
 import org.jfrog.build.api.util.Log;
 import org.jfrog.build.client.ProxyConfiguration;
+import org.jfrog.build.client.Version;
 import org.jfrog.build.extractor.BuildInfoExtractor;
 import org.jfrog.build.extractor.builder.ModuleBuilder;
 import org.jfrog.build.extractor.ci.BuildInfo;
@@ -189,9 +190,11 @@ private void createTempNpmrc(Path workingDir, List<String> commandArgs) throws I
  npmrcBuilder.append("proxy = ").append(this.npmProxy).append("\n");
  }
 
+ // Update Auth property for newer npm versions
+ handleNpmCompatibility(npmAuth, workingDir);
+
  // Save npm auth
  npmAuth.forEach((key, value) -> npmrcBuilder.append(key).append("=").append(value).append("\n"));
-
  // Write npmrc file
  try (FileWriter fileWriter = new FileWriter(npmrcPath.toFile());
  BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) {
@@ -200,6 +203,47 @@ private void createTempNpmrc(Path workingDir, List<String> commandArgs) throws I
  }
  }
 
+ /**
+ * Handles when Npm is at least version 8.19 the Auth related settings needing to be scoped to a specific registry.
+ * Results in transforming:
+ * 
+ * old NPMRC of :
+ * 
+ * registry=http://NO-NO-Repo/
+ * _auth={{AuthString}} 
+ * 
+ * 
+ * into the new NPMRC:
+ * 
+ * registry=http://NO-NO-Repo/
+ * //NO-NO-Repo/:_auth={{AuthString}}
+ * 
+ */
+ private void handleNpmCompatibility(Properties npmAuth, Path workingDir) throws IOException, InterruptedException{
+ Version npmVersion = new Version(this.npmDriver.version(workingDir.toFile()));
+ if(npmVersion.isAtLeast(new Version("8.19"))){
+ logger.debug("NPM version at least 8.19");
+ try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) {
+ String newAuthKey = artifactoryManager.getUrl();
+ if (!StringUtils.endsWith(newAuthKey, "/")) {
+ newAuthKey += "/";
+ }
+ newAuthKey += "api/npm/:";
+ newAuthKey = newAuthKey.replaceAll("^http(s)?:","");
+
+ String[] checkList = { "_auth","_authToken","username","_password", "email", "certfile", "keyfile"};
+ for(String propKey: checkList){
+ String prop = npmAuth.getProperty(propKey);
+ if(prop != null){
+ logger.debug("Found "+ propKey +", replacing with " + newAuthKey + propKey);
+ npmAuth.setProperty(newAuthKey+propKey, prop);
+ npmAuth.remove(propKey);
+ }
+ }
+ }
+ }
+ }
+
  /**
  * Adds an array-value config to a StringBuilder of .npmrc file, in the following format:
  * key[] = value

build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java

@@ -1,26 +1,70 @@
 package org.jfrog.build.extractor.npm.extractor;
 
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.jfrog.build.IntegrationTestsBase;
 import org.jfrog.build.extractor.builder.BuildInfoBuilder;
 import org.jfrog.build.extractor.builder.DependencyBuilder;
 import org.jfrog.build.extractor.builder.ModuleBuilder;
 import org.jfrog.build.extractor.ci.BuildInfo;
 import org.jfrog.build.extractor.ci.Dependency;
 import org.jfrog.build.extractor.ci.Module;
+import org.jfrog.build.extractor.npm.NpmDriver;
+import org.jfrog.build.extractor.npm.types.NpmProject;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.testng.Assert.*;
 
 import static org.jfrog.build.extractor.npm.extractor.NpmBuildInfoExtractor.getDependenciesMapFromBuild;
-import static org.testng.Assert.assertEquals;
 
 @Test
-public class NpmBuildInfoExtractorTest {
+public class NpmBuildInfoExtractorTest extends IntegrationTestsBase {
+
+ private static final String TEST_SPACE = "npm_test_space";
+ private static final File tempWorkspace = new File(System.getProperty("java.io.tmpdir"), TEST_SPACE);
+ private static final Path PROJECTS_ROOT = Paths.get(".").toAbsolutePath().normalize().resolve(Paths.get("src", "test", "resources", "org", "jfrog", "build", "extractor"));
+
+
+ private static final String NPM_LOCAL_REPO = "build-info-tests-npm-local";
+ private static final Set<String> DEV_SCOPE = Stream.of("dev").collect(Collectors.toSet());
+ private static final Set<String> PROD_SCOPE = Stream.of("prod").collect(Collectors.toSet());
+ private static final Set<String> DEV_PROD_SCOPE = Stream.of("prod", "dev").collect(Collectors.toSet());
+
+ public NpmBuildInfoExtractorTest() {
+ localRepo1 = getKeyWithTimestamp(NPM_LOCAL_REPO);
+ remoteRepo = "";
+ virtualRepo = "";
+ }
 
+ @AfterMethod
+ protected void cleanup() throws IOException {
+ FileUtils.deleteDirectory(tempWorkspace);
+ }
+
+ @BeforeMethod
+ protected void init2() throws IOException {
+ FileUtils.forceMkdir(tempWorkspace);
+ }
  @DataProvider
  private Object[][] getDependenciesMapFromBuildProvider() {
  return new Object[][]{
@@ -122,4 +166,126 @@ private Module createTestModule(String id, List<Dependency> dependencies) {
  new DependencyBuilder().id("mod2dep1:2.1.0").sha1("sha1-mod2dep1").md5("md5-mod2dep1").build(),
  new DependencyBuilder().id("mod2dep2:2.2.0").sha1("sha1-mod2dep2").md5("md5-mod2dep2").build()
  };
+
+ private enum Project {
+ // Dependencies
+ ASGARD("asgard", "jfrog-asgard", "jfrog-asgard", "2.0.0", "a1fc28aa8733a161fa92d03379b71468d19292cd", "2fb7c420d2119831bc38559138d3444e"),
+ MIDGARD("midgard", "jfrog-midgard", "jfrog-midgard", "1.0.0", "547b8c7bb019863cc26438ef36e9b2d33668a626", "82f1558593727a7c89fb0b91859dab26"),
+ ALFHEIM("alfheim", "jfrog-alfheim", "jfrog-alfheim", "3.5.2", "f5592b523d2693649a94bbc2377cc653607a4053", "93e19985bb1c7c815abef052b67be244"),
+ SVARTALFHEIM("svartalfheim", "jfrog-svartalfheim", "jfrog-svartalfheim", "0.5.0", "473a5e001c67d716b8c9993245bd0ba2010c7374", "b1678118e32908b8e57f26fef1a23473"),
+
+ // Test projects
+ A("a", "NpmExtractorTest Project A", "package-name1", "v0.0.1", "", ""),
+ B("b", "NpmExtractorTest-Project-B", "package-name2", "0.0.2", "", ""),
+ C("c", "NpmExtractorTestProjectC", "package-name3", "=0.0.3", "", "");
+
+ private final File projectOrigin;
+ private final String targetDir;
+ private final String name;
+ private final String version;
+ private final String sha1;
+ private final String md5;
+
+ Project(String sourceDir, String targetDir, String name, String version, String sha1, String md5) {
+ this.projectOrigin = PROJECTS_ROOT.resolve(sourceDir).toFile();
+ this.targetDir = targetDir;
+ this.name = name;
+ this.version = version;
+ this.sha1 = sha1;
+ this.md5 = md5;
+ }
+
+ private String getModuleId() {
+ return String.format("%s:%s", name, version);
+ }
+
+ private String getPackedFileName() {
+ return String.format("%s-%s.tgz", name, version);
+ }
+
+ private String getDependencyId() {
+ return String.format("%s:%s", name, version);
+ }
+
+ private Dependency toDependency(String[][] requestedBy, Set<String> scope) {
+ return new DependencyBuilder().id(getDependencyId())
+ .sha1(sha1)
+ .md5(md5)
+ .scopes(scope)
+ .requestedBy(requestedBy)
+ .build();
+ }
+
+ private String getRemotePath() {
+ return String.format("%s/-", name);
+ }
+
+ private String getTargetPath() {
+ return String.format("%s/%s", getRemotePath(), getPackedFileName());
+ }
+ }
+
+
+ @DataProvider
+ private Object[][] npmCiProvider() {
+ Dependency[] expectedDepsStep1 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"package-name1:v0.0.1"}}, PROD_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name1:v0.0.1"}}, PROD_SCOPE)};
+ Dependency[] expectedDepsStep2 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "@jscope/package-name2:0.0.2"}}, DEV_SCOPE), Project.MIDGARD.toDependency(new String[][]{{"@jscope/package-name2:0.0.2"}}, DEV_SCOPE), Project.ALFHEIM.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "@jscope/package-name2:0.0.2"}}, DEV_SCOPE)};
+ Dependency[] expectedDepsStep3 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "package-name3:=0.0.3"}, {"package-name3:=0.0.3"}}, DEV_PROD_SCOPE), Project.MIDGARD.toDependency(new String[][]{{"package-name3:=0.0.3"}}, DEV_SCOPE), Project.ALFHEIM.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "package-name3:=0.0.3"}}, DEV_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE)};
+ Dependency[] expectedDepsStep4 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE)};
+ return new Object[][]{
+ {Project.A, expectedDepsStep1, "", true},
+ {Project.B, expectedDepsStep2, "", true},
+ {Project.B, new Dependency[]{}, "--production", false},
+ {Project.C, expectedDepsStep3, "", true},
+ {Project.C, expectedDepsStep4, "--only=production", true}
+ };
+ }
+
+ @SuppressWarnings("unused")
+ @Test(dataProvider = "npmCiProvider")
+ public void npmCiTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath) {
+ runNpmTest(project, expectedDependencies, args, packageJsonPath, true);
+ }
+
+ private void runNpmTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath, boolean isNpmCi) {
+ args += " --verbose --no-audit";
+ Path projectDir = null;
+ try {
+ // Prepare.
+ projectDir = createProjectDir(project);
+ Path path = packageJsonPath ? projectDir.resolve("package.json") : projectDir;
+ if (isNpmCi) {
+ // Run npm install to generate package-lock.json file.
+ new NpmInstallCi(artifactoryManagerBuilder, localRepo1, args, log, path, null, null, null, false, null).execute();
+ }
+
+ NpmDriver driver = new NpmDriver(null);
+ List<String> commandArgs = StringUtils.isBlank(args) ? new ArrayList<>() : Arrays.asList(args.trim().split("\\s+"));
+ NpmProject proj = new NpmProject(commandArgs, localRepo1, path, isNpmCi);
+ // Execute command.
+
+ NpmBuildInfoExtractor buildExtractor = new NpmBuildInfoExtractor(artifactoryManagerBuilder, driver, log, null, null,null);
+ BuildInfo buildInfo = buildExtractor.extract(proj);
+
+ // Validate.
+ assertEquals(buildInfo.getModules().size(), 1);
+ Module module = buildInfo.getModules().get(0);
+ assertEquals(module.getType(), "npm");
+ assertEquals(module.getId(), project.getModuleId());
+ assertEqualsNoOrder(module.getDependencies().toArray(), expectedDependencies);
+ } catch (Exception e) {
+ fail(ExceptionUtils.getStackTrace(e));
+ } finally {
+ if (projectDir != null) {
+ FileUtils.deleteQuietly(projectDir.toFile());
+ }
+ }
+ }
+
+ private Path createProjectDir(Project project) throws IOException {
+ File projectDir = Files.createTempDirectory(project.targetDir).toFile();
+ FileUtils.copyDirectory(project.projectOrigin, projectDir);
+ return projectDir.toPath();
+ }
+
 }