feat: Use Apache HTTPClient for downloads of public resources (#6949)

jeremylong · Sep 18, 2024 · 865e39e · 865e39e
2 parents cef8f0b + 11ae195
commit 865e39e
Show file tree

Hide file tree

Showing 36 changed files with 1,007 additions and 782 deletions.
diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java
@@ -38,6 +38,8 @@
 import org.owasp.dependencycheck.exception.ExceptionCollection;
 import org.owasp.dependencycheck.exception.ReportException;
 import org.owasp.dependencycheck.reporting.ReportGenerator.Format;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.owasp.dependencycheck.utils.SeverityUtil;
 import org.slf4j.impl.StaticLoggerBinder;
@@ -2050,6 +2052,11 @@ protected void executeWithContextClassloader() throws BuildException {
  dealWithReferences();
  validateConfiguration();
  populateSettings();
+ try {
+ Downloader.getInstance().configure(getSettings());
+ } catch (InvalidSettingException e) {
+ throw new BuildException(e);
+ }
  try (Engine engine = new Engine(Check.class.getClassLoader(), getSettings())) {
  for (Resource resource : getPath()) {
  final FileProvider provider = resource.as(FileProvider.class);

diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java
@@ -26,6 +26,8 @@
 import org.apache.tools.ant.Project;
 import org.apache.tools.ant.Task;
 import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.impl.StaticLoggerBinder;
 
@@ -179,6 +181,11 @@ private void muteNoisyLoggers() {
  @SuppressWarnings("squid:RedundantThrowsDeclarationCheck")
  protected void executeWithContextClassloader() throws BuildException {
  populateSettings();
+ try {
+ Downloader.getInstance().configure(settings);
+ } catch (InvalidSettingException e) {
+ throw new BuildException(e);
+ }
  try (Engine engine = new Engine(Engine.Mode.EVIDENCE_PROCESSING, getSettings())) {
  engine.purge();
  } finally {

diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java
@@ -22,6 +22,8 @@
 import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.impl.StaticLoggerBinder;
 
@@ -594,6 +596,11 @@ public void setHostedSuppressionsEnabled(Boolean hostedSuppressionsEnabled) {
  @Override
  protected void executeWithContextClassloader() throws BuildException {
  populateSettings();
+ try {
+ Downloader.getInstance().configure(getSettings());
+ } catch (InvalidSettingException e) {
+ throw new BuildException(e);
+ }
  try (Engine engine = new Engine(Update.class.getClassLoader(), getSettings())) {
  engine.doUpdates();
  } catch (UpdateException ex) {

diff --git a/cli/src/main/java/org/owasp/dependencycheck/App.java b/cli/src/main/java/org/owasp/dependencycheck/App.java
@@ -33,6 +33,7 @@
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
 import org.owasp.dependencycheck.exception.ExceptionCollection;
 import org.owasp.dependencycheck.exception.ReportException;
+import org.owasp.dependencycheck.utils.Downloader;
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;
@@ -141,6 +142,7 @@ public int run(String[] args) {
  } else {
  try {
  populateSettings(cli);
+ Downloader.getInstance().configure(settings);
  } catch (InvalidSettingException ex) {
  LOGGER.error(ex.getMessage());
  LOGGER.debug(ERROR_LOADING_PROPERTIES_FILE, ex);
@@ -162,6 +164,7 @@ public int run(String[] args) {
  try {
  populateSettings(cli);
  settings.setBoolean(Settings.KEYS.AUTO_UPDATE, true);
+ Downloader.getInstance().configure(settings);
  } catch (InvalidSettingException ex) {
  LOGGER.error(ex.getMessage());
  LOGGER.debug(ERROR_LOADING_PROPERTIES_FILE, ex);
@@ -182,6 +185,7 @@ public int run(String[] args) {
  } else if (cli.isRunScan()) {
  try {
  populateSettings(cli);
+ Downloader.getInstance().configure(settings);
  } catch (InvalidSettingException ex) {
  LOGGER.error(ex.getMessage(), ex);
  LOGGER.debug(ERROR_LOADING_PROPERTIES_FILE, ex);

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractSuppressionAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractSuppressionAnalyzer.java
@@ -345,14 +345,13 @@ private List<SuppressionRule> loadSuppressionFile(final SuppressionParser parser
  deleteTempFile = true;
  file = getSettings().getTempFile("suppression", "xml");
  final URL url = new URL(suppressionFilePath);
- final Downloader downloader = new Downloader(getSettings());
  try {
- downloader.fetchFile(url, file, false, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD);
+ Downloader.getInstance().fetchFile(url, file, false, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD);
  } catch (DownloadFailedException ex) {
  LOGGER.trace("Failed download suppression file - first attempt", ex);
  try {
  Thread.sleep(500);
- downloader.fetchFile(url, file, true, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD);
+ Downloader.getInstance().fetchFile(url, file, true, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD);
  } catch (TooManyRequestsException ex1) {
  throw new SuppressionParseException("Unable to download supression file `" + file
  + "`; received 429 - too many requests", ex1);

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java
@@ -234,8 +234,7 @@ private void processPom(Dependency dependency, MavenArtifact ma) throws IOExcept
  Files.delete(pomFile.toPath());
  LOGGER.debug("Downloading {}", ma.getPomUrl());
  //TODO add caching
- final Downloader downloader = new Downloader(getSettings());
- downloader.fetchFile(new URL(ma.getPomUrl()), pomFile,
+ Downloader.getInstance().fetchFile(new URL(ma.getPomUrl()), pomFile, true,
  Settings.KEYS.ANALYZER_ARTIFACTORY_API_USERNAME,
  Settings.KEYS.ANALYZER_ARTIFACTORY_API_TOKEN);
  PomUtils.analyzePOM(dependency, pomFile);

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java
@@ -241,7 +241,6 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy
  + "this could result in undetected CPE/CVEs.", dependency.getFileName());
  LOGGER.debug("Unable to delete temp file");
  }
- final Downloader downloader = new Downloader(getSettings());
  final int maxAttempts = this.getSettings().getInt(Settings.KEYS.ANALYZER_CENTRAL_RETRY_COUNT, 3);
  int retryCount = 0;
  long sleepingTimeBetweenRetriesInMillis = BASE_RETRY_WAIT;
@@ -258,7 +257,7 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy
  do {
  //CSOFF: NestedTryDepth
  try {
- downloader.fetchFile(new URL(ma.getPomUrl()), pomFile);
+ Downloader.getInstance().fetchFile(new URL(ma.getPomUrl()), pomFile);
  success = true;
  } catch (DownloadFailedException ex) {
  try {

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/HintAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/HintAnalyzer.java
@@ -267,13 +267,12 @@ private void loadHintRules() throws HintParseException {
  deleteTempFile = true;
  file = getSettings().getTempFile("hint", "xml");
  final URL url = new URL(filePath);
- final Downloader downloader = new Downloader(getSettings());
  try {
- downloader.fetchFile(url, file, false);
+ Downloader.getInstance().fetchFile(url, file, false);
  } catch (DownloadFailedException ex) {
  try {
  Thread.sleep(500);
- downloader.fetchFile(url, file, true);
+ Downloader.getInstance().fetchFile(url, file, true);
  } catch (TooManyRequestsException ex1) {
  throw new HintParseException("Unable to download hint file `" + file + "`; received 429 - too many requests", ex1);
  } catch (ResourceNotFoundException ex1) {

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java
@@ -282,8 +282,7 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy
  LOGGER.debug("Unable to delete temp file");
  }
  LOGGER.debug("Downloading {}", ma.getPomUrl());
- final Downloader downloader = new Downloader(getSettings());
- downloader.fetchFile(new URL(ma.getPomUrl()), pomFile);
+ Downloader.getInstance().fetchFile(new URL(ma.getPomUrl()), pomFile);
  PomUtils.analyzePOM(dependency, pomFile);
  } catch (DownloadFailedException ex) {
  LOGGER.warn("Unable to download pom.xml for {} from Nexus repository; "