Location for user dependencies +Ivy Dependency downloader

kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala

@@ -33,70 +33,52 @@ import org.apache.ivy.util.{DefaultMessageLogger, Message}
 import org.springframework.core.io.support._
 
 /**
- * Represents a dependency downloader for jars that uses Ivy underneath.
- */
+  * Represents a dependency downloader for jars that uses Ivy underneath.
+  */
 class IvyDependencyDownloader(
-  val repositoryUrl: String,
-  val baseDirectory: String
-) extends DependencyDownloader {
-  private val ivySettings = new IvySettings()
-  private val resolver = new IBiblioResolver
+                               val baseDirectory: String,
+                               val ivySettingsFile: String
+                             ) extends DependencyDownloader {
 
-  resolver.setUsepoms(true)
-  resolver.setM2compatible(true)
-  resolver.setName("central")
-
-  // Add our resolver as the main resolver (IBiblio goes to Maven Central)
-  ivySettings.addResolver(resolver)
+  private val ivySettings = new IvySettings()
 
-  // Mark our resolver as the default one to use
-  ivySettings.setDefaultResolver(resolver.getName)
+  private val settingsFile = new File(ivySettingsFile)
+
+  if (settingsFile.exists()) {
+    ivySettings.load(settingsFile)
+  } else {
+    // Use default maven resolver, when ivy settings file doesn't exist.
+    val resolver = new IBiblioResolver
+    resolver.setUsepoms(true)
+    resolver.setM2compatible(true)
+    resolver.setName("central")
+    ivySettings.addResolver(resolver)
+    ivySettings.setDefaultResolver(resolver.getName)
+  }
 
   // Set the destination
   ivySettings.setBaseDir(new File(baseDirectory))
   ivySettings.setDefaultResolutionCacheBasedir(baseDirectory)
   ivySettings.setDefaultRepositoryCacheBasedir(baseDirectory)
 
   //creates an Ivy instance with settings
-  val ivy = Ivy.newInstance(ivySettings)
-
-  private def getBaseDependencies: Iterable[DependencyDescriptor] = {
-    val xmlModuleDescriptor = XmlModuleDescriptorParser.getInstance()
-    val getDependencies = (url: URL) => xmlModuleDescriptor.parseDescriptor(
-      new IvySettings(), url, false
-    ).getDependencies
-
-    // Find all of the *ivy.xml files on the classpath.
-    val ivyFiles = new PathMatchingResourcePatternResolver().getResources(
-      "classpath*:**/*ivy.xml"
-    )
-    val classpathURLs = ivyFiles.map(_.getURI.toURL)
-
-    // Get all of the dependencies from the *ivy.xml files
-    val dependencies = classpathURLs.map(getDependencies).flatMap(_.toSeq)
-
-    // Remove duplicates based on artifact name
-    val distinctDependencies =
-      dependencies.groupBy(_.getDependencyId.getName).map(_._2.head)
-
-    distinctDependencies
-  }
+  private val ivy = Ivy.newInstance(ivySettings)
 
   override def retrieve(
-    groupId: String,
-    artifactId: String,
-    version: String,
-    transitive: Boolean = true,
-    excludeBaseDependencies: Boolean,
-    ignoreResolutionErrors: Boolean,
-    extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
-    verbose: Boolean,
-    trace: Boolean,
-    configuration: Option[String] = None,
-    artifactType: Option[String] = None,
-    artifactClassifier: Option[String] = None,
-    excludes: Set[(String,String)] = Set.empty
-  ): Seq[URI] = {
+                         groupId: String,
+                         artifactId: String,
+                         version: String,
+                         transitive: Boolean = true,
+                         excludeBaseDependencies: Boolean,
+                         ignoreResolutionErrors: Boolean,
+                         extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
+                         verbose: Boolean,
+                         trace: Boolean,
+                         configuration: Option[String] = None,
+                         artifactType: Option[String] = None,
+                         artifactClassifier: Option[String] = None,
+                         excludes: Set[(String, String)] = Set.empty
+                       ): Seq[URI] = {
     // Start building the ivy.xml file
     val ivyFile = File.createTempFile("ivy-custom", ".xml")
     ivyFile.deleteOnExit()
@@ -153,8 +135,8 @@ class IvyDependencyDownloader(
 
 
     excludes.foreach(x => {
-      val moduleId = new ModuleId(x._1,x._2);
-      val artifactId = new ArtifactId(moduleId,"*","*","*");
+      val moduleId = new ModuleId(x._1, x._2);
+      val artifactId = new ArtifactId(moduleId, "*", "*", "*");
       val exclusion = new DefaultExcludeRule(artifactId, new RegexpPatternMatcher(), null);
       md.addExcludeRule(exclusion);
     })
@@ -196,11 +178,33 @@ class IvyDependencyDownloader(
     artifactURLs.map(_.toURI)
   }
 
+  private def getBaseDependencies: Iterable[DependencyDescriptor] = {
+    val xmlModuleDescriptor = XmlModuleDescriptorParser.getInstance()
+    val getDependencies = (url: URL) => xmlModuleDescriptor.parseDescriptor(
+      new IvySettings(), url, false
+    ).getDependencies
+
+    // Find all of the *ivy.xml files on the classpath.
+    val ivyFiles = new PathMatchingResourcePatternResolver().getResources(
+      "classpath*:**/*ivy.xml"
+    )
+    val classpathURLs = ivyFiles.map(_.getURI.toURL)
+
+    // Get all of the dependencies from the *ivy.xml files
+    val dependencies = classpathURLs.map(getDependencies).flatMap(_.toSeq)
+
+    // Remove duplicates based on artifact name
+    val distinctDependencies =
+      dependencies.groupBy(_.getDependencyId.getName).map(_._2.head)
+
+    distinctDependencies
+  }
+
   /**
-   * Uses our printstream in Ivy's LoggingEngine
-   *
-   * @param printStream the print stream to use
-   */
+    * Uses our printstream in Ivy's LoggingEngine
+    *
+    * @param printStream the print stream to use
+    */
   override def setPrintStream(printStream: PrintStream): Unit = {
     ivy.getLoggerEngine.setDefaultLogger(
       new DefaultMessageLogger(Message.MSG_INFO) {
@@ -215,40 +219,40 @@ class IvyDependencyDownloader(
   }
 
   /**
-   * Adds the specified resolver url as an additional search option.
-   *
-   * @param url The url of the repository
-   */
+    * Adds the specified resolver url as an additional search option.
+    *
+    * @param url The url of the repository
+    */
   override def addMavenRepository(url: URL, credentials: Option[Credentials]): Unit = ???
 
   /**
-   * Remove the specified resolver url from the search options.
-   *
-   * @param url The url of the repository
-   */
+    * Remove the specified resolver url from the search options.
+    *
+    * @param url The url of the repository
+    */
   override def removeMavenRepository(url: URL): Unit = ???
 
   /**
-   * Returns a list of all repositories used by the downloader.
-   *
-   * @return The list of repositories as URIs
-   */
+    * Returns a list of all repositories used by the downloader.
+    *
+    * @return The list of repositories as URIs
+    */
   override def getRepositories: Seq[URI] = Seq(
     DependencyDownloader.DefaultMavenRepository.toURI
   )
 
   /**
-   * Sets the directory where all downloaded jars will be stored.
-   *
-   * @param directory The directory to use
-   * @return True if successfully set directory, otherwise false
-   */
+    * Sets the directory where all downloaded jars will be stored.
+    *
+    * @param directory The directory to use
+    * @return True if successfully set directory, otherwise false
+    */
   override def setDownloadDirectory(directory: File): Boolean = false
 
   /**
-   * Returns the current directory where dependencies will be downloaded.
-   *
-   * @return The directory as a string
-   */
+    * Returns the current directory where dependencies will be downloaded.
+    *
+    * @return The directory as a string
+    */
   override def getDownloadDirectory: String = baseDirectory
 }

kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala

@@ -20,8 +20,8 @@ package org.apache.toree.boot
 import java.io.{File, OutputStream}
 
 import com.typesafe.config.{Config, ConfigFactory}
-import joptsimple.{OptionParser, OptionSpec}
 import joptsimple.util.RegexMatcher._
+import joptsimple.{OptionParser, OptionSpec}
 
 import scala.collection.JavaConverters._
 
@@ -88,6 +88,16 @@ class CommandLineOptions(args: Seq[String]) {
     "directory where user added jars are stored (MUST EXIST)"
   ).withRequiredArg().ofType(classOf[String])
 
+  private val _deps_dir = parser.accepts(
+    "deps-dir",
+    "directory where user dependencies are stored (MUST EXIST)"
+  ).withRequiredArg().ofType(classOf[String])
+
+  private val _ivy_settings = parser.accepts(
+    "ivy-settings",
+    "location of ivy settings xml file"
+  ).withRequiredArg().ofType(classOf[String])
+
   private val _default_interpreter =
     parser.accepts("default-interpreter", "default interpreter for the kernel")
       .withRequiredArg().ofType(classOf[String])
@@ -171,6 +181,8 @@ class CommandLineOptions(args: Seq[String]) {
       "max_interpreter_threads" -> get(_max_interpreter_threads),
       "alternate_sigint" -> get(_alternate_sigint),
       "jar_dir" -> get(_jar_dir),
+      "deps_dir" -> get(_deps_dir),
+      "ivy_settings" -> get(_ivy_settings),
       "default_interpreter" -> get(_default_interpreter),
       // deprecated in favor of spark-context-initialization-mode none
       // "nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),