SOLR-16781: Remove solrconfig.xml <lib> directives

solr/CHANGES.txt

@@ -96,6 +96,10 @@ Deprecation Removals
 
 * SOLR-17576: Remove deprecated master/slave option language from ReplicationHandler. (Eric Pugh)
 
+* SOLR-16781: Support for `<lib/>` directives (used in solrconfig.xml to add JARs on a core-by-core basis) has been removed.  Users
+  looking for similar functionality can use Solr's package manager.  Users that don't need to vary JAR access on a per-core basis
+  have many options, including the `<sharedLib/>` tag and directly modifying Solr's classpath prior to JVM startup. (Jason Gerlowski)
+
 Dependency Upgrades
 ---------------------
 (No changes)

solr/core/src/java/org/apache/solr/cli/RunExampleTool.java

@@ -647,6 +647,11 @@ protected Map<String, Object> startSolr(
     if (!isWindows && cwdPath.length() > 1 && solrHome.startsWith(cwdPath))
       solrHome = solrHome.substring(cwdPath.length() + 1);
 
+    final var syspropArg =
+        ("techproducts".equals(cli.getOptionValue(EXAMPLE_OPTION)))
+            ? "-Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true"
+            : "";
+
     String startCmd =
         String.format(
             Locale.ROOT,
@@ -661,7 +666,8 @@ protected Map<String, Object> startSolr(
             forceArg,
             verboseArg,
             extraArgs,
-            jvmOptsArg);
+            jvmOptsArg,
+            syspropArg);
     startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing
 
     echo("\nStarting up Solr on port " + port + " using command:");

solr/core/src/java/org/apache/solr/core/ConfigSetService.java

@@ -272,7 +272,7 @@ public final ConfigSet loadConfigSet(CoreDescriptor dcore) {
       NamedList<?> properties = loadConfigSetProperties(dcore, coreLoader);
       boolean trusted = isConfigSetTrusted(coreLoader);
 
-      SolrConfig solrConfig = createSolrConfig(dcore, coreLoader, trusted);
+      SolrConfig solrConfig = createSolrConfig(dcore, coreLoader);
       return new ConfigSet(
           configSetName(dcore),
           solrConfig,
@@ -314,13 +314,12 @@ public ConfigSetService(SolrResourceLoader loader, boolean shareSchema) {
    *
    * @param cd the core's CoreDescriptor
    * @param loader the core's resource loader
-   * @param isTrusted is the configset trusted?
    * @return a SolrConfig object
    */
-  protected SolrConfig createSolrConfig(
-      CoreDescriptor cd, SolrResourceLoader loader, boolean isTrusted) throws IOException {
+  protected SolrConfig createSolrConfig(CoreDescriptor cd, SolrResourceLoader loader)
+      throws IOException {
     return SolrConfig.readFromResourceLoader(
-        loader, cd.getConfigName(), isTrusted, cd.getSubstitutableProperties());
+        loader, cd.getConfigName(), cd.getSubstitutableProperties());
   }
 
   /**

solr/core/src/java/org/apache/solr/core/SolrConfig.java

@@ -17,7 +17,6 @@
 package org.apache.solr.core;
 
 import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.common.params.CommonParams.PATH;
 import static org.apache.solr.core.ConfigOverlay.ZNODEVER;
 import static org.apache.solr.core.SolrConfig.PluginOpts.LAZY;
 import static org.apache.solr.core.SolrConfig.PluginOpts.MULTI_OK;
@@ -31,7 +30,6 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.lang.invoke.MethodHandles;
-import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -96,7 +94,7 @@
 
 /**
  * Provides a static reference to a Config object modeling the main configuration data for a Solr
- * instance -- typically found in "solrconfig.xml".
+ * core -- typically found in "solrconfig.xml".
  */
 public class SolrConfig implements MapSerializable {
 
@@ -143,16 +141,13 @@ public enum PluginOpts {
    * @param name the configuration name used by the loader if the stream is null
    */
   public SolrConfig(Path instanceDir, String name) throws IOException {
-    this(new SolrResourceLoader(instanceDir), name, true, null);
+    this(new SolrResourceLoader(instanceDir), name, null);
   }
 
   public static SolrConfig readFromResourceLoader(
-      SolrResourceLoader loader,
-      String name,
-      boolean isConfigsetTrusted,
-      Properties substitutableProperties) {
+      SolrResourceLoader loader, String name, Properties substitutableProperties) {
     try {
-      return new SolrConfig(loader, name, isConfigsetTrusted, substitutableProperties);
+      return new SolrConfig(loader, name, substitutableProperties);
     } catch (Exception e) {
       String resource;
       if (loader instanceof ZkSolrResourceLoader) {
@@ -196,15 +191,9 @@ public InputStream apply(String s) {
    *
    * @param loader the resource loader
    * @param name the configuration name
-   * @param isConfigsetTrusted false if configset was uploaded using unsecured configset upload API,
-   *     true otherwise
    * @param substitutableProperties optional properties to substitute into the XML
    */
-  private SolrConfig(
-      SolrResourceLoader loader,
-      String name,
-      boolean isConfigsetTrusted,
-      Properties substitutableProperties) {
+  private SolrConfig(SolrResourceLoader loader, String name, Properties substitutableProperties) {
     this.resourceLoader = loader;
     this.resourceName = name;
     this.substituteProperties = substitutableProperties;
@@ -237,7 +226,7 @@ private SolrConfig(
       rootDataHashCode = this.root.txt().hashCode();
 
       getRequestParams();
-      initLibs(loader, isConfigsetTrusted);
+      initLibs(loader);
       String val =
           root.child(
                   IndexSchema.LUCENE_MATCH_VERSION_PARAM,
@@ -934,11 +923,10 @@ public PluginInfo getPluginInfo(String type) {
         SolrException.ErrorCode.SERVER_ERROR, "Multiple plugins configured for type: " + type);
   }
 
-  private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) {
+  private void initLibs(SolrResourceLoader loader) {
     // TODO Want to remove SolrResourceLoader.getInstancePath; it can be on a Standalone subclass.
     // For Zk subclass, it's needed for the time being as well.  We could remove that one if we
-    // remove two things in SolrCloud: (1) instancePath/lib  and (2) solrconfig lib directives with
-    // relative paths. Can wait till 9.0.
+    // remove "instancePath/lib" in SolrCloud. Can wait till 9.0.
     Path instancePath = loader.getInstancePath();
     List<URL> urls = new ArrayList<>();
 
@@ -950,48 +938,15 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) {
         log.warn("Couldn't add files from {} to classpath: {}", libPath, e);
       }
     }
-
-    List<ConfigNode> nodes = root.getAll("lib");
-    if (nodes != null && nodes.size() > 0) {
-      if (!isConfigsetTrusted) {
-        throw new SolrException(
-            ErrorCode.UNAUTHORIZED,
-            "The configset for this collection was uploaded without any authentication in place,"
-                + " and use of <lib> is not available for collections with untrusted configsets. To use this component, re-upload the configset"
-                + " after enabling authentication and authorization.");
-      }
-
-      for (int i = 0; i < nodes.size(); i++) {
-        ConfigNode node = nodes.get(i);
-        String baseDir = node.attr("dir");
-        String path = node.attr(PATH);
-        if (null != baseDir) {
-          // :TODO: add support for a simpler 'glob' mutually exclusive of regex
-          Path dir = instancePath.resolve(baseDir);
-          String regex = node.attr("regex");
-          try {
-            if (regex == null) urls.addAll(SolrResourceLoader.getURLs(dir));
-            else urls.addAll(SolrResourceLoader.getFilteredURLs(dir, regex));
-          } catch (IOException e) {
-            log.warn("Couldn't add files from {} filtered by {} to classpath: {}", dir, regex, e);
-          }
-        } else if (null != path) {
-          final Path dir = instancePath.resolve(path);
-          try {
-            urls.add(dir.toUri().toURL());
-          } catch (MalformedURLException e) {
-            log.warn("Couldn't add file {} to classpath: {}", dir, e);
-          }
-        } else {
-          throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'");
-        }
-      }
-    }
-
     if (!urls.isEmpty()) {
       loader.addToClassLoader(urls);
       loader.reloadLuceneSPI();
     }
+
+    List<ConfigNode> nodes = root.getAll("lib");
+    if (nodes != null && nodes.size() > 0) {
+      log.warn("<lib/> entries no longer supported in solrconfig.xml; ignoring...");
+    }
   }
 
   public int getMultipartUploadLimitKB() {

solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java

@@ -254,7 +254,7 @@ public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp)
       try {
         InMemoryResourceLoader loader =
             new InMemoryResourceLoader(coreContainer, mutableId, SOLR_CONFIG_XML, data);
-        SolrConfig.readFromResourceLoader(loader, SOLR_CONFIG_XML, requestIsTrusted, null);
+        SolrConfig.readFromResourceLoader(loader, SOLR_CONFIG_XML, null);
       } catch (Exception exc) {
         updateFileError = exc;
       }

solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java

@@ -677,9 +677,7 @@ ManagedIndexSchema deleteNestedDocsFieldsIfNeeded(ManagedIndexSchema schema, boo
 
   SolrConfig loadSolrConfig(String configSet) {
     ZkSolrResourceLoader zkLoader = zkLoaderForConfigSet(configSet);
-    boolean trusted = isConfigSetTrusted(configSet);
-
-    return SolrConfig.readFromResourceLoader(zkLoader, SOLR_CONFIG_XML, trusted, null);
+    return SolrConfig.readFromResourceLoader(zkLoader, SOLR_CONFIG_XML, null);
   }
 
   ManagedIndexSchema loadLatestSchema(String configSet) {

solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml

@@ -28,12 +28,6 @@
   <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>
   <schemaFactory class="ClassicIndexSchemaFactory"/>
 
-  <!-- see TestConfig.testLib() -->
-  <lib dir="../../lib-dirs/a" />
-  <lib dir="../../lib-dirs/b" regex="b." />
-  <lib dir="../../lib-dirs/c" regex="c1" />
-  <lib path="../../lib-dirs/d/d1/" />
-
   <!-- see TestConfig.testJavaProperty -->
   <propTest attr1="${solr.test.sys.prop1}-$${literal}"
             attr2="${non.existent.sys.prop:default-from-config}">prefix-${solr.test.sys.prop2}-suffix</propTest>

solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java

@@ -1421,48 +1421,6 @@ public void testUploadWithScriptUpdateProcessor() throws Exception {
     scriptRequest("newcollection2");
   }
 
-  @Test
-  public void testUploadWithLibDirective() throws Exception {
-    final String untrustedSuffix = "-untrusted";
-    uploadConfigSetWithAssertions("with-lib-directive", untrustedSuffix, null);
-    // try to create a collection with the uploaded configset
-    ignoreException("without any authentication in place");
-    Throwable thrown =
-        expectThrows(
-            SolrClient.RemoteSolrException.class,
-            () -> {
-              createCollection(
-                  "newcollection3",
-                  "with-lib-directive" + untrustedSuffix,
-                  1,
-                  1,
-                  cluster.getSolrClient());
-            });
-    unIgnoreException("without any authentication in place");
-
-    assertThat(thrown.getMessage(), containsString("Underlying core creation failed"));
-
-    // Authorization on
-    final String trustedSuffix = "-trusted";
-    uploadConfigSetWithAssertions("with-lib-directive", trustedSuffix, "solr");
-    // try to create a collection with the uploaded configset
-    CollectionAdminResponse resp =
-        createCollection(
-            "newcollection3", "with-lib-directive" + trustedSuffix, 1, 1, cluster.getSolrClient());
-
-    SolrInputDocument doc = sdoc("id", "4055", "subject", "Solr");
-    cluster.getSolrClient().add("newcollection3", doc);
-    cluster.getSolrClient().commit("newcollection3");
-    assertEquals(
-        "4055",
-        cluster
-            .getSolrClient()
-            .query("newcollection3", params("q", "*:*"))
-            .getResults()
-            .get(0)
-            .get("id"));
-  }
-
   @Test
   public void testUploadWithForbiddenContent() throws Exception {
     // Uploads a config set containing a script, a class file and jar file, will return 400 error

solr/core/src/test/org/apache/solr/core/TestConfLoadPerf.java

@@ -84,7 +84,7 @@ public InputStream openResource(String resource) throws IOException {
     long startTime = System.currentTimeMillis();
     int numReads = 100;
     for (int i = 0; i < numReads; i++) {
-      allConfigs.add(SolrConfig.readFromResourceLoader(srl, "solrconfig.xml", true, null));
+      allConfigs.add(SolrConfig.readFromResourceLoader(srl, "solrconfig.xml", null));
     }
     assertEquals(numReads, allConfigs.size());
     System.gc();

solr/core/src/test/org/apache/solr/core/TestConfig.java

@@ -45,20 +45,6 @@ public static void beforeClass() throws Exception {
   public void testLib() throws IOException {
     SolrResourceLoader loader = h.getCore().getResourceLoader();
     InputStream data = null;
-    String[] expectedFiles =
-        new String[] {
-          "empty-file-main-lib.txt",
-          "empty-file-a1.txt",
-          "empty-file-a2.txt",
-          "empty-file-b1.txt",
-          "empty-file-b2.txt",
-          "empty-file-c1.txt"
-        };
-    for (String f : expectedFiles) {
-      data = loader.openResource(f);
-      assertNotNull("Should have found file " + f, data);
-      data.close();
-    }
     String[] unexpectedFiles = new String[] {"empty-file-c2.txt", "empty-file-d2.txt"};
     for (String f : unexpectedFiles) {
       data = null;

solr/core/src/test/org/apache/solr/core/TestMinimalConfig.java

@@ -29,8 +29,7 @@ public static void beforeClass() throws Exception {
     initCore("solrconfig-minimal.xml", "schema-minimal.xml");
   }
 
-  // Make sure the content of the lib/ core subfolder is loaded even if there is no <lib> node in
-  // the solrconfig
+  // Make sure the content of the lib/ core subfolder is loaded
   @Test
   public void testLib() throws IOException {
     SolrResourceLoader loader = h.getCore().getResourceLoader();