From 1a2afd0b3a37a31beee784ef013d3cf914005316 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 8 May 2024 09:35:48 +0200 Subject: [PATCH 001/103] address @link not working with precommit, now it does --- solr/core/src/java/org/apache/solr/core/BlobRepository.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java index b2960a67151..e16bd8a461e 100644 --- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java +++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java @@ -91,13 +91,10 @@ public BlobRepository(CoreContainer coreContainer) { this.coreContainer = coreContainer; } - // I wanted to {@link SolrCore#loadDecodeAndCacheBlob(String, Decoder)} below but precommit - // complains - /** * Returns the contents of a blob containing a ByteBuffer and increments a reference count. Please * return the same object to decrease the refcount. This is normally used for storing jar files, - * and binary raw data. If you are caching Java Objects you want to use {@code + * and binary raw data. If you are caching Java Objects you want to use {@link * SolrCore#loadDecodeAndCacheBlob(String, Decoder)} * * @param key it is a combination of blobname and version like blobName/version From 80f5c807ead39ce021c9847fbf0edf4fc3b44bd2 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 8 May 2024 11:50:29 +0200 Subject: [PATCH 002/103] commented code is confusing to reader... --- .../test/org/apache/solr/core/BlobRepositoryCloudTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java index 8fe8c34f79b..5837f76996c 100644 --- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java +++ b/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java @@ -44,7 +44,7 @@ public static void setupCluster() throws Exception { configureCluster(1) // only sharing *within* a node .addConfig("configname", TEST_PATH.resolve("resource-sharing")) .configure(); - // Thread.sleep(2000); + CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 1) .process(cluster.getSolrClient()); // test component will fail if it can't find a blob with this data by this name @@ -53,7 +53,7 @@ public static void setupCluster() throws Exception { findLiveNodeURI(), "testResource", ByteBuffer.wrap("foo,bar\nbaz,bam".getBytes(StandardCharsets.UTF_8))); - // Thread.sleep(2000); + // if these don't load we probably failed to post the blob above CollectionAdminRequest.createCollection("col1", "configname", 1, 1) .process(cluster.getSolrClient()); From e5450b8969517a6c319d781e1bd53ffc6379a41c Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 8 May 2024 11:50:42 +0200 Subject: [PATCH 003/103] address some intellj prompted warnings --- solr/core/src/java/org/apache/solr/core/BlobRepository.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java index e16bd8a461e..2a296abf3c9 100644 --- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java +++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java @@ -79,7 +79,7 @@ public class BlobRepository { private final CoreContainer coreContainer; @SuppressWarnings({"rawtypes"}) - private Map blobs = createMap(); + private final Map blobs = createMap(); // for unit tests to override @SuppressWarnings({"rawtypes"}) @@ -188,7 +188,7 @@ private ByteBuffer fetchBlobAndVerify(String key, String url, String sha512) { } public static String sha512Digest(ByteBuffer byteBuffer) { - MessageDigest digest = null; + MessageDigest digest; try { digest = MessageDigest.getInstance("SHA-512"); } catch (NoSuchAlgorithmException e) { @@ -216,7 +216,7 @@ ByteBuffer fetchFromUrl(String key, String url) { HttpClient httpClient = coreContainer.getUpdateShardHandler().getDefaultHttpClient(); HttpGet httpGet = new HttpGet(url); ByteBuffer b; - HttpResponse response = null; + HttpResponse response; HttpEntity entity = null; try { response = httpClient.execute(httpGet); From 74b395b39b5ed75a22bf8795d328264e01d0a6c7 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 8 May 2024 14:00:50 +0100 Subject: [PATCH 004/103] Remove deprecated BlobRepository We have newer approaches such as the FileStoreAPI and the PackageManager. --- .../org/apache/solr/core/BlobRepository.java | 364 ------------------ .../org/apache/solr/core/CoreContainer.java | 6 - .../java/org/apache/solr/core/SolrCore.java | 35 -- .../apache/solr/filestore/FileStoreAPI.java | 3 +- .../solr/packagemanager/PackageUtils.java | 3 +- .../packagemanager/RepositoryManager.java | 5 +- .../configsets/resource-sharing/schema.xml | 21 - .../resource-sharing/solrconfig.xml | 51 --- .../solr/core/BlobRepositoryCloudTest.java | 127 ------ .../solr/core/BlobRepositoryMockingTest.java | 191 --------- .../ResourceSharingTestComponent.java | 142 ------- .../designer/TestSchemaDesignerAPI.java | 2 +- .../org/apache/solr/common/util/Utils.java | 31 ++ 13 files changed, 36 insertions(+), 945 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/core/BlobRepository.java delete mode 100644 solr/core/src/test-files/solr/configsets/resource-sharing/schema.xml delete mode 100644 solr/core/src/test-files/solr/configsets/resource-sharing/solrconfig.xml delete mode 100644 solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java delete mode 100644 solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java delete mode 100644 solr/core/src/test/org/apache/solr/handler/component/ResourceSharingTestComponent.java diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java deleted file mode 100644 index 2a296abf3c9..00000000000 --- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.core; - -import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; -import static org.apache.solr.common.SolrException.ErrorCode.SERVICE_UNAVAILABLE; - -import java.io.InputStream; -import java.lang.invoke.MethodHandles; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; -import org.apache.zookeeper.server.ByteBufferInputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The purpose of this class is to store the Jars loaded in memory and to keep only one copy of the - * Jar in a single node. - */ -public class BlobRepository { - - private static final long MAX_JAR_SIZE = - Long.parseLong(System.getProperty("runtime.lib.size", String.valueOf(5 * 1024 * 1024))); - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final Random RANDOM; - static final Pattern BLOB_KEY_PATTERN_CHECKER = Pattern.compile(".*/\\d+"); - - static { - // We try to make things reproducible in the context of our tests by initializing the random - // instance based on the current seed - String seed = System.getProperty("tests.seed"); - if (seed == null) { - RANDOM = new Random(); - } else { - RANDOM = new Random(seed.hashCode()); - } - } - - private final CoreContainer coreContainer; - - @SuppressWarnings({"rawtypes"}) - private final Map blobs = createMap(); - - // for unit tests to override - @SuppressWarnings({"rawtypes"}) - ConcurrentHashMap createMap() { - return new ConcurrentHashMap<>(); - } - - public BlobRepository(CoreContainer coreContainer) { - this.coreContainer = coreContainer; - } - - /** - * Returns the contents of a blob containing a ByteBuffer and increments a reference count. Please - * return the same object to decrease the refcount. This is normally used for storing jar files, - * and binary raw data. If you are caching Java Objects you want to use {@link - * SolrCore#loadDecodeAndCacheBlob(String, Decoder)} - * - * @param key it is a combination of blobname and version like blobName/version - * @return The reference of a blob - */ - public BlobContentRef getBlobIncRef(String key) { - return getBlobIncRef(key, () -> addBlob(key)); - } - - /** - * Internal method that returns the contents of a blob and increments a reference count. Please - * return the same object to decrease the refcount. Only the decoded content will be cached when - * this method is used. Component authors attempting to share objects across cores should use - * {@code SolrCore#loadDecodeAndCacheBlob(String, Decoder)} which ensures that a proper close hook - * is also created. - * - * @param key it is a combination of blob name and version like blobName/version - * @param decoder a decoder that knows how to interpret the bytes from the blob - * @return The reference of a blob - */ - BlobContentRef getBlobIncRef(String key, Decoder decoder) { - return getBlobIncRef(key.concat(decoder.getName()), () -> addBlob(key, decoder)); - } - - BlobContentRef getBlobIncRef(String key, Decoder decoder, String url, String sha512) { - StringBuilder keyBuilder = new StringBuilder(key); - if (decoder != null) keyBuilder.append(decoder.getName()); - keyBuilder.append("/").append(sha512); - - return getBlobIncRef( - keyBuilder.toString(), - () -> new BlobContent<>(key, fetchBlobAndVerify(key, url, sha512), decoder)); - } - - // do the actual work returning the appropriate type... - @SuppressWarnings({"unchecked"}) - private BlobContentRef getBlobIncRef(String key, Callable> blobCreator) { - BlobContent aBlob; - if (this.coreContainer.isZooKeeperAware()) { - synchronized (blobs) { - aBlob = blobs.get(key); - if (aBlob == null) { - try { - aBlob = blobCreator.call(); - } catch (Exception e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "Blob loading failed: " + e.getMessage(), e); - } - } - } - } else { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "Blob loading is not supported in non-cloud mode"); - // todo - } - BlobContentRef ref = new BlobContentRef<>(aBlob); - synchronized (aBlob.references) { - aBlob.references.add(ref); - } - return ref; - } - - // For use cases sharing raw bytes - private BlobContent addBlob(String key) { - ByteBuffer b = fetchBlob(key); - BlobContent aBlob = new BlobContent<>(key, b); - blobs.put(key, aBlob); - return aBlob; - } - - // for use cases sharing java objects - private BlobContent addBlob(String key, Decoder decoder) { - ByteBuffer b = fetchBlob(key); - String keyPlusName = key + decoder.getName(); - BlobContent aBlob = new BlobContent<>(keyPlusName, b, decoder); - blobs.put(keyPlusName, aBlob); - return aBlob; - } - - static String INVALID_JAR_MSG = - "Invalid jar from {0} , expected sha512 hash : {1} , actual : {2}"; - - private ByteBuffer fetchBlobAndVerify(String key, String url, String sha512) { - ByteBuffer byteBuffer = fetchFromUrl(key, url); - String computedDigest = sha512Digest(byteBuffer); - if (!computedDigest.equals(sha512)) { - throw new SolrException( - SERVER_ERROR, StrUtils.formatString(INVALID_JAR_MSG, url, sha512, computedDigest)); - } - return byteBuffer; - } - - public static String sha512Digest(ByteBuffer byteBuffer) { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-512"); - } catch (NoSuchAlgorithmException e) { - // unlikely - throw new SolrException(SERVER_ERROR, e); - } - digest.update(byteBuffer); - return String.format(Locale.ROOT, "%0128x", new BigInteger(1, digest.digest())); - } - - /** Package local for unit tests only please do not use elsewhere */ - ByteBuffer fetchBlob(String key) { - Replica replica = getSystemCollReplica(); - String url = - replica.getBaseUrl() - + "/" - + CollectionAdminParams.SYSTEM_COLL - + "/blob/" - + key - + "?wt=filestream"; - return fetchFromUrl(key, url); - } - - ByteBuffer fetchFromUrl(String key, String url) { - HttpClient httpClient = coreContainer.getUpdateShardHandler().getDefaultHttpClient(); - HttpGet httpGet = new HttpGet(url); - ByteBuffer b; - HttpResponse response; - HttpEntity entity = null; - try { - response = httpClient.execute(httpGet); - entity = response.getEntity(); - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode != 200) { - throw new SolrException( - SolrException.ErrorCode.NOT_FOUND, "no such blob or version available: " + key); - } - - try (InputStream is = entity.getContent()) { - b = Utils.toByteArray(is, MAX_JAR_SIZE); - } - } catch (Exception e) { - if (e instanceof SolrException) { - throw (SolrException) e; - } else { - throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "could not load : " + key, e); - } - } finally { - Utils.consumeFully(entity); - } - return b; - } - - private Replica getSystemCollReplica() { - ZkStateReader zkStateReader = this.coreContainer.getZkController().getZkStateReader(); - ClusterState cs = zkStateReader.getClusterState(); - DocCollection coll = cs.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL); - if (coll == null) - throw new SolrException( - SERVICE_UNAVAILABLE, CollectionAdminParams.SYSTEM_COLL + " collection not available"); - ArrayList slices = new ArrayList<>(coll.getActiveSlices()); - if (slices.isEmpty()) - throw new SolrException( - SERVICE_UNAVAILABLE, - "No active slices for " + CollectionAdminParams.SYSTEM_COLL + " collection"); - Collections.shuffle(slices, RANDOM); // do load balancing - - Replica replica = null; - for (Slice slice : slices) { - List replicas = new ArrayList<>(slice.getReplicasMap().values()); - Collections.shuffle(replicas, RANDOM); - for (Replica r : replicas) { - if (r.getState() == Replica.State.ACTIVE) { - if (zkStateReader - .getClusterState() - .getLiveNodes() - .contains(r.get(ZkStateReader.NODE_NAME_PROP))) { - replica = r; - break; - } else { - if (log.isInfoEnabled()) { - log.info( - "replica {} says it is active but not a member of live nodes", - r.get(ZkStateReader.NODE_NAME_PROP)); - } - } - } - } - } - if (replica == null) { - throw new SolrException( - SERVICE_UNAVAILABLE, - "No active replica available for " + CollectionAdminParams.SYSTEM_COLL + " collection"); - } - return replica; - } - - /** - * This is to decrement a ref count - * - * @param ref The reference that is already there. Doing multiple calls with same ref will not - * matter - */ - public void decrementBlobRefCount(BlobContentRef ref) { - if (ref == null) return; - synchronized (ref.blob.references) { - if (!ref.blob.references.remove(ref)) { - log.error("Multiple releases for the same reference"); - } - if (ref.blob.references.isEmpty()) { - blobs.remove(ref.blob.key); - } - } - } - - public static class BlobContent { - public final String key; - // holds byte buffer or cached object, holding both is a waste of memory ref counting mechanism - private final T content; - private final Set> references = new HashSet<>(); - - @SuppressWarnings("unchecked") - public BlobContent(String key, ByteBuffer buffer, Decoder decoder) { - this.key = key; - this.content = - decoder == null ? (T) buffer : decoder.decode(new ByteBufferInputStream(buffer)); - } - - @SuppressWarnings("unchecked") - public BlobContent(String key, ByteBuffer buffer) { - this.key = key; - this.content = (T) buffer; - } - - /** - * Get the cached object. - * - * @return the object representing the content that is cached. - */ - public T get() { - return this.content; - } - } - - public interface Decoder { - - /** - * A name by which to distinguish this decoding. This only needs to be implemented if you want - * to support decoding the same blob content with more than one decoder. - * - * @return The name of the decoding, defaults to empty string. - */ - default String getName() { - return ""; - } - - /** - * A routine that knows how to convert the stream of bytes from the blob into a Java object. - * - * @param inputStream the bytes from a blob - * @return A Java object of the specified type. - */ - T decode(InputStream inputStream); - } - - public static class BlobContentRef { - public final BlobContent blob; - - private BlobContentRef(BlobContent blob) { - this.blob = blob; - } - } -} diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 8f3ee608011..a47b0f9b544 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -246,8 +246,6 @@ public JerseyAppHandlerCache getJerseyAppHandlerCache() { private volatile String hostName; - private final BlobRepository blobRepository = new BlobRepository(this); - private volatile boolean asyncSolrCoreLoad; protected volatile SecurityConfHandler securityConfHandler; @@ -2304,10 +2302,6 @@ public SolrCore getCore(String name, UUID id) { return core; } - public BlobRepository getBlobRepository() { - return blobRepository; - } - /** * If using asyncSolrCoreLoad=true, calling this after {@link #load()} will not return until all * cores have finished loading. diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 2b1bd65f292..92a963f6bbe 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -89,7 +89,6 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CommonParams.EchoParamStyle; import org.apache.solr.common.params.SolrParams; @@ -3540,40 +3539,6 @@ public List getImplicitHandlers() { return ImplicitHolder.INSTANCE; } - /** - * Convenience method to load a blob. This method minimizes the degree to which component and - * other code needs to depend on the structure of solr's object graph and ensures that a proper - * close hook is registered. This method should normally be called in {@link - * SolrCoreAware#inform(SolrCore)}, and should never be called during request processing. The - * Decoder will only run on the first invocations, subsequent invocations will return the cached - * object. - * - * @param key A key in the format of name/version for a blob stored in the {@link - * CollectionAdminParams#SYSTEM_COLL} blob store via the Blob Store API - * @param decoder a decoder with which to convert the blob into a Java Object representation - * (first time only) - * @return a reference to the blob that has already cached the decoded version. - */ - public BlobRepository.BlobContentRef loadDecodeAndCacheBlob( - String key, BlobRepository.Decoder decoder) { - // make sure component authors don't give us oddball keys with no version... - if (!BlobRepository.BLOB_KEY_PATTERN_CHECKER.matcher(key).matches()) { - throw new IllegalArgumentException( - "invalid key format, must end in /N where N is the version number"); - } - // define the blob - BlobRepository.BlobContentRef blobRef = - coreContainer.getBlobRepository().getBlobIncRef(key, decoder); - addCloseHook( - new CloseHook() { - @Override - public void postClose(SolrCore core) { - coreContainer.getBlobRepository().decrementBlobRefCount(blobRef); - } - }); - return blobRef; - } - public CancellableQueryTracker getCancellableQueryTracker() { return cancellableQueryTracker; } diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java index 52dee0f5e9a..dc420d37b31 100644 --- a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java @@ -45,7 +45,6 @@ import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; -import org.apache.solr.core.BlobRepository; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.pkg.PackageAPI; @@ -84,7 +83,7 @@ public ArrayList shuffledNodes() { coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes(); ArrayList l = new ArrayList<>(liveNodes); l.remove(coreContainer.getZkController().getNodeName()); - Collections.shuffle(l, BlobRepository.RANDOM); + Collections.shuffle(l, Utils.RANDOM); return l; } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java index d6e765b0d4a..c0d50dd85e2 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java @@ -48,7 +48,6 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; -import org.apache.solr.core.BlobRepository; import org.apache.solr.filestore.DistribFileStore; import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.packagemanager.SolrPackage.Manifest; @@ -209,7 +208,7 @@ public static Manifest fetchManifest( NamedList response = solrClient.request(request); String manifestJson = (String) response.get("response"); String calculatedSHA512 = - BlobRepository.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8))); + Utils.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8))); if (expectedSHA512.equals(calculatedSHA512) == false) { throw new SolrException( ErrorCode.UNAUTHORIZED, diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 109468f854c..d92d027d1c0 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -50,7 +50,7 @@ import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.BlobRepository; +import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; @@ -193,8 +193,7 @@ private boolean installPackage(String packageName, String version) throws SolrEx } String manifestJson = getMapper().writeValueAsString(release.manifest); String manifestSHA512 = - BlobRepository.sha512Digest( - ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8))); + Utils.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8))); PackageUtils.postFile( solrClient, ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)), diff --git a/solr/core/src/test-files/solr/configsets/resource-sharing/schema.xml b/solr/core/src/test-files/solr/configsets/resource-sharing/schema.xml deleted file mode 100644 index 287d4fe0149..00000000000 --- a/solr/core/src/test-files/solr/configsets/resource-sharing/schema.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - diff --git a/solr/core/src/test-files/solr/configsets/resource-sharing/solrconfig.xml b/solr/core/src/test-files/solr/configsets/resource-sharing/solrconfig.xml deleted file mode 100644 index 1dd92feef2e..00000000000 --- a/solr/core/src/test-files/solr/configsets/resource-sharing/solrconfig.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - ${solr.data.dir:} - - - - - ${tests.luceneMatchVersion:LATEST} - - - - ${solr.commitwithin.softcommit:true} - - - - - - - - explicit - true - text - - - testComponent - - - - diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java deleted file mode 100644 index 5837f76996c..00000000000 --- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.core; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.handler.TestBlobHandler; -import org.junit.BeforeClass; -import org.junit.Test; - -public class BlobRepositoryCloudTest extends SolrCloudTestCase { - - public static final Path TEST_PATH = getFile("solr/configsets").toPath(); - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(1) // only sharing *within* a node - .addConfig("configname", TEST_PATH.resolve("resource-sharing")) - .configure(); - - CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 1) - .process(cluster.getSolrClient()); - // test component will fail if it can't find a blob with this data by this name - TestBlobHandler.postData( - cluster.getSolrClient(), - findLiveNodeURI(), - "testResource", - ByteBuffer.wrap("foo,bar\nbaz,bam".getBytes(StandardCharsets.UTF_8))); - - // if these don't load we probably failed to post the blob above - CollectionAdminRequest.createCollection("col1", "configname", 1, 1) - .process(cluster.getSolrClient()); - CollectionAdminRequest.createCollection("col2", "configname", 1, 1) - .process(cluster.getSolrClient()); - - SolrInputDocument document = new SolrInputDocument(); - document.addField("id", "1"); - document.addField("text", "col1"); - CloudSolrClient solrClient = cluster.getSolrClient(); - solrClient.add("col1", document); - solrClient.commit("col1"); - document = new SolrInputDocument(); - document.addField("id", "1"); - document.addField("text", "col2"); - solrClient.add("col2", document); - solrClient.commit("col2"); - Thread.sleep(2000); - } - - @Test - public void test() throws Exception { - // This test relies on the installation of ResourceSharingTestComponent which has 2 useful - // properties: - // 1. it will fail to initialize if it doesn't find a 2 line CSV like foo,bar\nbaz,bam thus - // validating that we are properly pulling data from the blob store - // 2. It replaces any q for a query request to /select with "text:" where is - // the name of the last collection to run a query. It does this by caching a shared resource of - // type ResourceSharingTestComponent.TestObject, and the following sequence is proof that either - // collection can tell if it was (or was not) the last collection to issue a query by consulting - // the shared object - assertLastQueryNotToCollection("col1"); - assertLastQueryNotToCollection("col2"); - assertLastQueryNotToCollection("col1"); - assertLastQueryToCollection("col1"); - assertLastQueryNotToCollection("col2"); - assertLastQueryToCollection("col2"); - } - - // TODO: move this up to parent class? - private static String findLiveNodeURI() { - ZkStateReader zkStateReader = cluster.getZkStateReader(); - return zkStateReader.getBaseUrlForNodeName( - zkStateReader - .getClusterState() - .getCollection(".system") - .getSlices() - .iterator() - .next() - .getLeader() - .getNodeName()); - } - - private void assertLastQueryToCollection(String collection) - throws SolrServerException, IOException { - assertEquals(1, getSolrDocuments(collection).size()); - } - - private void assertLastQueryNotToCollection(String collection) - throws SolrServerException, IOException { - assertEquals(0, getSolrDocuments(collection).size()); - } - - private SolrDocumentList getSolrDocuments(String collection) - throws SolrServerException, IOException { - SolrQuery query = new SolrQuery("*:*"); - CloudSolrClient client = cluster.getSolrClient(); - QueryResponse resp1 = client.query(collection, query); - return resp1.getResults(); - } -} diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java deleted file mode 100644 index cf37174bbc3..00000000000 --- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.core; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.StringWriter; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.SolrException; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class BlobRepositoryMockingTest extends SolrTestCaseJ4 { - - private static final Charset UTF8 = StandardCharsets.UTF_8; - private static final String[][] PARSED = - new String[][] {{"foo", "bar", "baz"}, {"bang", "boom", "bash"}}; - private static final String BLOBSTR = "foo,bar,baz\nbang,boom,bash"; - private CoreContainer mockContainer = mock(CoreContainer.class); - - @SuppressWarnings({"unchecked", "rawtypes"}) - private ConcurrentHashMap blobStorage; - - BlobRepository repository; - ByteBuffer blobData = ByteBuffer.wrap(BLOBSTR.getBytes(UTF8)); - boolean blobFetched = false; - String blobKey = ""; - String url = null; - ByteBuffer filecontent = null; - - @BeforeClass - public static void beforeClass() { - SolrTestCaseJ4.assumeWorkingMockito(); - } - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - blobFetched = false; - blobKey = ""; - reset(mockContainer); - blobStorage = new ConcurrentHashMap<>(); - repository = - new BlobRepository(mockContainer) { - @Override - ByteBuffer fetchBlob(String key) { - blobKey = key; - blobFetched = true; - return blobData; - } - - @Override - ByteBuffer fetchFromUrl(String key, String url) { - if (!Objects.equals(url, BlobRepositoryMockingTest.this.url)) return null; - blobKey = key; - blobFetched = true; - return filecontent; - } - - @Override - @SuppressWarnings({"rawtypes"}) - ConcurrentHashMap createMap() { - return blobStorage; - } - }; - } - - @Test(expected = SolrException.class) - public void testCloudOnly() { - when(mockContainer.isZooKeeperAware()).thenReturn(false); - try { - repository.getBlobIncRef("foo!"); - } catch (SolrException e) { - verify(mockContainer).isZooKeeperAware(); - throw e; - } - } - - @Test - public void testGetBlobIncrRefString() { - when(mockContainer.isZooKeeperAware()).thenReturn(true); - BlobRepository.BlobContentRef ref = repository.getBlobIncRef("foo!"); - assertEquals("foo!", blobKey); - assertTrue(blobFetched); - assertNotNull(ref.blob); - assertEquals(blobData, ref.blob.get()); - verify(mockContainer).isZooKeeperAware(); - assertNotNull(blobStorage.get("foo!")); - } - - @Test - public void testGetBlobIncrRefByUrl() throws Exception { - when(mockContainer.isZooKeeperAware()).thenReturn(true); - filecontent = TestSolrConfigHandler.getFileContent("runtimecode/runtimelibs_v2.jar.bin"); - url = "http://localhost:8080/myjar/location.jar"; - BlobRepository.BlobContentRef ref = - repository.getBlobIncRef( - "filefoo", - null, - url, - "bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417"); - assertEquals("filefoo", blobKey); - assertTrue(blobFetched); - assertNotNull(ref.blob); - assertEquals(filecontent, ref.blob.get()); - verify(mockContainer).isZooKeeperAware(); - try { - repository.getBlobIncRef("filefoo", null, url, "WRONG-SHA512-KEY"); - fail("expected exception"); - } catch (Exception e) { - assertTrue(e.getMessage().contains(" expected sha512 hash : WRONG-SHA512-KEY , actual :")); - } - - url = null; - filecontent = null; - } - - @Test - public void testCachedAlready() { - when(mockContainer.isZooKeeperAware()).thenReturn(true); - blobStorage.put("foo!", new BlobRepository.BlobContent("foo!", blobData)); - BlobRepository.BlobContentRef ref = repository.getBlobIncRef("foo!"); - assertEquals("", blobKey); - assertFalse(blobFetched); - assertNotNull(ref.blob); - assertEquals(blobData, ref.blob.get()); - verify(mockContainer).isZooKeeperAware(); - assertNotNull("Key was not mapped to a BlobContent instance.", blobStorage.get("foo!")); - } - - @Test - public void testGetBlobIncrRefStringDecoder() { - when(mockContainer.isZooKeeperAware()).thenReturn(true); - BlobRepository.BlobContentRef ref = - repository.getBlobIncRef( - "foo!", - new BlobRepository.Decoder<>() { - @Override - public String[][] decode(InputStream inputStream) { - StringWriter writer = new StringWriter(); - try { - new InputStreamReader(inputStream, UTF8).transferTo(writer); - } catch (IOException e) { - throw new RuntimeException(e); - } - - assertEquals(BLOBSTR, writer.toString()); - return PARSED; - } - - @Override - public String getName() { - return "mocked"; - } - }); - assertEquals("foo!", blobKey); - assertTrue(blobFetched); - assertNotNull(ref.blob); - assertEquals(PARSED, ref.blob.get()); - verify(mockContainer).isZooKeeperAware(); - assertNotNull(blobStorage.get("foo!mocked")); - } -} diff --git a/solr/core/src/test/org/apache/solr/handler/component/ResourceSharingTestComponent.java b/solr/core/src/test/org/apache/solr/handler/component/ResourceSharingTestComponent.java deleted file mode 100644 index e653b1f6de3..00000000000 --- a/solr/core/src/test/org/apache/solr/handler/component/ResourceSharingTestComponent.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.handler.component; - -import static org.junit.Assert.assertEquals; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Stream; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.core.BlobRepository; -import org.apache.solr.core.SolrCore; -import org.apache.solr.util.plugin.SolrCoreAware; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ResourceSharingTestComponent extends SearchComponent implements SolrCoreAware { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private SolrCore core; - private volatile BlobRepository.BlobContent blob; - - @SuppressWarnings("SynchronizeOnNonFinalField") - @Override - public void prepare(ResponseBuilder rb) { - SolrParams params = rb.req.getParams(); - ModifiableSolrParams mParams = new ModifiableSolrParams(params); - String q = "text:" + getTestObj().getLastCollection(); - mParams.set("q", q); // search for the last collection name. - // This should cause the param to show up in the response... - rb.req.setParams(mParams); - getTestObj().setLastCollection(core.getCoreDescriptor().getCollectionName()); - } - - @Override - public void process(ResponseBuilder rb) {} - - @Override - public String getDescription() { - return "ResourceSharingTestComponent"; - } - - TestObject getTestObj() { - return this.blob.get(); - } - - @SuppressWarnings("unchecked") - @Override - public void inform(SolrCore core) { - log.info("Informing test component..."); - this.core = core; - this.blob = core.loadDecodeAndCacheBlob(getKey(), new DumbCsvDecoder()).blob; - log.info("Test component informed!"); - } - - private String getKey() { - return getResourceName() + "/" + getResourceVersion(); - } - - public String getResourceName() { - return "testResource"; - } - - public String getResourceVersion() { - return "1"; - } - - class DumbCsvDecoder implements BlobRepository.Decoder { - private final Map dict = new HashMap<>(); - - public DumbCsvDecoder() {} - - void processSimpleCsvRow(String string) { - String[] row = string.split(","); // dumbest csv parser ever... :) - getDict().put(row[0], row[1]); - } - - public Map getDict() { - return dict; - } - - @Override - public TestObject decode(InputStream inputStream) { - // loading a tiny csv like: - // - // foo,bar - // baz,bam - - try (Stream lines = - new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)).lines()) { - lines.forEach(this::processSimpleCsvRow); - } catch (Exception e) { - log.error("failed to read dictionary {}", getResourceName()); - throw new RuntimeException("Cannot load dictionary ", e); - } - - assertEquals("bar", dict.get("foo")); - assertEquals("bam", dict.get("baz")); - if (log.isInfoEnabled()) { - log.info("Loaded {} using {}", getDict().size(), this.getClass().getClassLoader()); - } - - // if we get here we have seen the data from the blob and all we need is to test that two - // collections are able to see the same object... - return new TestObject(); - } - } - - public static class TestObject { - public static final String NEVER_UPDATED = "never updated"; - private volatile String lastCollection = NEVER_UPDATED; - - public String getLastCollection() { - return this.lastCollection; - } - - public void setLastCollection(String lastCollection) { - this.lastCollection = lastCollection; - } - } -} diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java index 57d5a6b47a8..90a7685d83e 100644 --- a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java +++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java @@ -70,7 +70,7 @@ public static void createCluster() throws Exception { configureCluster(1) .addConfig(DEFAULT_CONFIGSET_NAME, new File(ExternalPaths.DEFAULT_CONFIGSET).toPath()) .configure(); - // SchemaDesignerAPI depends on the blob store + // SchemaDesignerAPI depends on the blob store ".system" collection existing. CollectionAdminRequest.createCollection(BLOB_STORE_ID, 1, 1).process(cluster.getSolrClient()); cluster.waitForActiveCollection(BLOB_STORE_ID, 1, 1); } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java index 4a1b39b378c..018151afbce 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java @@ -19,6 +19,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.singletonList; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; import com.fasterxml.jackson.annotation.JsonAnyGetter; import java.io.ByteArrayInputStream; @@ -38,10 +39,13 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.math.BigInteger; import java.net.URL; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; @@ -52,8 +56,10 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Random; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -90,6 +96,31 @@ public class Utils { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final Random RANDOM; + + static { + // We try to make things reproducible in the context of our tests by initializing the random + // instance based on the current seed + String seed = System.getProperty("tests.seed"); + if (seed == null) { + RANDOM = new Random(); + } else { + RANDOM = new Random(seed.hashCode()); + } + } + + public static String sha512Digest(ByteBuffer byteBuffer) { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-512"); + } catch (NoSuchAlgorithmException e) { + // unlikely + throw new SolrException(SERVER_ERROR, e); + } + digest.update(byteBuffer); + return String.format(Locale.ROOT, "%0128x", new BigInteger(1, digest.digest())); + } + @SuppressWarnings({"rawtypes"}) public static Map getDeepCopy(Map map, int maxDepth) { return getDeepCopy(map, maxDepth, true, false); From 79863b459ace1590f6c5311ddf246ae0ebd92311 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 8 May 2024 14:12:05 +0100 Subject: [PATCH 005/103] checkpoint --- .../solr/handler/component/UBIComponent.java | 114 ++++++++++++++++++ .../conf/solrconfig-ubi-component.xml | 61 ++++++++++ .../handler/component/UBIComponentTest.java | 93 ++++++++++++++ solr/server/resources/log4j2.xml | 20 ++- 4 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-component.xml create mode 100644 solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java new file mode 100644 index 00000000000..f4b1722e632 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.component; + +import java.io.IOException; +import java.util.Collections; +import java.util.Set; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.response.ResultContext; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrIndexSearcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Inspired by the ResponseLogComponent. + * + *

Adds to the .ubi_queries system collection the original user query and the document IDs that + * are sent in the query response. + * + *

Tracks the collection name, the end user query, as json blob, and the resulting document id's. + * + *

Add it to a requestHandler in solrconfig.xml like this: + * + * 

+ * <searchComponent name="ubi" class="solr.UBIComponent"/>
+ *
+ * <requestHandler name="/select" class="solr.SearchHandler">
+ *   <lst name="defaults">
+ *
+ *     ...
+ *
+ *   </lst>
+ *   <arr name="components">
+ *     <str>ubi</str>
+ *   </arr>
+ * </requestHandler>
+ * + * It can then be enabled at query time by supplying + * + * 
ubi=true
+ * + * query parameter. + */ +public class UBIComponent extends SearchComponent { + + public static final String COMPONENT_NAME = "ubi"; + + private static final Logger ubiRequestLogger = + LoggerFactory.getLogger(SolrCore.class.getName() + ".UBIRequest"); + + @Override + public void prepare(ResponseBuilder rb) throws IOException {} + + @Override + public void process(ResponseBuilder rb) throws IOException { + SolrParams params = rb.req.getParams(); + if (!params.getBool(COMPONENT_NAME, false)) { + return; + } + + SolrIndexSearcher searcher = rb.req.getSearcher(); + IndexSchema schema = searcher.getSchema(); + if (schema.getUniqueKeyField() == null) { + return; + } + + ResultContext rc = (ResultContext) rb.rsp.getResponse(); + + DocList docs = rc.getDocList(); + + processIds(rb, docs, schema, searcher); + } + + protected void processIds( + ResponseBuilder rb, DocList dl, IndexSchema schema, SolrIndexSearcher searcher) + throws IOException { + + StringBuilder sb = new StringBuilder(); + + Set fields = Collections.singleton(schema.getUniqueKeyField().getName()); + for (DocIterator iter = dl.iterator(); iter.hasNext(); ) { + + sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(','); + } + String docIds = sb.substring(0, sb.length() - 1); + // if (sb.length() > 0) { + rb.rsp.addToLog("ubi", docIds); + // } + ubiRequestLogger.info("docIds: {}", docIds); + } + + @Override + public String getDescription() { + return "A component that inserts the retrieved documents into the response log entry"; + } +} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-component.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-component.xml new file mode 100644 index 00000000000..deedac7f878 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-component.xml @@ -0,0 +1,61 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + ${solr.data.dir:} + + + + + + + + + + + dismax + + + ubi + + + + + + dismax + + + + + + text + + + + diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java new file mode 100644 index 00000000000..5a9f6f729b4 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.component; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.BeforeClass; +import org.junit.Test; + +public class UBIComponentTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeTest() throws Exception { + initCore("solrconfig-ubi-component.xml", "schema12.xml"); + assertNull(h.validateUpdate(adoc("id", "1", "subject", "aa"))); + assertNull(h.validateUpdate(adoc("id", "two", "subject", "aa"))); + assertNull(h.validateUpdate(adoc("id", "3", "subject", "aa"))); + assertU(commit()); + } + + @Test + public void testToLogIds() throws Exception { + SolrQueryRequest req = null; + try { + String handler = "/withubi"; + req = + req( + "indent", + "true", + "qt", + "/withubi", + "q", + "aa", + "rows", + "2", + "fl", + "id,subject", + "ubi", + "true"); + SolrQueryResponse qr = h.queryAndResponse(handler, req); + NamedList entries = qr.getToLog(); + String docIds = (String) entries.get("ubi"); + assertNotNull(docIds); + assertTrue(docIds.matches("\\w+,\\w+")); + } finally { + req.close(); + } + } + + @Test + public void testDisabling() throws Exception { + SolrQueryRequest req = null; + try { + String handler = "/withubi"; + req = + req( + "indent", + "true", + "qt", + "/withubi", + "q", + "aa", + "rows", + "2", + "fl", + "id,subject", + "ubi", + "false"); + SolrQueryResponse qr = h.queryAndResponse(handler, req); + NamedList entries = qr.getToLog(); + String responseLog = (String) entries.get("responseLog"); + assertNull(responseLog); + } finally { + req.close(); + } + } +} diff --git a/solr/server/resources/log4j2.xml b/solr/server/resources/log4j2.xml index 006de0c965c..8d1148cfbc5 100644 --- a/solr/server/resources/log4j2.xml +++ b/solr/server/resources/log4j2.xml @@ -59,6 +59,22 @@ + + + + + %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n + + + + + + + + @@ -74,6 +90,9 @@ + + + @@ -84,4 +103,3 @@ - From 4fbfa9d764ab31554e1c5edf6d633fc0d2568ef5 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 8 May 2024 17:41:18 -0400 Subject: [PATCH 006/103] better name, but I don't have the output pattern working yet --- solr/server/resources/log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/server/resources/log4j2.xml b/solr/server/resources/log4j2.xml index 8d1148cfbc5..e94a2c68bb9 100644 --- a/solr/server/resources/log4j2.xml +++ b/solr/server/resources/log4j2.xml @@ -62,8 +62,8 @@ + fileName="${sys:solr.log.dir}/solr_ubi_queries.log" + filePattern="${sys:solr.log.dir}/solr_ubi_queries.log.%i" > %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n From 80123eb1819cb264cd4d960e3dde444608fb7cc3 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 8 May 2024 17:41:41 -0400 Subject: [PATCH 007/103] write to console out the docids, and add a hard coded query_id --- .../solr/handler/component/UBIComponent.java | 12 ++- .../handler/component/UBIComponentTest.java | 94 ++++++++++--------- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index f4b1722e632..b8acd689b95 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.Set; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.response.ResultContext; import org.apache.solr.schema.IndexSchema; @@ -100,11 +101,14 @@ protected void processIds( sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(','); } - String docIds = sb.substring(0, sb.length() - 1); - // if (sb.length() > 0) { - rb.rsp.addToLog("ubi", docIds); - // } + String docIds = sb.length() > 0 ? sb.substring(0, sb.length() - 1) : ""; + + ubiRequestLogger.error("bob dole"); ubiRequestLogger.info("docIds: {}", docIds); + System.out.println(" docIds:" + docIds); + NamedList ubiInfo = new NamedList<>(); + ubiInfo.add("query_id", "1234"); // change to generateing + rb.rsp.add("ubi", ubiInfo); } @Override diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java index 5a9f6f729b4..6cb7a2a0588 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java @@ -17,9 +17,7 @@ package org.apache.solr.handler.component; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; import org.junit.BeforeClass; import org.junit.Test; @@ -39,55 +37,61 @@ public void testToLogIds() throws Exception { SolrQueryRequest req = null; try { String handler = "/withubi"; - req = - req( - "indent", - "true", - "qt", - "/withubi", - "q", - "aa", - "rows", - "2", - "fl", - "id,subject", - "ubi", - "true"); - SolrQueryResponse qr = h.queryAndResponse(handler, req); - NamedList entries = qr.getToLog(); - String docIds = (String) entries.get("ubi"); - assertNotNull(docIds); - assertTrue(docIds.matches("\\w+,\\w+")); + req = req("qt", "/withubi", "q", "aa", "rows", "2", "ubi", "true"); + + assertQ( + "Make sure we generate a query id", + req, + "//lst[@name='ubi']/str[@name='query_id'][.='1234']"); + // Need to test the writing out to the logs.. + // SolrQueryResponse qr = h.queryAndResponse(handler, req); + // NamedList entries = qr.getToLog(); + // String docIds = (String) entries.get("ubi"); + // assertNotNull(docIds); + // assertTrue(docIds.matches("\\w+,\\w+")); } finally { req.close(); } } + @Test + public void testZeroResults() throws Exception { + // SolrQueryRequest req = null; + // try { + // String handler = "/withubi"; + // req = req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true"); + + assertQ( + "Make sure we generate a query id", + req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true"), + "//lst[@name='ubi']/str[@name='query_id'][.='1234']"); + + // SolrQueryResponse qr = h.queryAndResponse(handler, req); + // NamedList entries = qr.getToLog(); + // String docIds = (String) entries.get("ubi"); + // assertNull(docIds); + // } finally { + // req.close(); + // } + } + @Test public void testDisabling() throws Exception { - SolrQueryRequest req = null; - try { - String handler = "/withubi"; - req = - req( - "indent", - "true", - "qt", - "/withubi", - "q", - "aa", - "rows", - "2", - "fl", - "id,subject", - "ubi", - "false"); - SolrQueryResponse qr = h.queryAndResponse(handler, req); - NamedList entries = qr.getToLog(); - String responseLog = (String) entries.get("responseLog"); - assertNull(responseLog); - } finally { - req.close(); - } + // SolrQueryRequest req = null; + // try { + // String handler = "/withubi"; + + assertQ( + "Make sure we don't generate a query_id", + req("qt", "/withubi", "q", "aa", "ubi", "false"), + "count(//lst[@name='ubi'])=0"); + + // SolrQueryResponse qr = h.queryAndResponse(handler, req); + // NamedList entries = qr.getToLog(); + // String ubi = (String) entries.get("ubi"); + // assertNull(ubi); + // } finally { + // req.close(); + // } } } From f6cb356a2b2e6be3a04f55a146a51fa67e8e24dc Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 8 May 2024 18:44:06 -0400 Subject: [PATCH 008/103] now handling passing in query_id instead of internally generated, and testing works with json query --- .../solr/handler/component/UBIComponent.java | 21 +++-- .../handler/component/UBIComponentTest.java | 22 +++++ solr/packaging/test/test_ubi.bats | 83 +++++++++++++++++++ 3 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 solr/packaging/test/test_ubi.bats diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index b8acd689b95..3ffed420fb4 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -20,7 +20,7 @@ import java.util.Collections; import java.util.Set; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.response.ResultContext; import org.apache.solr.schema.IndexSchema; @@ -63,6 +63,7 @@ public class UBIComponent extends SearchComponent { public static final String COMPONENT_NAME = "ubi"; + public static final String QUERY_ID = "query_id"; private static final Logger ubiRequestLogger = LoggerFactory.getLogger(SolrCore.class.getName() + ".UBIRequest"); @@ -83,15 +84,25 @@ public void process(ResponseBuilder rb) throws IOException { return; } + String queryId = params.get(QUERY_ID, null); + + if (queryId == null) { + queryId = "1234"; + } + ResultContext rc = (ResultContext) rb.rsp.getResponse(); DocList docs = rc.getDocList(); - processIds(rb, docs, schema, searcher); + processIds(rb, docs, queryId, schema, searcher); } protected void processIds( - ResponseBuilder rb, DocList dl, IndexSchema schema, SolrIndexSearcher searcher) + ResponseBuilder rb, + DocList dl, + String queryId, + IndexSchema schema, + SolrIndexSearcher searcher) throws IOException { StringBuilder sb = new StringBuilder(); @@ -106,8 +117,8 @@ protected void processIds( ubiRequestLogger.error("bob dole"); ubiRequestLogger.info("docIds: {}", docIds); System.out.println(" docIds:" + docIds); - NamedList ubiInfo = new NamedList<>(); - ubiInfo.add("query_id", "1234"); // change to generateing + SimpleOrderedMap ubiInfo = new SimpleOrderedMap<>(); + ubiInfo.add("query_id", queryId); rb.rsp.add("ubi", ubiInfo); } diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java index 6cb7a2a0588..b8ce0ac70f0 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java @@ -75,6 +75,28 @@ public void testZeroResults() throws Exception { // } } + @Test + public void testExternallyGeneratedQueryId() throws Exception { + assertQ( + "Make sure we generate a query id", + req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true", "query_id", "123abc"), + "//lst[@name='ubi']/str[@name='query_id'][.='123abc']"); + } + + @Test + public void testTrackingOfUserQuery() throws Exception { + assertQ( + "Make sure we generate a query id", + req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true", "user_query", "fresh air"), + "//lst[@name='ubi']/str[@name='query_id'][.='123abc']"); + + // How do we handle this nested data? + assertQ( + "Make sure we generate a query id", + req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true", "user_query", "fresh air"), + "//lst[@name='ubi']/str[@name='query_id'][.='123abc']"); + } + @Test public void testDisabling() throws Exception { // SolrQueryRequest req = null; diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats new file mode 100644 index 00000000000..a790c7a467f --- /dev/null +++ b/solr/packaging/test/test_ubi.bats @@ -0,0 +1,83 @@ +#!/usr/bin/env bats + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load bats_helper + +setup() { + common_clean_setup +} + +teardown() { + # save a snapshot of SOLR_HOME for failed tests + save_home_on_failure + + delete_all_collections + SOLR_STOP_WAIT=1 solr stop -all >/dev/null 2>&1 +} + +@test "Run set up process" { + + solr start -c -e films + + run solr healthcheck -c films + refute_output --partial 'error' + + #echo "Here is the logs dir" + #echo $SOLR_LOGS_DIR + #run ls ${SOLR_LOGS_DIR} + #assert_output --partial "Initializing authentication plugin: solr.KerberosPlugin" + + #assert [ -e ${SOLR_LOGS_DIR}/solr_ubi_queries.log ] + + run curl -X POST -H 'Content-type:application/json' -d '{ + "add-searchcomponent": { + "name": "ubi", + "class": "solr.UBIComponent", + "defaults":{ } + } + }' "http://localhost:${SOLR_PORT}/api/collections/films/config" + + assert_output --partial '"status":0' + + curl -X POST -H 'Content-type:application/json' -d '{ + "update-requesthandler": { + "name": "/select", + "class": "solr.SearchHandler", + "last-components":["ubi"] + } + }' "http://localhost:${SOLR_PORT}/api/collections/films/config" + + assert_output --partial '"status":0' + + curl -X POST -H 'Content-type:application/json' -d '{ + "update-requesthandler": { + "name": "/query", + "class": "solr.SearchHandler", + "last-components":["ubi"] + } + }' "http://localhost:${SOLR_PORT}/api/collections/films/config" + + assert_output --partial '"status":0' + + run curl "http://localhost:${SOLR_PORT}/solr/films/select?q=*:*&rows=3&ubi=true" + assert_output --partial '"status":0' + assert_output --partial '"query_id":"1234' + + #run cat "${SOLR_LOGS_DIR}/solr.log" + #assert_output --partial "Initializing authentication plugin: solr.KerberosPlugin" + #assert_file_contains "${SOLR_LOGS_DIR}/solr_ubi_queries.log" 'eric' +} From 861f92272b9346e81b633abd71e6fe0b2c3d0fac Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 9 May 2024 13:19:50 -0400 Subject: [PATCH 009/103] now logging user_query as a map (hash) to our jsonl log file --- .../solr/handler/component/UBIComponent.java | 92 ++++++++++++++++-- .../handler/component/UBIComponentTest.java | 97 ++++++++++++++++--- solr/packaging/test/test_ubi.bats | 33 +++++-- 3 files changed, 195 insertions(+), 27 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 3ffed420fb4..4ad86453f02 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -16,17 +16,31 @@ */ package org.apache.solr.handler.component; +import java.io.BufferedOutputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.response.ResultContext; import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.plugin.SolrCoreAware; +import org.noggit.CharArr; +import org.noggit.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,14 +74,49 @@ * * query parameter. */ -public class UBIComponent extends SearchComponent { +public class UBIComponent extends SearchComponent implements SolrCoreAware { public static final String COMPONENT_NAME = "ubi"; public static final String QUERY_ID = "query_id"; + public static final String USER_QUERY = "user_query"; + public static final String UBI_QUERY_JSONL_LOG = "ubi_queries.jsonl"; + + protected PluginInfo info = PluginInfo.EMPTY_INFO; private static final Logger ubiRequestLogger = LoggerFactory.getLogger(SolrCore.class.getName() + ".UBIRequest"); + private final CharArr charArr = new CharArr(1024 * 2); + JSONWriter jsonWriter = new JSONWriter(charArr, -1); + private Writer writer; + OutputStream fos; + + @Override + public void inform(SolrCore core) { + List children = info.getChildren("ubi"); + String j = EnvUtils.getProperty("solr.log.dir"); + // error handlin gon missing prop? + + try { + System.out.println("writing to " + j); + fos = new BufferedOutputStream(new FileOutputStream(j + "/" + UBI_QUERY_JSONL_LOG)); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8); + + if (children.isEmpty()) { + // DefaultSolrHighlighter defHighlighter = new DefaultSolrHighlighter(core); + // defHighlighter.init(PluginInfo.EMPTY_INFO); + // solrConfigHighlighter = defHighlighter; + } else { + // solrConfigHighlighter = + // core.createInitInstance( + // children.get(0), SolrHighlighter.class, null, + // DefaultSolrHighlighter.class.getName()); + } + } + @Override public void prepare(ResponseBuilder rb) throws IOException {} @@ -90,21 +139,39 @@ public void process(ResponseBuilder rb) throws IOException { queryId = "1234"; } + // See if the user passed in a user query as a query parameter + Object userQuery = params.get(USER_QUERY); + + if (userQuery != null && userQuery.toString().startsWith("{")) { + // Look up the original nested JSON format, typically passed in + // via the JSON formatted query. + @SuppressWarnings("rawtypes") + Map jsonProperties = rb.req.getJSON(); + if (jsonProperties.containsKey("params")) { + @SuppressWarnings("rawtypes") + Map paramsProperties = (Map) jsonProperties.get("params"); + if (paramsProperties.containsKey("user_query")) { + userQuery = paramsProperties.get("user_query"); + } + } + } + ResultContext rc = (ResultContext) rb.rsp.getResponse(); DocList docs = rc.getDocList(); - processIds(rb, docs, queryId, schema, searcher); + processIds(rb, docs, queryId, userQuery, schema, searcher); } protected void processIds( ResponseBuilder rb, DocList dl, String queryId, + Object userQuery, IndexSchema schema, SolrIndexSearcher searcher) throws IOException { - + charArr.reset(); StringBuilder sb = new StringBuilder(); Set fields = Collections.singleton(schema.getUniqueKeyField().getName()); @@ -114,12 +181,23 @@ protected void processIds( } String docIds = sb.length() > 0 ? sb.substring(0, sb.length() - 1) : ""; - ubiRequestLogger.error("bob dole"); ubiRequestLogger.info("docIds: {}", docIds); System.out.println(" docIds:" + docIds); - SimpleOrderedMap ubiInfo = new SimpleOrderedMap<>(); - ubiInfo.add("query_id", queryId); - rb.rsp.add("ubi", ubiInfo); + SimpleOrderedMap ubiResponseInfo = new SimpleOrderedMap<>(); + SimpleOrderedMap ubiQueryLogInfo = new SimpleOrderedMap<>(); + ubiResponseInfo.add("query_id", queryId); + rb.rsp.add("ubi", ubiResponseInfo); + + ubiQueryLogInfo.add("query_id", queryId); + ubiQueryLogInfo.add("user_query", userQuery); + + jsonWriter.write(ubiQueryLogInfo); + System.out.println("This is what the data looks like:" + charArr.toString()); + writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd()); + writer.append('\n'); + writer.flush(); + // writer.flush(); // maybe don't keep me + } @Override diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java index b8ce0ac70f0..02e702decb1 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java @@ -16,24 +16,38 @@ */ package org.apache.solr.handler.component; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.apache.commons.io.input.ReversedLinesFileReader; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.EnvUtils; import org.apache.solr.request.SolrQueryRequest; import org.junit.BeforeClass; import org.junit.Test; public class UBIComponentTest extends SolrTestCaseJ4 { + private static File ubiQueriesLog; + @BeforeClass public static void beforeTest() throws Exception { + + System.setProperty("solr.log.dir", createTempDir("solr_logs").toString()); + initCore("solrconfig-ubi-component.xml", "schema12.xml"); assertNull(h.validateUpdate(adoc("id", "1", "subject", "aa"))); assertNull(h.validateUpdate(adoc("id", "two", "subject", "aa"))); assertNull(h.validateUpdate(adoc("id", "3", "subject", "aa"))); assertU(commit()); + + ubiQueriesLog = + new File(EnvUtils.getProperty("solr.log.dir") + "/" + UBIComponent.UBI_QUERY_JSONL_LOG); + assertTrue(ubiQueriesLog.exists()); } @Test - public void testToLogIds() throws Exception { + public void testToLogIds() { SolrQueryRequest req = null; try { String handler = "/withubi"; @@ -55,7 +69,7 @@ public void testToLogIds() throws Exception { } @Test - public void testZeroResults() throws Exception { + public void testZeroResults() { // SolrQueryRequest req = null; // try { // String handler = "/withubi"; @@ -76,7 +90,7 @@ public void testZeroResults() throws Exception { } @Test - public void testExternallyGeneratedQueryId() throws Exception { + public void testExternallyGeneratedQueryId() { assertQ( "Make sure we generate a query id", req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true", "query_id", "123abc"), @@ -84,21 +98,71 @@ public void testExternallyGeneratedQueryId() throws Exception { } @Test - public void testTrackingOfUserQuery() throws Exception { - assertQ( - "Make sure we generate a query id", - req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true", "user_query", "fresh air"), - "//lst[@name='ubi']/str[@name='query_id'][.='123abc']"); + public void testJSONQuerySyntax() throws Exception { + // need to fix this. It just checks the number of docs, + // but doesn't do anything about the ubi clauses... + // doesn't appear to trigger the ubi... + assertJQ( + req( + "qt", + "/withubi", + "json", + "{\n" + + " 'query': 'aa',\n" + + " 'fields': '*',\n" + + " 'offset': 0,\n" + + " 'limit': 2,\n" + + " 'params': {\n" + + " 'df': 'subject',\n" + + " 'qt': '/withubi',\n" + + " 'ubi': 'true'\n" + + " }\n" + + "}"), + "response/numFound==3", + "ubi/query_id=='1234'"); + + assertJQ( + req( + "qt", + "/withubi", + "json", + "{\n" + + " 'query': 'aa',\n" + + " 'fields': '*',\n" + + " 'offset': 0,\n" + + " 'limit': 2,\n" + + " 'params': {\n" + + " 'df': 'subject',\n" + + " 'ubi': 'true',\n" + + " 'query_id': 'xjy-42-1rj'\n" + + " 'user_query': {\n" + + " 'query': 'aa',\n" + + " 'page': 2,\n" + + " 'filter': 'inStock:true',\n" + + " }\n" + + " }\n" + + "}"), + "response/numFound==3", + "ubi/query_id=='xjy-42-1rj'"); - // How do we handle this nested data? + String lastLine = readLastLineOfFile(ubiQueriesLog); + + // String json = "{\"query_id\":\"xjy-42-1rj\"}"; + String json = + "{\"query_id\":\"xjy-42-1rj\",\"user_query\":{\"query\":\"aa\",\"page\":2,\"filter\":\"inStock:true\"}}"; + assertJSONEquals(json, lastLine); + } + + @Test + public void testTrackingOfUserQuery() { assertQ( "Make sure we generate a query id", req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true", "user_query", "fresh air"), - "//lst[@name='ubi']/str[@name='query_id'][.='123abc']"); + "//lst[@name='ubi']/str[@name='query_id'][.='1234']"); } @Test - public void testDisabling() throws Exception { + public void testDisabling() { // SolrQueryRequest req = null; // try { // String handler = "/withubi"; @@ -116,4 +180,15 @@ public void testDisabling() throws Exception { // req.close(); // } } + + private static String readLastLineOfFile(File file) { + try (ReversedLinesFileReader reader = + ReversedLinesFileReader.builder().setFile(file).setCharset(StandardCharsets.UTF_8).get()) { + String line = reader.readLine(); + return line; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } } diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats index a790c7a467f..076d76dc225 100644 --- a/solr/packaging/test/test_ubi.bats +++ b/solr/packaging/test/test_ubi.bats @@ -30,18 +30,13 @@ teardown() { } @test "Run set up process" { - solr start -c -e films run solr healthcheck -c films refute_output --partial 'error' - - #echo "Here is the logs dir" - #echo $SOLR_LOGS_DIR - #run ls ${SOLR_LOGS_DIR} - #assert_output --partial "Initializing authentication plugin: solr.KerberosPlugin" - #assert [ -e ${SOLR_LOGS_DIR}/solr_ubi_queries.log ] + # No luck with this + # assert [ -e ${SOLR_LOGS_DIR}/solr_ubi_queries.log ] run curl -X POST -H 'Content-type:application/json' -d '{ "add-searchcomponent": { @@ -73,11 +68,31 @@ teardown() { assert_output --partial '"status":0' + # Simple ubi enabled query run curl "http://localhost:${SOLR_PORT}/solr/films/select?q=*:*&rows=3&ubi=true" assert_output --partial '"status":0' assert_output --partial '"query_id":"1234' + + # Rich ubi enabled query + run curl -X POST -H 'Content-type:application/json' -d '{ + "query" : "*:*", + "limit":2, + params: { + "ubi": "true" + "query_id": "xyz890", + "user_query": { + "query": "hot air", + "page": 2, + "filter": "inStock:true" + } + } + }' "http://localhost:${SOLR_PORT}/solr/films/query" + assert_output --partial '"query_id":"xyz890"' + + # No luck on getting the logs to read. #run cat "${SOLR_LOGS_DIR}/solr.log" - #assert_output --partial "Initializing authentication plugin: solr.KerberosPlugin" - #assert_file_contains "${SOLR_LOGS_DIR}/solr_ubi_queries.log" 'eric' + #run tail -n 1 "${SOLR_LOGS_DIR}/ubi_queries.jsonl" + #assert_output --partial "inStock:false" + #assert_file_contains "${SOLR_LOGS_DIR}/ubi_queries.jsonl" 'eric' } From eaa56e1f5ea10560e1926f5044d94058b4999e93 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 9 May 2024 14:24:40 -0400 Subject: [PATCH 010/103] Log wasn't really working, we want complex nested data, so lets not try to jam it into log4j --- .../solr/handler/component/UBIComponent.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 4ad86453f02..8bc457c8a2d 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -28,6 +28,8 @@ import java.util.List; import java.util.Map; import java.util.Set; + +import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.SimpleOrderedMap; @@ -41,8 +43,6 @@ import org.apache.solr.util.plugin.SolrCoreAware; import org.noggit.CharArr; import org.noggit.JSONWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Inspired by the ResponseLogComponent. @@ -83,9 +83,6 @@ public class UBIComponent extends SearchComponent implements SolrCoreAware { protected PluginInfo info = PluginInfo.EMPTY_INFO; - private static final Logger ubiRequestLogger = - LoggerFactory.getLogger(SolrCore.class.getName() + ".UBIRequest"); - private final CharArr charArr = new CharArr(1024 * 2); JSONWriter jsonWriter = new JSONWriter(charArr, -1); private Writer writer; @@ -95,13 +92,14 @@ public class UBIComponent extends SearchComponent implements SolrCoreAware { public void inform(SolrCore core) { List children = info.getChildren("ubi"); String j = EnvUtils.getProperty("solr.log.dir"); - // error handlin gon missing prop? - + String ubiQueryJSONLLog = EnvUtils.getProperty("solr.log.dir") + "/" + UBI_QUERY_JSONL_LOG; try { - System.out.println("writing to " + j); - fos = new BufferedOutputStream(new FileOutputStream(j + "/" + UBI_QUERY_JSONL_LOG)); - } catch (FileNotFoundException e) { - e.printStackTrace(); + fos = new BufferedOutputStream(new FileOutputStream(ubiQueryJSONLLog)); + } catch (FileNotFoundException exception) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Error creating file " + ubiQueryJSONLLog, + exception); } writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8); @@ -180,8 +178,6 @@ protected void processIds( sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(','); } String docIds = sb.length() > 0 ? sb.substring(0, sb.length() - 1) : ""; - - ubiRequestLogger.info("docIds: {}", docIds); System.out.println(" docIds:" + docIds); SimpleOrderedMap ubiResponseInfo = new SimpleOrderedMap<>(); SimpleOrderedMap ubiQueryLogInfo = new SimpleOrderedMap<>(); @@ -196,12 +192,11 @@ protected void processIds( writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd()); writer.append('\n'); writer.flush(); - // writer.flush(); // maybe don't keep me } @Override public String getDescription() { - return "A component that inserts the retrieved documents into the response log entry"; + return "A component that tracks original user query and the resulting documents returned."; } } From aa45a838d95342a75ad4202568736ff0159a4800 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 9 May 2024 14:41:19 -0400 Subject: [PATCH 011/103] actually track the doc_ids in our jsonl file --- .../apache/solr/handler/component/UBIComponent.java | 3 +-- .../solr/handler/component/UBIComponentTest.java | 12 ++++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 8bc457c8a2d..8a90bb653a9 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -178,7 +178,6 @@ protected void processIds( sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(','); } String docIds = sb.length() > 0 ? sb.substring(0, sb.length() - 1) : ""; - System.out.println(" docIds:" + docIds); SimpleOrderedMap ubiResponseInfo = new SimpleOrderedMap<>(); SimpleOrderedMap ubiQueryLogInfo = new SimpleOrderedMap<>(); ubiResponseInfo.add("query_id", queryId); @@ -186,9 +185,9 @@ protected void processIds( ubiQueryLogInfo.add("query_id", queryId); ubiQueryLogInfo.add("user_query", userQuery); + ubiQueryLogInfo.add("doc_ids", docIds); jsonWriter.write(ubiQueryLogInfo); - System.out.println("This is what the data looks like:" + charArr.toString()); writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd()); writer.append('\n'); writer.flush(); diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java index 02e702decb1..d1241201076 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java @@ -121,6 +121,11 @@ public void testJSONQuerySyntax() throws Exception { "response/numFound==3", "ubi/query_id=='1234'"); + String lastLine = readLastLineOfFile(ubiQueriesLog); + + String json = "{\"query_id\":\"1234\",\"user_query\":null,\"doc_ids\":\"1,two\"}"; + assertJSONEquals(json, lastLine); + assertJQ( req( "qt", @@ -145,11 +150,10 @@ public void testJSONQuerySyntax() throws Exception { "response/numFound==3", "ubi/query_id=='xjy-42-1rj'"); - String lastLine = readLastLineOfFile(ubiQueriesLog); + lastLine = readLastLineOfFile(ubiQueriesLog); - // String json = "{\"query_id\":\"xjy-42-1rj\"}"; - String json = - "{\"query_id\":\"xjy-42-1rj\",\"user_query\":{\"query\":\"aa\",\"page\":2,\"filter\":\"inStock:true\"}}"; + json = + "{\"query_id\":\"xjy-42-1rj\",\"user_query\":{\"query\":\"aa\",\"page\":2,\"filter\":\"inStock:true\"},\"doc_ids\":\"1,two\"}"; assertJSONEquals(json, lastLine); } From 9523c18f324b9f0a4f068512f243b4f6162252ab Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 9 May 2024 15:06:07 -0400 Subject: [PATCH 012/103] tidy --- .../org/apache/solr/handler/component/UBIComponent.java | 8 +++----- .../apache/solr/handler/component/UBIComponentTest.java | 5 +---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 8a90bb653a9..c8a8ae4ea76 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.EnvUtils; @@ -97,9 +96,9 @@ public void inform(SolrCore core) { fos = new BufferedOutputStream(new FileOutputStream(ubiQueryJSONLLog)); } catch (FileNotFoundException exception) { throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Error creating file " + ubiQueryJSONLLog, - exception); + SolrException.ErrorCode.SERVER_ERROR, + "Error creating file " + ubiQueryJSONLLog, + exception); } writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8); @@ -191,7 +190,6 @@ protected void processIds( writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd()); writer.append('\n'); writer.flush(); - } @Override diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java index d1241201076..20cb0826098 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java @@ -185,14 +185,11 @@ public void testDisabling() { // } } - private static String readLastLineOfFile(File file) { + private static String readLastLineOfFile(File file) throws IOException { try (ReversedLinesFileReader reader = ReversedLinesFileReader.builder().setFile(file).setCharset(StandardCharsets.UTF_8).get()) { String line = reader.readLine(); return line; - } catch (IOException e) { - e.printStackTrace(); } - return null; } } From c7a939ec6c6f4893dca32fa81a9b7ec9475e511a Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 15 May 2024 08:57:29 -0400 Subject: [PATCH 013/103] provide more context to how to use UBI --- .../solr/handler/component/UBIComponent.java | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index c8a8ae4ea76..4a136221a87 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -44,12 +44,20 @@ import org.noggit.JSONWriter; /** - * Inspired by the ResponseLogComponent. + * User Behavior Insights (UBI) is a open standard for gathering query and event data from users and + * storing it in a structured format. UBI can be used for in session personalization, implicit + * judgements, powering recommendation systems among others. Learn more about the UBI standard at https://github.com/o19s/ubi. * - *

Adds to the .ubi_queries system collection the original user query and the document IDs that - * are sent in the query response. + *

Query data is gathered by this component. Data tracked is the collection name, the end user + * query, as json blob, and the resulting document id's. * - *

Tracks the collection name, the end user query, as json blob, and the resulting document id's. + *

Data is written out data to "ubi_queries.jsonl", a JSON with Lines formatted file, or you can + * provide a streaming expression that is parsed and loaded by the component to stream query data to + * a target of your choice. + * + *

Event data is tracked by letting the user write events directly to the event repository of + * your choice, it could be a Solr collection, it could be a file or S3 bucket. * *

Add it to a requestHandler in solrconfig.xml like this: * @@ -72,6 +80,33 @@ * 

ubi=true
* * query parameter. + * + *

Ideally this component is used with the JSON Query syntax, as that facilitates passing in the + * additional data to be tracked with a query. Here is an example: + * + * 

+ *     {
+ *     "query" : "apple AND ipod",
+ *     "limit":2,
+ *     "start":2,
+ *     "filter": [
+ *        "inStock:true"
+ *      ]
+ *     params: {
+ *       "ubi": "true"
+ *       "query_id": "xyz890",
+ *       "user_query": {
+ *         "query": "Apple iPod",
+ *         "page": 2,
+ *         "in_stock": "true"
+ *       }
+ *     }
+ *   }
+ *
+ * + * Notice that we are enabling UBI query tracking, we are providing an explicit query_id and passing + * in the user's specific choices for querying. The user_query parameters are not specific to Solr + * syntax, they are defined by the front end user interface. */ public class UBIComponent extends SearchComponent implements SolrCoreAware { From a6d33a91fb752c68e224a6b731a9061555db4b0d Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 20 May 2024 14:04:45 -0400 Subject: [PATCH 014/103] techproducts gives a better example because of the inStock filter --- .../UBIComponentStreamingQueriesTest.java | 149 ++++++++++++++++++ solr/packaging/test/test_ubi.bats | 32 ++-- 2 files changed, 167 insertions(+), 14 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java new file mode 100644 index 00000000000..36101242af9 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.component; + +import java.util.ArrayList; +import java.util.List; +import org.apache.lucene.util.IOUtils; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.request.json.JsonQueryRequest; +import org.apache.solr.client.solrj.request.json.TermsFacetMap; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.json.BucketBasedJsonFacet; +import org.apache.solr.client.solrj.response.json.BucketJsonFacet; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.update.processor.SignatureUpdateProcessorFactory; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** + * Tests the ability to use {@link SignatureUpdateProcessorFactory} to generate uniqueKeys for + * "duplicate" documents in cloud mode. + */ +public class UBIComponentLoggingTest extends SolrCloudTestCase { + public static final String COLLECTION = "dedup_col"; + + /** One client per node */ + private static final List NODE_CLIENTS = new ArrayList<>(7); + + /** + * clients (including cloud client) for easy randomization and looping of collection level + * requests + */ + private static final List CLIENTS = new ArrayList<>(7); + + @BeforeClass + public static void setupCluster() throws Exception { + final int numShards = usually() ? 2 : 1; + final int numReplicas = usually() ? 2 : 1; + final int numNodes = 1 + (numShards * numReplicas); // at least one node w/o any replicas + configureCluster(numNodes).addConfig("conf", configset("dedup")).configure(); + + CLIENTS.add(cluster.getSolrClient()); + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + final SolrClient c = getHttpSolrClient(jetty.getBaseUrl().toString()); + NODE_CLIENTS.add(c); + CLIENTS.add(c); + } + + assertEquals( + "failed to create collection", + 0, + CollectionAdminRequest.createCollection(COLLECTION, "conf", numShards, numReplicas) + .process(cluster.getSolrClient()) + .getStatus()); + + cluster.waitForActiveCollection(COLLECTION, numShards, numShards * numReplicas); + } + + @AfterClass + public static void closeClients() throws Exception { + try { + IOUtils.close(NODE_CLIENTS); + } finally { + NODE_CLIENTS.clear(); + CLIENTS.clear(); + } + } + + @After + public void clearCollection() throws Exception { + assertEquals( + "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(COLLECTION, "*:*").getStatus()); + assertEquals("commit failed", 0, cluster.getSolrClient().commit(COLLECTION).getStatus()); + } + + public void testRandomDocs() throws Exception { + + // index some random documents, using a mix-match of batches, to various SolrClients + + final int uniqueMod = atLeast(43); // the number of unique sig values expected + final int numBatches = atLeast(uniqueMod); // we'll add at least one doc per batch + int docCounter = 0; + for (int batchId = 0; batchId < numBatches; batchId++) { + final UpdateRequest ureq = new UpdateRequest(); + final int batchSize = atLeast(2); + for (int i = 0; i < batchSize; i++) { + docCounter++; + ureq.add( + sdoc( // NOTE: No 'id' field, SignatureUpdateProcessor fills it in for us + "data_s", (docCounter % uniqueMod))); + } + assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION).getStatus()); + } + assertEquals("commit failed", 0, getRandClient().commit(COLLECTION).getStatus()); + + assertTrue(docCounter > uniqueMod); + + // query our collection and confirm no duplicates on the signature field (using faceting) + // Check every (node) for consistency... + final JsonQueryRequest req = + new JsonQueryRequest() + .setQuery("*:*") + .setLimit(0) + .withFacet("data_facet", new TermsFacetMap("data_s").setLimit(uniqueMod + 1)); + for (SolrClient client : CLIENTS) { + final QueryResponse rsp = req.process(client, COLLECTION); + try { + assertEquals(0, rsp.getStatus()); + assertEquals(uniqueMod, rsp.getResults().getNumFound()); + + final BucketBasedJsonFacet facet = + rsp.getJsonFacetingResponse().getBucketBasedFacets("data_facet"); + assertEquals(uniqueMod, facet.getBuckets().size()); + for (BucketJsonFacet bucket : facet.getBuckets()) { + assertEquals("Bucket " + bucket.getVal(), 1, bucket.getCount()); + } + } catch (AssertionError e) { + throw new AssertionError(rsp + " + " + client + " => " + e.getMessage(), e); + } + } + } + + /** + * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed at a node + * in our cluster. + */ + private static SolrClient getRandClient() { + return CLIENTS.get(random().nextInt(CLIENTS.size())); + } +} diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats index 076d76dc225..b9fb7016205 100644 --- a/solr/packaging/test/test_ubi.bats +++ b/solr/packaging/test/test_ubi.bats @@ -30,9 +30,9 @@ teardown() { } @test "Run set up process" { - solr start -c -e films + solr start -c -e techproducts - run solr healthcheck -c films + run solr healthcheck -c techproducts refute_output --partial 'error' # No luck with this @@ -44,7 +44,7 @@ teardown() { "class": "solr.UBIComponent", "defaults":{ } } - }' "http://localhost:${SOLR_PORT}/api/collections/films/config" + }' "http://localhost:${SOLR_PORT}/api/collections/techproducts/config" assert_output --partial '"status":0' @@ -54,7 +54,7 @@ teardown() { "class": "solr.SearchHandler", "last-components":["ubi"] } - }' "http://localhost:${SOLR_PORT}/api/collections/films/config" + }' "http://localhost:${SOLR_PORT}/api/collections/techproducts/config" assert_output --partial '"status":0' @@ -64,30 +64,34 @@ teardown() { "class": "solr.SearchHandler", "last-components":["ubi"] } - }' "http://localhost:${SOLR_PORT}/api/collections/films/config" + }' "http://localhost:${SOLR_PORT}/api/collections/techproducts/config" assert_output --partial '"status":0' # Simple ubi enabled query - run curl "http://localhost:${SOLR_PORT}/solr/films/select?q=*:*&rows=3&ubi=true" + run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=3&ubi=true" assert_output --partial '"status":0' assert_output --partial '"query_id":"1234' - # Rich ubi enabled query + # Rich UBI user query tracking enabled query run curl -X POST -H 'Content-type:application/json' -d '{ - "query" : "*:*", + "query" : "ram OR memory", + "filter": [ + "inStock:true" + ], "limit":2, - params: { - "ubi": "true" + "params": { + "ubi": "true", "query_id": "xyz890", "user_query": { - "query": "hot air", - "page": 2, - "filter": "inStock:true" + "query": "RAM memory", + "experiment": "supersecret", + "page": 1, + "filter": "productStatus:available" } } - }' "http://localhost:${SOLR_PORT}/solr/films/query" + }' "http://localhost:${SOLR_PORT}/solr/techproducts/query" assert_output --partial '"query_id":"xyz890"' # No luck on getting the logs to read. From 34a04bfd0342e8c9f98ffd892bf394eedd087fb9 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 20 May 2024 20:26:39 -0400 Subject: [PATCH 015/103] Working on trying to get streaming expressions to provide a pluggable backend for events. --- .../solr/handler/component/UBIComponent.java | 226 ++++++++++++++- ...xml => solrconfig-ubi-local-component.xml} | 4 +- .../configsets/ubi-enabled/conf/schema.xml | 23 ++ .../ubi-enabled/conf/solrconfig.xml | 54 ++++ .../ubi-enabled/conf/ubi-query-pipeline.expr | 6 + .../UBIComponentStreamingQueriesTest.java | 274 +++++++++++------- .../handler/component/UBIComponentTest.java | 2 +- .../client/solrj/io/stream/LetStream.java | 5 + 8 files changed, 476 insertions(+), 118 deletions(-) rename solr/core/src/test-files/solr/collection1/conf/{solrconfig-ubi-component.xml => solrconfig-ubi-local-component.xml} (94%) create mode 100644 solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml create mode 100644 solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml create mode 100644 solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 4a136221a87..d08b22ab3da 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -17,21 +17,43 @@ package org.apache.solr.handler.component; import java.io.BufferedOutputStream; +import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.LineNumberReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.solr.client.solrj.io.Lang; +import org.apache.solr.client.solrj.io.SolrClientCache; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.comp.StreamComparator; +import org.apache.solr.client.solrj.io.stream.LetStream; +import org.apache.solr.client.solrj.io.stream.PushBackStream; +import org.apache.solr.client.solrj.io.stream.StreamContext; +import org.apache.solr.client.solrj.io.stream.TupleStream; +import org.apache.solr.client.solrj.io.stream.expr.Explanation; +import org.apache.solr.client.solrj.io.stream.expr.Expressible; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser; +import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.CoreContainer; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.response.ResultContext; @@ -42,6 +64,8 @@ import org.apache.solr.util.plugin.SolrCoreAware; import org.noggit.CharArr; import org.noggit.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * User Behavior Insights (UBI) is a open standard for gathering query and event data from users and @@ -117,27 +141,45 @@ public class UBIComponent extends SearchComponent implements SolrCoreAware { protected PluginInfo info = PluginInfo.EMPTY_INFO; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final CharArr charArr = new CharArr(1024 * 2); JSONWriter jsonWriter = new JSONWriter(charArr, -1); private Writer writer; OutputStream fos; + private StreamFactory streamFactory; + private StreamExpression streamExpression; + + protected SolrParams initArgs; + private SolrClientCache solrClientCache; + + @Override + public void init(NamedList args) { + this.initArgs = args.toSolrParams(); + } + @Override + @SuppressWarnings({"rawtypes"}) public void inform(SolrCore core) { List children = info.getChildren("ubi"); - String j = EnvUtils.getProperty("solr.log.dir"); - String ubiQueryJSONLLog = EnvUtils.getProperty("solr.log.dir") + "/" + UBI_QUERY_JSONL_LOG; - try { - fos = new BufferedOutputStream(new FileOutputStream(ubiQueryJSONLLog)); - } catch (FileNotFoundException exception) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Error creating file " + ubiQueryJSONLLog, - exception); + String defaultZkhost = null; + CoreContainer coreContainer = core.getCoreContainer(); + this.solrClientCache = coreContainer.getSolrClientCache(); + if (coreContainer.isZooKeeperAware()) { + defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress(); } - writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8); if (children.isEmpty()) { + String ubiQueryJSONLLog = EnvUtils.getProperty("solr.log.dir") + "/" + UBI_QUERY_JSONL_LOG; + try { + fos = new BufferedOutputStream(new FileOutputStream(ubiQueryJSONLLog)); + } catch (FileNotFoundException exception) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Error creating file " + ubiQueryJSONLLog, + exception); + } + writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8); // DefaultSolrHighlighter defHighlighter = new DefaultSolrHighlighter(core); // defHighlighter.init(PluginInfo.EMPTY_INFO); // solrConfigHighlighter = defHighlighter; @@ -147,6 +189,63 @@ public void inform(SolrCore core) { // children.get(0), SolrHighlighter.class, null, // DefaultSolrHighlighter.class.getName()); } + // do i need this check? + if (initArgs != null) { + log.info("Initializing UBIComponent"); + String streamQueriesExpressionFile = initArgs.get("streamQueriesExpressionFile"); + + if (streamQueriesExpressionFile != null) { + System.out.println("got" + streamQueriesExpressionFile); + + String exprFile = streamQueriesExpressionFile; + + LineNumberReader bufferedReader = null; + boolean verbose = false; + + try { + bufferedReader = + new LineNumberReader( + new InputStreamReader(core.getResourceLoader().openResource(exprFile))); + + String args[] = {}; // maybe we have variables? + String expr = readExpression(bufferedReader, args); + + bufferedReader.close(); + + streamExpression = StreamExpressionParser.parse(expr); + streamFactory = new StreamFactory(); + + String defaultZk = null; + String[] outputHeaders = null; + String delim = " "; + String arrayDelim = "|"; + boolean includeHeaders = false; + streamFactory.withDefaultZkHost(defaultZkhost); + + Lang.register(streamFactory); + + TupleStream stream = constructStream(streamFactory, streamExpression); + + // not sure i need this? Except maybe we assume let? + Map params = validateLetAndGetParams(stream, expr); + + + + } catch (IOException ioe) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + exprFile, ioe); + } finally { + + // if (pushBackStream != null) { + //try { + //pushBackStream.close(); + //} catch (IOException e) { + // e.printStackTrace(); + //} + // } + } + } + } } @Override @@ -221,10 +320,109 @@ protected void processIds( ubiQueryLogInfo.add("user_query", userQuery); ubiQueryLogInfo.add("doc_ids", docIds); - jsonWriter.write(ubiQueryLogInfo); - writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd()); - writer.append('\n'); - writer.flush(); + if (writer != null) { + jsonWriter.write(ubiQueryLogInfo); + writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd()); + writer.append('\n'); + writer.flush(); + } + + if (streamFactory != null){ + //streamFactory.withFunctionName("stdin", StandardInStream.class); + TupleStream stream = null; + //PushBackStream pushBackStream = null; + stream = constructStream(streamFactory, streamExpression); + + //Map params = validateLetAndGetParams(stream, expr); + + //pushBackStream = new PushBackStream(stream); + + StreamContext streamContext = new StreamContext(); + streamContext.setSolrClientCache(solrClientCache); + stream.setStreamContext(streamContext); + List tuples = getTuples(stream); + System.out.println("tuples:" + tuples); + //assertEquals(4, tuples.size()); + //pushBackStream.open(); + } + } + + protected List getTuples(TupleStream tupleStream) throws IOException { + tupleStream.open(); + List tuples = new ArrayList<>(); + for (; ; ) { + Tuple t = tupleStream.read(); + if (t.EOF) { + break; + } else { + tuples.add(t); + } + } + tupleStream.close(); + return tuples; + } + + protected Tuple getTuple(TupleStream tupleStream) throws IOException { + tupleStream.open(); + Tuple t = tupleStream.read(); + tupleStream.close(); + return t; + } + + public static String readExpression(LineNumberReader bufferedReader, String args[]) + throws IOException { + + StringBuilder exprBuff = new StringBuilder(); + + boolean comment = false; + while (true) { + String line = bufferedReader.readLine(); + if (line == null) { + break; + } + + if (line.indexOf("/*") == 0) { + comment = true; + continue; + } + + if (line.indexOf("*/") == 0) { + comment = false; + continue; + } + + if (comment || line.startsWith("#") || line.startsWith("//")) { + continue; + } + + // Substitute parameters + + if (line.length() > 0) { + for (int i = 1; i < args.length; i++) { + String arg = args[i]; + line = line.replace("$" + i, arg); + } + } + + exprBuff.append(line); + } + + return exprBuff.toString(); + } + + public static TupleStream constructStream( + StreamFactory streamFactory, StreamExpression streamExpression) throws IOException { + return streamFactory.constructStream(streamExpression); + } + + @SuppressWarnings({"rawtypes"}) + public static Map validateLetAndGetParams(TupleStream stream, String expr) throws IOException { + if (stream instanceof LetStream) { + LetStream mainStream = (LetStream) stream; + return mainStream.getLetParams(); + } else { + throw new IOException("No enclosing let function found in expression:" + expr); + } } @Override diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-component.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml similarity index 94% rename from solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-component.xml rename to solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml index deedac7f878..ddedb35856f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-component.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml @@ -34,7 +34,9 @@ - + + + diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml new file mode 100644 index 00000000000..aaf3e5077e4 --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml @@ -0,0 +1,23 @@ + + + + + + + id + diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml new file mode 100644 index 00000000000..9b59dd671ef --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml @@ -0,0 +1,54 @@ + + + + + + + + + ${solr.data.dir:} + + + + + ${tests.luceneMatchVersion:LATEST} + + + + ${solr.commitwithin.softcommit:true} + + + + + + ubi-query-pipeline.expr + + + + + explicit + true + text + + + ubi + + + + diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr new file mode 100644 index 00000000000..f6e0c67ad3a --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr @@ -0,0 +1,6 @@ +let(cli-zkhost="localhost:9983", + commit(ubi, + update(ubi, + tuple(id=49,a_i=1,b_i=5)) + ) +) diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java index 36101242af9..a85bf0b7373 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java @@ -18,7 +18,10 @@ package org.apache.solr.handler.component; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Random; import org.apache.lucene.util.IOUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -30,120 +33,187 @@ import org.apache.solr.client.solrj.response.json.BucketJsonFacet; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.update.processor.SignatureUpdateProcessorFactory; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; /** - * Tests the ability to use {@link SignatureUpdateProcessorFactory} to generate uniqueKeys for - * "duplicate" documents in cloud mode. + * Tests the ability for {@link UBIComponent} to stream the gathered query data to another Solr + * index using Streaming Expressions. */ -public class UBIComponentLoggingTest extends SolrCloudTestCase { - public static final String COLLECTION = "dedup_col"; - - /** One client per node */ - private static final List NODE_CLIENTS = new ArrayList<>(7); - - /** - * clients (including cloud client) for easy randomization and looping of collection level - * requests - */ - private static final List CLIENTS = new ArrayList<>(7); - - @BeforeClass - public static void setupCluster() throws Exception { - final int numShards = usually() ? 2 : 1; - final int numReplicas = usually() ? 2 : 1; - final int numNodes = 1 + (numShards * numReplicas); // at least one node w/o any replicas - configureCluster(numNodes).addConfig("conf", configset("dedup")).configure(); - - CLIENTS.add(cluster.getSolrClient()); - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - final SolrClient c = getHttpSolrClient(jetty.getBaseUrl().toString()); - NODE_CLIENTS.add(c); - CLIENTS.add(c); - } - - assertEquals( - "failed to create collection", - 0, - CollectionAdminRequest.createCollection(COLLECTION, "conf", numShards, numReplicas) - .process(cluster.getSolrClient()) - .getStatus()); - - cluster.waitForActiveCollection(COLLECTION, numShards, numShards * numReplicas); +public class UBIComponentStreamingQueriesTest extends SolrCloudTestCase { + public static final String COLLECTION = "conf2_col"; + public static final String UBI_COLLECTION = "ubi"; + + /** One client per node */ + private static final List NODE_CLIENTS = new ArrayList<>(7); + + /** + * clients (including cloud client) for easy randomization and looping of collection level + * requests + */ + private static final List CLIENTS = new ArrayList<>(7); + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("solr.log.dir", createTempDir("solr_logs").toString()); + + final int numShards = usually() ? 2 : 1; + final int numReplicas = usually() ? 2 : 1; + final int numNodes = 1 + (numShards * numReplicas); // at least one node w/o any replicas + + // The configset ubi_enabled has the UBIComponent configured and set to log to a collection + // called "ubi". + // The ubi collection itself just depends on the typical _default configset. + configureCluster(numNodes) + .addConfig("ubi-enabled", configset("ubi-enabled")) + .addConfig("ubi", configset("_default")) + .configure(); + + CLIENTS.add(cluster.getSolrClient()); + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + final SolrClient c = getHttpSolrClient(jetty.getBaseUrl().toString()); + NODE_CLIENTS.add(c); + CLIENTS.add(c); } - @AfterClass - public static void closeClients() throws Exception { - try { - IOUtils.close(NODE_CLIENTS); - } finally { - NODE_CLIENTS.clear(); - CLIENTS.clear(); - } + assertEquals( + "failed to create collection", + 0, + CollectionAdminRequest.createCollection(COLLECTION, "ubi-enabled", numShards, numReplicas) + .process(cluster.getSolrClient()) + .getStatus()); + + cluster.waitForActiveCollection(COLLECTION, numShards, numShards * numReplicas); + + assertEquals( + "failed to create UBI collection", + 0, + CollectionAdminRequest.createCollection(UBI_COLLECTION, "_default", numShards, numReplicas) + .process(cluster.getSolrClient()) + .getStatus()); + + cluster.waitForActiveCollection(UBI_COLLECTION, numShards, numShards * numReplicas); + } + + @AfterClass + public static void closeClients() throws Exception { + try { + IOUtils.close(NODE_CLIENTS); + } finally { + NODE_CLIENTS.clear(); + CLIENTS.clear(); } + } - @After - public void clearCollection() throws Exception { - assertEquals( - "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(COLLECTION, "*:*").getStatus()); - assertEquals("commit failed", 0, cluster.getSolrClient().commit(COLLECTION).getStatus()); - } + @After + public void clearCollection() throws Exception { + assertEquals( + "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(COLLECTION, "*:*").getStatus()); + assertEquals("commit failed", 0, cluster.getSolrClient().commit(COLLECTION).getStatus()); + } - public void testRandomDocs() throws Exception { - - // index some random documents, using a mix-match of batches, to various SolrClients - - final int uniqueMod = atLeast(43); // the number of unique sig values expected - final int numBatches = atLeast(uniqueMod); // we'll add at least one doc per batch - int docCounter = 0; - for (int batchId = 0; batchId < numBatches; batchId++) { - final UpdateRequest ureq = new UpdateRequest(); - final int batchSize = atLeast(2); - for (int i = 0; i < batchSize; i++) { - docCounter++; - ureq.add( - sdoc( // NOTE: No 'id' field, SignatureUpdateProcessor fills it in for us - "data_s", (docCounter % uniqueMod))); - } - assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION).getStatus()); - } - assertEquals("commit failed", 0, getRandClient().commit(COLLECTION).getStatus()); - - assertTrue(docCounter > uniqueMod); - - // query our collection and confirm no duplicates on the signature field (using faceting) - // Check every (node) for consistency... - final JsonQueryRequest req = - new JsonQueryRequest() - .setQuery("*:*") - .setLimit(0) - .withFacet("data_facet", new TermsFacetMap("data_s").setLimit(uniqueMod + 1)); - for (SolrClient client : CLIENTS) { - final QueryResponse rsp = req.process(client, COLLECTION); - try { - assertEquals(0, rsp.getStatus()); - assertEquals(uniqueMod, rsp.getResults().getNumFound()); - - final BucketBasedJsonFacet facet = - rsp.getJsonFacetingResponse().getBucketBasedFacets("data_facet"); - assertEquals(uniqueMod, facet.getBuckets().size()); - for (BucketJsonFacet bucket : facet.getBuckets()) { - assertEquals("Bucket " + bucket.getVal(), 1, bucket.getCount()); - } - } catch (AssertionError e) { - throw new AssertionError(rsp + " + " + client + " => " + e.getMessage(), e); - } - } + @SuppressWarnings({"rawtypes", "unchecked"}) + public void testRandomDocs() throws Exception { + + final UpdateRequest ureq = new UpdateRequest(); + for (int i = 0; i < 1; i++) { + + ureq.add(sdoc("id", i, "data_s", "data:" + i)); + } + assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION).getStatus()); + assertEquals("commit failed", 0, getRandClient().commit(COLLECTION).getStatus()); + + // query our collection to generate a UBI event and then confirm it was recorded. + + Map userQuery = new HashMap(); + userQuery.put("query", "hot air"); + userQuery.put("results_wanted", 1); + + final JsonQueryRequest req = + new JsonQueryRequest() + .setQuery("id:1") + .setLimit(1) + .withParam("ubi", "true") + .withParam("query_id", "123") + .withParam("user_query", userQuery); + + // Randomly grab a client, it shouldn't matter which is used to generate the query event. + SolrClient client = getRandClient(); + final QueryResponse rsp = req.process(client, COLLECTION); + try { + assertEquals(0, rsp.getStatus()); + assertEquals(1, rsp.getResults().getNumFound()); + } catch (AssertionError e) { + throw new AssertionError(rsp + " + " + client + " => " + e.getMessage(), e); } - /** - * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed at a node - * in our cluster. - */ - private static SolrClient getRandClient() { - return CLIENTS.get(random().nextInt(CLIENTS.size())); + // Check the UBI collection + final JsonQueryRequest requestUBI = new JsonQueryRequest().setQuery("id:49").setLimit(1); + + // Randomly grab a client, it shouldn't matter which is used to check UBI event. + client = getRandClient(); + final QueryResponse responseUBI = requestUBI.process(client, UBI_COLLECTION); + try { + assertEquals(0, responseUBI.getStatus()); + assertEquals(1, responseUBI.getResults().getNumFound()); + } catch (AssertionError e) { + throw new AssertionError(responseUBI + " + " + client + " => " + e.getMessage(), e); + } + } + + public void randomDocs() throws Exception { + + // index some random documents, using a mix-match of batches, to various SolrClients + + final int uniqueMod = atLeast(43); // the number of unique sig values expected + final int numBatches = atLeast(uniqueMod); // we'll add at least one doc per batch + int docCounter = 0; + for (int batchId = 0; batchId < numBatches; batchId++) { + final UpdateRequest ureq = new UpdateRequest(); + final int batchSize = atLeast(2); + for (int i = 0; i < batchSize; i++) { + docCounter++; + ureq.add( + sdoc( // NOTE: No 'id' field, SignatureUpdateProcessor fills it in for us + "data_s", (docCounter % uniqueMod))); + } + assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION).getStatus()); + } + assertEquals("commit failed", 0, getRandClient().commit(COLLECTION).getStatus()); + + assertTrue(docCounter > uniqueMod); + + // query our collection and confirm no duplicates on the signature field (using faceting) + // Check every (node) for consistency... + final JsonQueryRequest req = + new JsonQueryRequest() + .setQuery("*:*") + .setLimit(0) + .withFacet("data_facet", new TermsFacetMap("data_s").setLimit(uniqueMod + 1)); + for (SolrClient client : CLIENTS) { + final QueryResponse rsp = req.process(client, COLLECTION); + try { + assertEquals(0, rsp.getStatus()); + assertEquals(uniqueMod, rsp.getResults().getNumFound()); + + final BucketBasedJsonFacet facet = + rsp.getJsonFacetingResponse().getBucketBasedFacets("data_facet"); + assertEquals(uniqueMod, facet.getBuckets().size()); + for (BucketJsonFacet bucket : facet.getBuckets()) { + assertEquals("Bucket " + bucket.getVal(), 1, bucket.getCount()); + } + } catch (AssertionError e) { + throw new AssertionError(rsp + " + " + client + " => " + e.getMessage(), e); + } } + } + + /** + * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed at a node + * in our cluster. + */ + private static SolrClient getRandClient() { + return CLIENTS.get(random().nextInt(CLIENTS.size())); + } } diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java index 20cb0826098..be8523d8b6c 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java @@ -35,7 +35,7 @@ public static void beforeTest() throws Exception { System.setProperty("solr.log.dir", createTempDir("solr_logs").toString()); - initCore("solrconfig-ubi-component.xml", "schema12.xml"); + initCore("solrconfig-ubi-local-component.xml", "schema12.xml"); assertNull(h.validateUpdate(adoc("id", "1", "subject", "aa"))); assertNull(h.validateUpdate(adoc("id", "two", "subject", "aa"))); assertNull(h.validateUpdate(adoc("id", "3", "subject", "aa"))); diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java index 9576cf9658e..fc26a8972f7 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java @@ -224,4 +224,9 @@ public StreamComparator getStreamSort() { public int getCost() { return 0; } + + @SuppressWarnings({"rawtypes"}) + public Map getLetParams() { + return this.letParams; + } } From 43475c995f8ffb6024cd7f1003c69b7d2be696ad Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 20 May 2024 20:41:50 -0400 Subject: [PATCH 016/103] precommit failures --- .../solr/handler/component/UBIComponent.java | 9 +-------- .../ubi-enabled/conf/ubi-query-pipeline.expr | 15 +++++++++++++++ .../UBIComponentStreamingQueriesTest.java | 1 - 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index d08b22ab3da..a3354661960 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -17,11 +17,9 @@ package org.apache.solr.handler.component; import java.io.BufferedOutputStream; -import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.io.OutputStream; @@ -31,21 +29,16 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.solr.client.solrj.io.Lang; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; -import org.apache.solr.client.solrj.io.comp.StreamComparator; import org.apache.solr.client.solrj.io.stream.LetStream; -import org.apache.solr.client.solrj.io.stream.PushBackStream; import org.apache.solr.client.solrj.io.stream.StreamContext; import org.apache.solr.client.solrj.io.stream.TupleStream; -import org.apache.solr.client.solrj.io.stream.expr.Explanation; -import org.apache.solr.client.solrj.io.stream.expr.Expressible; -import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; +UBIComponentStreamingQueriesTest.javaimport org.apache.solr.client.solrj.io.stream.expr.StreamExpression; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; import org.apache.solr.common.SolrException; diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr index f6e0c67ad3a..4b132ee99a5 100644 --- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr +++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + let(cli-zkhost="localhost:9983", commit(ubi, update(ubi, diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java index a85bf0b7373..7988452a318 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Random; import org.apache.lucene.util.IOUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; From f0d2cb6788629de57945abb77f208bdaff86a23a Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 20 May 2024 21:08:14 -0400 Subject: [PATCH 017/103] cut n paste error --- .../java/org/apache/solr/handler/component/UBIComponent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index a3354661960..d38c01f0fee 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -38,7 +38,7 @@ import org.apache.solr.client.solrj.io.stream.LetStream; import org.apache.solr.client.solrj.io.stream.StreamContext; import org.apache.solr.client.solrj.io.stream.TupleStream; -UBIComponentStreamingQueriesTest.javaimport org.apache.solr.client.solrj.io.stream.expr.StreamExpression; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; import org.apache.solr.common.SolrException; From 9168998fe80d20621c4459f65723a90f8df3e6ca Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 20 May 2024 21:19:41 -0400 Subject: [PATCH 018/103] policeman failures fixed --- .../solr/handler/component/UBIComponent.java | 47 ++++++++----------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index d38c01f0fee..b16ec83be41 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -188,19 +188,17 @@ public void inform(SolrCore core) { String streamQueriesExpressionFile = initArgs.get("streamQueriesExpressionFile"); if (streamQueriesExpressionFile != null) { - System.out.println("got" + streamQueriesExpressionFile); - String exprFile = streamQueriesExpressionFile; - - LineNumberReader bufferedReader = null; - boolean verbose = false; + LineNumberReader bufferedReader; try { bufferedReader = new LineNumberReader( - new InputStreamReader(core.getResourceLoader().openResource(exprFile))); + new InputStreamReader( + core.getResourceLoader().openResource(streamQueriesExpressionFile), + StandardCharsets.UTF_8)); - String args[] = {}; // maybe we have variables? + String[] args = {}; // maybe we have variables? String expr = readExpression(bufferedReader, args); bufferedReader.close(); @@ -222,20 +220,11 @@ public void inform(SolrCore core) { // not sure i need this? Except maybe we assume let? Map params = validateLetAndGetParams(stream, expr); - - } catch (IOException ioe) { throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + exprFile, ioe); - } finally { - - // if (pushBackStream != null) { - //try { - //pushBackStream.close(); - //} catch (IOException e) { - // e.printStackTrace(); - //} - // } + SolrException.ErrorCode.SERVER_ERROR, + "Error reading file " + streamQueriesExpressionFile, + ioe); } } } @@ -320,23 +309,25 @@ protected void processIds( writer.flush(); } - if (streamFactory != null){ - //streamFactory.withFunctionName("stdin", StandardInStream.class); + if (streamFactory != null) { + // streamFactory.withFunctionName("stdin", StandardInStream.class); TupleStream stream = null; - //PushBackStream pushBackStream = null; + // PushBackStream pushBackStream = null; stream = constructStream(streamFactory, streamExpression); - //Map params = validateLetAndGetParams(stream, expr); + // Map params = validateLetAndGetParams(stream, expr); - //pushBackStream = new PushBackStream(stream); + // pushBackStream = new PushBackStream(stream); StreamContext streamContext = new StreamContext(); streamContext.setSolrClientCache(solrClientCache); stream.setStreamContext(streamContext); List tuples = getTuples(stream); - System.out.println("tuples:" + tuples); - //assertEquals(4, tuples.size()); - //pushBackStream.open(); + if (log.isInfoEnabled()) { + log.info("tuples:" + tuples); + } + // assertEquals(4, tuples.size()); + // pushBackStream.open(); } } @@ -362,7 +353,7 @@ protected Tuple getTuple(TupleStream tupleStream) throws IOException { return t; } - public static String readExpression(LineNumberReader bufferedReader, String args[]) + public static String readExpression(LineNumberReader bufferedReader, String[] args) throws IOException { StringBuilder exprBuff = new StringBuilder(); From 6364427d21c2b49d1c041043a778ef57c6f924e7 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 20 May 2024 21:24:52 -0400 Subject: [PATCH 019/103] argh, let the precommit pass --- .../apache/solr/handler/component/UBIComponent.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index b16ec83be41..df3e91f8dab 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -61,7 +61,7 @@ import org.slf4j.LoggerFactory; /** - * User Behavior Insights (UBI) is a open standard for gathering query and event data from users and + * User Behavior Insights (UBI) is an open standard for gathering query and event data from users and * storing it in a structured format. UBI can be used for in session personalization, implicit * judgements, powering recommendation systems among others. Learn more about the UBI standard at https://github.com/o19s/ubi. @@ -182,7 +182,7 @@ public void inform(SolrCore core) { // children.get(0), SolrHighlighter.class, null, // DefaultSolrHighlighter.class.getName()); } - // do i need this check? + // do I need this check? if (initArgs != null) { log.info("Initializing UBIComponent"); String streamQueriesExpressionFile = initArgs.get("streamQueriesExpressionFile"); @@ -217,7 +217,7 @@ public void inform(SolrCore core) { TupleStream stream = constructStream(streamFactory, streamExpression); - // not sure i need this? Except maybe we assume let? + // not sure if I need this? Except maybe, we assume let? Map params = validateLetAndGetParams(stream, expr); } catch (IOException ioe) { @@ -323,9 +323,7 @@ protected void processIds( streamContext.setSolrClientCache(solrClientCache); stream.setStreamContext(streamContext); List tuples = getTuples(stream); - if (log.isInfoEnabled()) { - log.info("tuples:" + tuples); - } + // assertEquals(4, tuples.size()); // pushBackStream.open(); } From 01ee6c723b2fe967d87cd5fb7c87d0cf6b8ab7b4 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 21 May 2024 06:22:43 -0400 Subject: [PATCH 020/103] tidy! --- .../org/apache/solr/handler/component/UBIComponent.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index df3e91f8dab..357384c22ae 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -61,8 +61,8 @@ import org.slf4j.LoggerFactory; /** - * User Behavior Insights (UBI) is an open standard for gathering query and event data from users and - * storing it in a structured format. UBI can be used for in session personalization, implicit + * User Behavior Insights (UBI) is an open standard for gathering query and event data from users + * and storing it in a structured format. UBI can be used for in session personalization, implicit * judgements, powering recommendation systems among others. Learn more about the UBI standard at https://github.com/o19s/ubi. * @@ -323,7 +323,7 @@ protected void processIds( streamContext.setSolrClientCache(solrClientCache); stream.setStreamContext(streamContext); List tuples = getTuples(stream); - + // assertEquals(4, tuples.size()); // pushBackStream.open(); } From 994bd343ba06bf251565aee6585ff0c98524ba36 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 21 May 2024 14:32:57 -0400 Subject: [PATCH 021/103] making user_query as a string, and then introducing query_attributes as a new variable --- .../solr/handler/component/UBIComponent.java | 23 ++++++++------ .../UBIComponentStreamingQueriesTest.java | 9 +++--- .../handler/component/UBIComponentTest.java | 31 ++++++++++++------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 357384c22ae..ff6d3c19077 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -129,6 +129,7 @@ public class UBIComponent extends SearchComponent implements SolrCoreAware { public static final String COMPONENT_NAME = "ubi"; public static final String QUERY_ID = "query_id"; + public static final String QUERY_ATTRIBUTES = "query_attributes"; public static final String USER_QUERY = "user_query"; public static final String UBI_QUERY_JSONL_LOG = "ubi_queries.jsonl"; @@ -253,9 +254,11 @@ public void process(ResponseBuilder rb) throws IOException { } // See if the user passed in a user query as a query parameter - Object userQuery = params.get(USER_QUERY); + String userQuery = params.get(USER_QUERY); - if (userQuery != null && userQuery.toString().startsWith("{")) { + Object queryAttributes = params.get(QUERY_ATTRIBUTES); + + if (queryAttributes != null && queryAttributes.toString().startsWith("{")) { // Look up the original nested JSON format, typically passed in // via the JSON formatted query. @SuppressWarnings("rawtypes") @@ -263,8 +266,8 @@ public void process(ResponseBuilder rb) throws IOException { if (jsonProperties.containsKey("params")) { @SuppressWarnings("rawtypes") Map paramsProperties = (Map) jsonProperties.get("params"); - if (paramsProperties.containsKey("user_query")) { - userQuery = paramsProperties.get("user_query"); + if (paramsProperties.containsKey(QUERY_ATTRIBUTES)) { + queryAttributes = paramsProperties.get(QUERY_ATTRIBUTES); } } } @@ -273,14 +276,15 @@ public void process(ResponseBuilder rb) throws IOException { DocList docs = rc.getDocList(); - processIds(rb, docs, queryId, userQuery, schema, searcher); + processIds(rb, docs, queryId, userQuery, queryAttributes, schema, searcher); } protected void processIds( ResponseBuilder rb, DocList dl, String queryId, - Object userQuery, + String userQuery, + Object queryAttributes, IndexSchema schema, SolrIndexSearcher searcher) throws IOException { @@ -295,11 +299,12 @@ protected void processIds( String docIds = sb.length() > 0 ? sb.substring(0, sb.length() - 1) : ""; SimpleOrderedMap ubiResponseInfo = new SimpleOrderedMap<>(); SimpleOrderedMap ubiQueryLogInfo = new SimpleOrderedMap<>(); - ubiResponseInfo.add("query_id", queryId); + ubiResponseInfo.add(QUERY_ID, queryId); rb.rsp.add("ubi", ubiResponseInfo); - ubiQueryLogInfo.add("query_id", queryId); - ubiQueryLogInfo.add("user_query", userQuery); + ubiQueryLogInfo.add(QUERY_ID, queryId); + ubiQueryLogInfo.add(USER_QUERY, userQuery); + ubiQueryLogInfo.add(QUERY_ATTRIBUTES, queryAttributes); ubiQueryLogInfo.add("doc_ids", docIds); if (writer != null) { diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java index 7988452a318..426197ba135 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java @@ -125,9 +125,9 @@ public void testRandomDocs() throws Exception { // query our collection to generate a UBI event and then confirm it was recorded. - Map userQuery = new HashMap(); - userQuery.put("query", "hot air"); - userQuery.put("results_wanted", 1); + String userQuery = "hot air"; + Map queryAttributes = new HashMap(); + queryAttributes.put("results_wanted", 1); final JsonQueryRequest req = new JsonQueryRequest() @@ -135,7 +135,8 @@ public void testRandomDocs() throws Exception { .setLimit(1) .withParam("ubi", "true") .withParam("query_id", "123") - .withParam("user_query", userQuery); + .withParam("user_query", userQuery) + .withParam("query_attributes", queryAttributes); // Randomly grab a client, it shouldn't matter which is used to generate the query event. SolrClient client = getRandClient(); diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java index be8523d8b6c..c6d0dcdb53e 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java @@ -90,13 +90,21 @@ public void testZeroResults() { } @Test - public void testExternallyGeneratedQueryId() { + public void testPassedInQueryId() { assertQ( - "Make sure we generate a query id", + "Make sure we reuse a passed in query id", req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true", "query_id", "123abc"), "//lst[@name='ubi']/str[@name='query_id'][.='123abc']"); } + @Test + public void testGenerateQueryId() { + assertQ( + "Make sure we reuse a passed in query id", + req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true"), + "//lst[@name='ubi']/str[@name='query_id'][.='1234']"); + } + @Test public void testJSONQuerySyntax() throws Exception { // need to fix this. It just checks the number of docs, @@ -114,8 +122,8 @@ public void testJSONQuerySyntax() throws Exception { + " 'limit': 2,\n" + " 'params': {\n" + " 'df': 'subject',\n" - + " 'qt': '/withubi',\n" - + " 'ubi': 'true'\n" + + " 'qt': '/withubi',\n" + + " 'ubi': 'true'\n" + " }\n" + "}"), "response/numFound==3", @@ -123,8 +131,9 @@ public void testJSONQuerySyntax() throws Exception { String lastLine = readLastLineOfFile(ubiQueriesLog); - String json = "{\"query_id\":\"1234\",\"user_query\":null,\"doc_ids\":\"1,two\"}"; - assertJSONEquals(json, lastLine); + String jsonlLogLine = + "{\"query_id\":\"1234\",\"user_query\":null,\"query_attributes\":null,\"doc_ids\":\"1,two\"}"; + assertJSONEquals(jsonlLogLine, lastLine); assertJQ( req( @@ -140,8 +149,8 @@ public void testJSONQuerySyntax() throws Exception { + " 'df': 'subject',\n" + " 'ubi': 'true',\n" + " 'query_id': 'xjy-42-1rj'\n" - + " 'user_query': {\n" - + " 'query': 'aa',\n" + + " 'user_query': 'aa'\n" + + " 'query_attributes': {\n" + " 'page': 2,\n" + " 'filter': 'inStock:true',\n" + " }\n" @@ -152,9 +161,9 @@ public void testJSONQuerySyntax() throws Exception { lastLine = readLastLineOfFile(ubiQueriesLog); - json = - "{\"query_id\":\"xjy-42-1rj\",\"user_query\":{\"query\":\"aa\",\"page\":2,\"filter\":\"inStock:true\"},\"doc_ids\":\"1,two\"}"; - assertJSONEquals(json, lastLine); + jsonlLogLine = + "{\"query_id\":\"xjy-42-1rj\",\"user_query\":\"aa\",\"query_attributes\":{\"page\":2,\"filter\":\"inStock:true\"},\"doc_ids\":\"1,two\"}"; + assertJSONEquals(jsonlLogLine, lastLine); } @Test From b1fc34e7d428201778234c1da9cadc955dfc4f2e Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 22 May 2024 12:35:55 -0400 Subject: [PATCH 022/103] Streaming expression dummy expression is now running --- .../solr/handler/component/UBIComponent.java | 34 ++++---- .../ubi-enabled/conf/ubi-query-pipeline.expr | 8 +- .../UBIComponentStreamingQueriesTest.java | 77 ++++++++++++++++++- 3 files changed, 97 insertions(+), 22 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index ff6d3c19077..c240dfa9655 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -161,6 +161,9 @@ public void inform(SolrCore core) { this.solrClientCache = coreContainer.getSolrClientCache(); if (coreContainer.isZooKeeperAware()) { defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress(); + } else { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "UBI is SolrCloud only feature."); } if (children.isEmpty()) { @@ -174,15 +177,8 @@ public void inform(SolrCore core) { exception); } writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8); - // DefaultSolrHighlighter defHighlighter = new DefaultSolrHighlighter(core); - // defHighlighter.init(PluginInfo.EMPTY_INFO); - // solrConfigHighlighter = defHighlighter; - } else { - // solrConfigHighlighter = - // core.createInitInstance( - // children.get(0), SolrHighlighter.class, null, - // DefaultSolrHighlighter.class.getName()); } + // do I need this check? if (initArgs != null) { log.info("Initializing UBIComponent"); @@ -207,12 +203,8 @@ public void inform(SolrCore core) { streamExpression = StreamExpressionParser.parse(expr); streamFactory = new StreamFactory(); - String defaultZk = null; - String[] outputHeaders = null; - String delim = " "; - String arrayDelim = "|"; - boolean includeHeaders = false; streamFactory.withDefaultZkHost(defaultZkhost); + // streamFactory.withCollectionZkHost("ubi", zkHost); Lang.register(streamFactory); @@ -316,19 +308,27 @@ protected void processIds( if (streamFactory != null) { // streamFactory.withFunctionName("stdin", StandardInStream.class); + StreamContext streamContext = new StreamContext(); + streamContext.setSolrClientCache(solrClientCache); TupleStream stream = null; // PushBackStream pushBackStream = null; - stream = constructStream(streamFactory, streamExpression); + // stream = constructStream(streamFactory, streamExpression); + stream = streamFactory.constructStream(streamExpression); + stream.setStreamContext(streamContext); // Map params = validateLetAndGetParams(stream, expr); // pushBackStream = new PushBackStream(stream); - StreamContext streamContext = new StreamContext(); - streamContext.setSolrClientCache(solrClientCache); - stream.setStreamContext(streamContext); List tuples = getTuples(stream); + for (Tuple tuple : tuples) { + System.out.println(tuple.getString("worker")); + System.out.println(tuple.getString("totalIndexed")); + } + + System.out.println("Total tuples returned " + tuples.size()); + // assertEquals(4, tuples.size()); // pushBackStream.open(); } diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr index 4b132ee99a5..58337e63315 100644 --- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr +++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr @@ -13,8 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +#commit(ubi, +# update(ubi, +# tuple(id=49,a_i=1,b_i=5) +# ) +#) + let(cli-zkhost="localhost:9983", - commit(ubi, + commit(ubi, update(ubi, tuple(id=49,a_i=1,b_i=5)) ) diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java index 426197ba135..a8ff9f6e1d6 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java @@ -17,12 +17,20 @@ package org.apache.solr.handler.component; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.lucene.util.IOUtils; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.io.Lang; +import org.apache.solr.client.solrj.io.SolrClientCache; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.stream.StreamContext; +import org.apache.solr.client.solrj.io.stream.TupleStream; +import org.apache.solr.client.solrj.io.stream.UpdateStream; +import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.json.JsonQueryRequest; @@ -53,6 +61,8 @@ public class UBIComponentStreamingQueriesTest extends SolrCloudTestCase { */ private static final List CLIENTS = new ArrayList<>(7); + private static String zkHost; + @BeforeClass public static void setupCluster() throws Exception { System.setProperty("solr.log.dir", createTempDir("solr_logs").toString()); @@ -69,6 +79,8 @@ public static void setupCluster() throws Exception { .addConfig("ubi", configset("_default")) .configure(); + zkHost = cluster.getZkServer().getZkAddress(); + CLIENTS.add(cluster.getSolrClient()); for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { final SolrClient c = getHttpSolrClient(jetty.getBaseUrl().toString()); @@ -112,14 +124,71 @@ public void clearCollection() throws Exception { assertEquals("commit failed", 0, cluster.getSolrClient().commit(COLLECTION).getStatus()); } + public void testUsingStreamingExpressionDirectly() throws Exception { + TupleStream stream; + List tuples; + StreamContext streamContext = new StreamContext(); + SolrClientCache solrClientCache = new SolrClientCache(); + + streamContext.setSolrClientCache(solrClientCache); + + StreamFactory streamFactory = new StreamFactory().withCollectionZkHost("ubi", zkHost); + + Lang.register(streamFactory); + + String clause = getClause(); + stream = streamFactory.constructStream(clause); + stream.setStreamContext(streamContext); + tuples = getTuples(stream); + stream.close(); + solrClientCache.close(); + + assertEquals("Total tuples returned",1,tuples.size()); + Tuple tuple = tuples.get(0); + assertEquals("1",tuple.getString(UpdateStream.BATCH_INDEXED_FIELD_NAME)); + assertEquals("1",tuple.getString("totalIndexed")); + + // Check the UBI collection + final JsonQueryRequest requestUBI = new JsonQueryRequest().setQuery("id:4.0").setLimit(1); + + // Randomly grab a client, it shouldn't matter which is used to check UBI event. + SolrClient client = getRandClient(); + final QueryResponse responseUBI = requestUBI.process(client, UBI_COLLECTION); + try { + assertEquals(0, responseUBI.getStatus()); + assertEquals(1, responseUBI.getResults().getNumFound()); + } catch (AssertionError e) { + throw new AssertionError(responseUBI + " + " + client + " => " + e.getMessage(), e); + } + } + + private List getTuples(TupleStream tupleStream) throws IOException { + tupleStream.open(); + List tuples = new ArrayList<>(); + for (; ; ) { + Tuple t = tupleStream.read(); + // log.info(" ... {}", t.fields); + if (t.EOF) { + break; + } else { + tuples.add(t); + } + } + tupleStream.close(); + return tuples; + } + + private static String getClause() { + String clause = "commit(ubi,update(ubi,tuple(id=add(1,3), name_s=bob)))"; + return clause; + } + @SuppressWarnings({"rawtypes", "unchecked"}) public void testRandomDocs() throws Exception { final UpdateRequest ureq = new UpdateRequest(); - for (int i = 0; i < 1; i++) { - ureq.add(sdoc("id", i, "data_s", "data:" + i)); - } + ureq.add(sdoc("id", 1, "data_s", "data_1")); assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION).getStatus()); assertEquals("commit failed", 0, getRandClient().commit(COLLECTION).getStatus()); @@ -131,7 +200,7 @@ public void testRandomDocs() throws Exception { final JsonQueryRequest req = new JsonQueryRequest() - .setQuery("id:1") + .setQuery("*:*") .setLimit(1) .withParam("ubi", "true") .withParam("query_id", "123") From 607588bbafb57d359da697252c44366d5239ee14 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 22 May 2024 13:18:41 -0400 Subject: [PATCH 023/103] tidy --- .../apache/solr/handler/component/UBIComponent.java | 10 ---------- .../component/UBIComponentStreamingQueriesTest.java | 6 +++--- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index c240dfa9655..14885e105d5 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -321,16 +321,6 @@ protected void processIds( // pushBackStream = new PushBackStream(stream); List tuples = getTuples(stream); - - for (Tuple tuple : tuples) { - System.out.println(tuple.getString("worker")); - System.out.println(tuple.getString("totalIndexed")); - } - - System.out.println("Total tuples returned " + tuples.size()); - - // assertEquals(4, tuples.size()); - // pushBackStream.open(); } } diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java index a8ff9f6e1d6..1772a157d9d 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java @@ -143,10 +143,10 @@ public void testUsingStreamingExpressionDirectly() throws Exception { stream.close(); solrClientCache.close(); - assertEquals("Total tuples returned",1,tuples.size()); + assertEquals("Total tuples returned", 1, tuples.size()); Tuple tuple = tuples.get(0); - assertEquals("1",tuple.getString(UpdateStream.BATCH_INDEXED_FIELD_NAME)); - assertEquals("1",tuple.getString("totalIndexed")); + assertEquals("1", tuple.getString(UpdateStream.BATCH_INDEXED_FIELD_NAME)); + assertEquals("1", tuple.getString("totalIndexed")); // Check the UBI collection final JsonQueryRequest requestUBI = new JsonQueryRequest().setQuery("id:4.0").setLimit(1); From 7d1040374fd6152752b09a59ac5346deaa49e070 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 22 May 2024 15:39:08 -0400 Subject: [PATCH 024/103] allow ubi to run in non solrcloud mode.. boo --- .../solr/handler/component/UBIComponent.java | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 14885e105d5..2de83a25627 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -161,9 +161,6 @@ public void inform(SolrCore core) { this.solrClientCache = coreContainer.getSolrClientCache(); if (coreContainer.isZooKeeperAware()) { defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress(); - } else { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "UBI is SolrCloud only feature."); } if (children.isEmpty()) { @@ -182,43 +179,47 @@ public void inform(SolrCore core) { // do I need this check? if (initArgs != null) { log.info("Initializing UBIComponent"); - String streamQueriesExpressionFile = initArgs.get("streamQueriesExpressionFile"); + if (coreContainer.isZooKeeperAware()) { + String streamQueriesExpressionFile = initArgs.get("streamQueriesExpressionFile"); - if (streamQueriesExpressionFile != null) { + if (streamQueriesExpressionFile != null) { - LineNumberReader bufferedReader; + LineNumberReader bufferedReader; - try { - bufferedReader = - new LineNumberReader( - new InputStreamReader( - core.getResourceLoader().openResource(streamQueriesExpressionFile), - StandardCharsets.UTF_8)); + try { + bufferedReader = + new LineNumberReader( + new InputStreamReader( + core.getResourceLoader().openResource(streamQueriesExpressionFile), + StandardCharsets.UTF_8)); - String[] args = {}; // maybe we have variables? - String expr = readExpression(bufferedReader, args); + String[] args = {}; // maybe we have variables? + String expr = readExpression(bufferedReader, args); - bufferedReader.close(); + bufferedReader.close(); - streamExpression = StreamExpressionParser.parse(expr); - streamFactory = new StreamFactory(); + streamExpression = StreamExpressionParser.parse(expr); + streamFactory = new StreamFactory(); - streamFactory.withDefaultZkHost(defaultZkhost); - // streamFactory.withCollectionZkHost("ubi", zkHost); + streamFactory.withDefaultZkHost(defaultZkhost); + // streamFactory.withCollectionZkHost("ubi", zkHost); - Lang.register(streamFactory); + Lang.register(streamFactory); - TupleStream stream = constructStream(streamFactory, streamExpression); + TupleStream stream = constructStream(streamFactory, streamExpression); - // not sure if I need this? Except maybe, we assume let? - Map params = validateLetAndGetParams(stream, expr); + // not sure if I need this? Except maybe, we assume let? + Map params = validateLetAndGetParams(stream, expr); - } catch (IOException ioe) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Error reading file " + streamQueriesExpressionFile, - ioe); + } catch (IOException ioe) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Error reading file " + streamQueriesExpressionFile, + ioe); + } } + } else { + log.info("Streaming UBI query data collection is only available in SolrCloud mode."); } } } From 7a365c1b62d8751a252c01a61e386ddddaeedc99 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 May 2024 08:45:31 -0400 Subject: [PATCH 025/103] Introduce an explicit UBIQuery class to wrap up the data required.. --- .../solr/handler/component/UBIComponent.java | 28 ++++----- .../solr/handler/component/UBIQuery.java | 61 +++++++++++++++++++ .../UBIComponentStreamingQueriesTest.java | 34 ++++++++++- 3 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 2de83a25627..4a340816e80 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -163,6 +163,7 @@ public void inform(SolrCore core) { defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress(); } + // we should provide a way to specify your own file. if (children.isEmpty()) { String ubiQueryJSONLLog = EnvUtils.getProperty("solr.log.dir") + "/" + UBI_QUERY_JSONL_LOG; try { @@ -240,14 +241,9 @@ public void process(ResponseBuilder rb) throws IOException { return; } - String queryId = params.get(QUERY_ID, null); + UBIQuery ubiQuery = new UBIQuery(params.get(QUERY_ID)); - if (queryId == null) { - queryId = "1234"; - } - - // See if the user passed in a user query as a query parameter - String userQuery = params.get(USER_QUERY); + ubiQuery.setUserQuery(params.get(USER_QUERY)); Object queryAttributes = params.get(QUERY_ATTRIBUTES); @@ -261,6 +257,7 @@ public void process(ResponseBuilder rb) throws IOException { Map paramsProperties = (Map) jsonProperties.get("params"); if (paramsProperties.containsKey(QUERY_ATTRIBUTES)) { queryAttributes = paramsProperties.get(QUERY_ATTRIBUTES); + ubiQuery.setQueryAttributes(queryAttributes); } } } @@ -269,15 +266,13 @@ public void process(ResponseBuilder rb) throws IOException { DocList docs = rc.getDocList(); - processIds(rb, docs, queryId, userQuery, queryAttributes, schema, searcher); + processIds(rb, docs, ubiQuery, schema, searcher); } protected void processIds( ResponseBuilder rb, DocList dl, - String queryId, - String userQuery, - Object queryAttributes, + UBIQuery ubiQuery, IndexSchema schema, SolrIndexSearcher searcher) throws IOException { @@ -292,12 +287,13 @@ protected void processIds( String docIds = sb.length() > 0 ? sb.substring(0, sb.length() - 1) : ""; SimpleOrderedMap ubiResponseInfo = new SimpleOrderedMap<>(); SimpleOrderedMap ubiQueryLogInfo = new SimpleOrderedMap<>(); - ubiResponseInfo.add(QUERY_ID, queryId); + ubiResponseInfo.add(QUERY_ID, ubiQuery.getQueryId()); rb.rsp.add("ubi", ubiResponseInfo); - ubiQueryLogInfo.add(QUERY_ID, queryId); - ubiQueryLogInfo.add(USER_QUERY, userQuery); - ubiQueryLogInfo.add(QUERY_ATTRIBUTES, queryAttributes); + // Maybe ubiQueryLogInfo should be a ubiQuery? But what about the doc_ids? + ubiQueryLogInfo.add(QUERY_ID, ubiQuery.getQueryId()); + ubiQueryLogInfo.add(USER_QUERY, ubiQuery.getUserQuery()); + ubiQueryLogInfo.add(QUERY_ATTRIBUTES, ubiQuery.getQueryAttributes()); ubiQueryLogInfo.add("doc_ids", docIds); if (writer != null) { @@ -311,7 +307,7 @@ protected void processIds( // streamFactory.withFunctionName("stdin", StandardInStream.class); StreamContext streamContext = new StreamContext(); streamContext.setSolrClientCache(solrClientCache); - TupleStream stream = null; + TupleStream stream; // PushBackStream pushBackStream = null; // stream = constructStream(streamFactory, streamExpression); stream = streamFactory.constructStream(streamExpression); diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java new file mode 100644 index 00000000000..d3789982e66 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java @@ -0,0 +1,61 @@ +package org.apache.solr.handler.component; + +/** + * Handles all the data required for tracking a query using User Behavior Insights. + * + *

Compatible with the + * https://github.com/o19s/ubi/blob/main/schema/X.Y.Z/query.request.schema.json. + */ +public class UBIQuery { + + private String queryId; + private String userQuery; + private Object queryAttributes; + + public UBIQuery(String queryId) { + + if (queryId == null) { + queryId = "1234"; + } + this.queryId = queryId; + } + + public String getQueryId() { + return queryId; + } + + public void setQueryId(String queryId) { + this.queryId = queryId; + } + + public String getUserQuery() { + return userQuery; + } + + public void setUserQuery(String userQuery) { + this.userQuery = userQuery; + } + + public Object getQueryAttributes() { + return queryAttributes; + } + + public void setQueryAttributes(Object queryAttributes) { + this.queryAttributes = queryAttributes; + } + + /** + * Convert the UBIQuery into the format consumed by a streaming expression tuple() + * + * @return String The tuple specific formatted data similar to "query_id=123,user_query=foo" + */ + public String toTuple() { + return UBIComponent.QUERY_ID + + "=" + + this.queryId + + "," + + UBIComponent.USER_QUERY + + "=" + + this.userQuery; + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java index 1772a157d9d..1ac4b1815b9 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java @@ -18,6 +18,9 @@ package org.apache.solr.handler.component; import java.io.IOException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -122,9 +125,31 @@ public void clearCollection() throws Exception { assertEquals( "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(COLLECTION, "*:*").getStatus()); assertEquals("commit failed", 0, cluster.getSolrClient().commit(COLLECTION).getStatus()); + assertEquals( + "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(UBI_COLLECTION, "*:*").getStatus()); + assertEquals("commit failed", 0, cluster.getSolrClient().commit(UBI_COLLECTION).getStatus()); + } + + public void testWritingStreamingExpression() { + UBIQuery ubiQuery = new UBIQuery("5678"); + ubiQuery.setUserQuery("Apple Memory"); + + String clause = getClause(ubiQuery); + assertEquals( + "Check the decoded version for ease of comparison", + "commit(ubi,update(ubi,tuple(id=4.0,query_id=5678,user_query=Apple Memory)))", + URLDecoder.decode(clause, StandardCharsets.UTF_8)); + assertEquals( + "Verify the encoded version", + "commit%28ubi%2Cupdate%28ubi%2Ctuple%28id%3D4.0%2Cquery_id%3D5678%2Cuser_query%3DApple+Memory%29%29%29", + clause); } public void testUsingStreamingExpressionDirectly() throws Exception { + + UBIQuery ubiQuery = new UBIQuery("5678"); + ubiQuery.setUserQuery("Apple Memory"); + TupleStream stream; List tuples; StreamContext streamContext = new StreamContext(); @@ -136,7 +161,7 @@ public void testUsingStreamingExpressionDirectly() throws Exception { Lang.register(streamFactory); - String clause = getClause(); + String clause = getClause(ubiQuery); stream = streamFactory.constructStream(clause); stream.setStreamContext(streamContext); tuples = getTuples(stream); @@ -178,7 +203,14 @@ private List getTuples(TupleStream tupleStream) throws IOException { return tuples; } + private static String getClause(UBIQuery ubiQuery) { + String clause = "commit(ubi,update(ubi,tuple(id=4.0," + ubiQuery.toTuple() + ")))"; + clause = URLEncoder.encode(clause, StandardCharsets.UTF_8); + return clause; + } + private static String getClause() { + String clause = "commit(ubi,update(ubi,tuple(id=add(1,3), name_s=bob)))"; return clause; } From cecd0754387c623d68344dafa4202aa1164af9a9 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 16 Aug 2024 16:44:01 -0400 Subject: [PATCH 026/103] some weridness in merging... --- .../cloud-minimal-field-limiting/conf/solrconfig.xml | 11 +---------- .../conf-using-mypkg-version-1/conf/solrconfig.xml | 9 --------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml index 13fcc24432d..d900190659f 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml @@ -34,29 +34,20 @@ ${solr.commitwithin.softcommit:true} - -<<<<<<<< HEAD:solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml - + ubi-query-pipeline.expr -======== - ->>>>>>>> upstream/main:solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml explicit true text -<<<<<<<< HEAD:solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml ubi - -======== ->>>>>>>> upstream/main:solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml diff --git a/solr/core/src/test-files/solr/configsets/conf-using-mypkg-version-1/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/conf-using-mypkg-version-1/conf/solrconfig.xml index 7f2a53854df..1c999fdc8a6 100644 --- a/solr/core/src/test-files/solr/configsets/conf-using-mypkg-version-1/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/conf-using-mypkg-version-1/conf/solrconfig.xml @@ -15,14 +15,6 @@ See the License for the specific language governing permissions and limitations under the License. --> -<<<<<<<< HEAD:solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml - - - - - id - -======== @@ -40,4 +32,3 @@ ->>>>>>>> upstream/main:solr/core/src/test-files/solr/configsets/conf-using-mypkg-version-1/conf/solrconfig.xml From cb8a2d1cbfceea875648a37e736de132c182a19b Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 16 Aug 2024 16:49:04 -0400 Subject: [PATCH 027/103] deal with merge changes.. --- .../configsets/ubi-enabled/conf/schema.xml | 21 +------------------ .../ubi-enabled/conf/solrconfig.xml | 8 ------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml index 7f2a53854df..661b02a0f96 100644 --- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml @@ -15,29 +15,10 @@ See the License for the specific language governing permissions and limitations under the License. --> -<<<<<<<< HEAD:solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml + id -======== - - - - ${solr.data.dir:} - - - - ${tests.luceneMatchVersion:LATEST} - - - - - - - - ->>>>>>>> upstream/main:solr/core/src/test-files/solr/configsets/conf-using-mypkg-version-1/conf/solrconfig.xml diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml index 13fcc24432d..62cdc180c57 100644 --- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml @@ -35,28 +35,20 @@ -<<<<<<<< HEAD:solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml ubi-query-pipeline.expr -======== - ->>>>>>>> upstream/main:solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml explicit true text -<<<<<<<< HEAD:solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml ubi - -======== ->>>>>>>> upstream/main:solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml From 3f79110c3044ed11e2722c0f6410ea706b36495e Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 16 Aug 2024 16:51:28 -0400 Subject: [PATCH 028/103] we don't need this as part of UBI, so don't add it... --- .../org/apache/solr/client/solrj/io/stream/LetStream.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java index fc26a8972f7..9576cf9658e 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java @@ -224,9 +224,4 @@ public StreamComparator getStreamSort() { public int getCost() { return 0; } - - @SuppressWarnings({"rawtypes"}) - public Map getLetParams() { - return this.letParams; - } } From 24507344a045a1230b221f9e771d1cc377fb3633 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 16 Aug 2024 16:54:27 -0400 Subject: [PATCH 029/103] we aren't using ubi with this minimal example --- .../cloud-minimal-field-limiting/conf/solrconfig.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml index d900190659f..4cfd6a26a9d 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml @@ -35,9 +35,6 @@ - - ubi-query-pipeline.expr - @@ -45,9 +42,6 @@ true text - - ubi - From d3b5cf3d8355a46b3bbea1ca82adb27d64833103 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 16 Aug 2024 17:10:22 -0400 Subject: [PATCH 030/103] lint --- .../solr/handler/component/UBIComponent.java | 22 +++++++++---------- .../solr/handler/component/UBIQuery.java | 18 ++++++++++++++- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 4a340816e80..73a66e7e63a 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -35,7 +35,6 @@ import org.apache.solr.client.solrj.io.Lang; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; -import org.apache.solr.client.solrj.io.stream.LetStream; import org.apache.solr.client.solrj.io.stream.StreamContext; import org.apache.solr.client.solrj.io.stream.TupleStream; import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; @@ -210,7 +209,7 @@ public void inform(SolrCore core) { TupleStream stream = constructStream(streamFactory, streamExpression); // not sure if I need this? Except maybe, we assume let? - Map params = validateLetAndGetParams(stream, expr); + // Map params = validateLetAndGetParams(stream, expr); } catch (IOException ioe) { throw new SolrException( @@ -389,16 +388,17 @@ public static TupleStream constructStream( return streamFactory.constructStream(streamExpression); } - @SuppressWarnings({"rawtypes"}) - public static Map validateLetAndGetParams(TupleStream stream, String expr) throws IOException { - if (stream instanceof LetStream) { - LetStream mainStream = (LetStream) stream; - return mainStream.getLetParams(); - } else { - throw new IOException("No enclosing let function found in expression:" + expr); + /* + @SuppressWarnings({"rawtypes"}) + public static Map validateLetAndGetParams(TupleStream stream, String expr) throws IOException { + if (stream instanceof LetStream) { + LetStream mainStream = (LetStream) stream; + return mainStream.getLetParams(); + } else { + throw new IOException("No enclosing let function found in expression:" + expr); + } } - } - + */ @Override public String getDescription() { return "A component that tracks original user query and the resulting documents returned."; diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java index d3789982e66..086a1ef22da 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java @@ -1,10 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.solr.handler.component; /** * Handles all the data required for tracking a query using User Behavior Insights. * *

Compatible with the - * https://github.com/o19s/ubi/blob/main/schema/X.Y.Z/query.request.schema.json. + * https://github.com/o19s/ubi/blob/main/schema/1.0.0/query.request.schema.json. */ public class UBIQuery { From 81faf198041f36a3a5a57c44f645146b2c2d8c90 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 16 Aug 2024 17:45:17 -0400 Subject: [PATCH 031/103] clean up the tests... --- .../handler/component/UBIComponentTest.java | 62 +++---------------- 1 file changed, 10 insertions(+), 52 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java index c6d0dcdb53e..a2dad6e8b91 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java @@ -22,7 +22,6 @@ import org.apache.commons.io.input.ReversedLinesFileReader; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.request.SolrQueryRequest; import org.junit.BeforeClass; import org.junit.Test; @@ -48,45 +47,20 @@ public static void beforeTest() throws Exception { @Test public void testToLogIds() { - SolrQueryRequest req = null; - try { - String handler = "/withubi"; - req = req("qt", "/withubi", "q", "aa", "rows", "2", "ubi", "true"); - - assertQ( - "Make sure we generate a query id", - req, - "//lst[@name='ubi']/str[@name='query_id'][.='1234']"); - // Need to test the writing out to the logs.. - // SolrQueryResponse qr = h.queryAndResponse(handler, req); - // NamedList entries = qr.getToLog(); - // String docIds = (String) entries.get("ubi"); - // assertNotNull(docIds); - // assertTrue(docIds.matches("\\w+,\\w+")); - } finally { - req.close(); - } + + assertQ( + "Make sure we generate a query id", + req("qt", "/withubi", "q", "aa", "rows", "2", "ubi", "true"), + "//lst[@name='ubi']/str[@name='query_id'][.='1234']"); } @Test public void testZeroResults() { - // SolrQueryRequest req = null; - // try { - // String handler = "/withubi"; - // req = req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true"); - assertQ( - "Make sure we generate a query id", - req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true"), + "Make sure we generate a query id even when no results are returned", + req("qt", "/withubi", "q", "abcdefgxyz", "rows", "0", "ubi", "true"), + "//*[@numFound='0']", "//lst[@name='ubi']/str[@name='query_id'][.='1234']"); - - // SolrQueryResponse qr = h.queryAndResponse(handler, req); - // NamedList entries = qr.getToLog(); - // String docIds = (String) entries.get("ubi"); - // assertNull(docIds); - // } finally { - // req.close(); - // } } @Test @@ -100,16 +74,13 @@ public void testPassedInQueryId() { @Test public void testGenerateQueryId() { assertQ( - "Make sure we reuse a passed in query id", + "Make sure we generate a query id if one is not passed in", req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true"), "//lst[@name='ubi']/str[@name='query_id'][.='1234']"); } @Test public void testJSONQuerySyntax() throws Exception { - // need to fix this. It just checks the number of docs, - // but doesn't do anything about the ubi clauses... - // doesn't appear to trigger the ubi... assertJQ( req( "qt", @@ -176,29 +147,16 @@ public void testTrackingOfUserQuery() { @Test public void testDisabling() { - // SolrQueryRequest req = null; - // try { - // String handler = "/withubi"; - assertQ( "Make sure we don't generate a query_id", req("qt", "/withubi", "q", "aa", "ubi", "false"), "count(//lst[@name='ubi'])=0"); - - // SolrQueryResponse qr = h.queryAndResponse(handler, req); - // NamedList entries = qr.getToLog(); - // String ubi = (String) entries.get("ubi"); - // assertNull(ubi); - // } finally { - // req.close(); - // } } private static String readLastLineOfFile(File file) throws IOException { try (ReversedLinesFileReader reader = ReversedLinesFileReader.builder().setFile(file).setCharset(StandardCharsets.UTF_8).get()) { - String line = reader.readLine(); - return line; + return reader.readLine(); } } } From 7c31300653b1a279826a917e51433dcea6a6e3b4 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 19 Aug 2024 08:27:12 -0400 Subject: [PATCH 032/103] Strip out writing to a log file, we have a general purpose streaming.. We can stream to wahtever we want, a local log file, a Solr collection, s3, etc! --- .../solr/handler/component/UBIComponent.java | 89 +++++-------------- .../UBIComponentStreamingQueriesTest.java | 17 +++- .../handler/component/UBIComponentTest.java | 34 ------- 3 files changed, 34 insertions(+), 106 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java index 73a66e7e63a..b42d3efcd97 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java @@ -16,15 +16,9 @@ */ package org.apache.solr.handler.component; -import java.io.BufferedOutputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.LineNumberReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -35,14 +29,12 @@ import org.apache.solr.client.solrj.io.Lang; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; -import org.apache.solr.client.solrj.io.stream.StreamContext; import org.apache.solr.client.solrj.io.stream.TupleStream; import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.CoreContainer; @@ -54,8 +46,6 @@ import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.plugin.SolrCoreAware; -import org.noggit.CharArr; -import org.noggit.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,17 +55,20 @@ * judgements, powering recommendation systems among others. Learn more about the UBI standard at https://github.com/o19s/ubi. * - *

Query data is gathered by this component. Data tracked is the collection name, the end user - * query, as json blob, and the resulting document id's. + *

The response from Solr is augmented by this component, and optionally the query details can be + * tracked and logged to various systems including log files or other backend systems. * - *

Data is written out data to "ubi_queries.jsonl", a JSON with Lines formatted file, or you can - * provide a streaming expression that is parsed and loaded by the component to stream query data to - * a target of your choice. + *

Data tracked is the collection name, the end user query, as a JSON blob, and the resulting + * document id's. + * + *

You provide a streaming expression that is parsed and loaded by the component to stream query + * data to a target of your choice. * *

Event data is tracked by letting the user write events directly to the event repository of - * your choice, it could be a Solr collection, it could be a file or S3 bucket. + * your choice, it could be a Solr collection, it could be a file or S3 bucket, and that is NOT + * handled by this component. * - *

Add it to a requestHandler in solrconfig.xml like this: + *

Add the component to a requestHandler in solrconfig.xml like this: * * 

  * <searchComponent name="ubi" class="solr.UBIComponent"/>
@@ -130,18 +123,14 @@ public class UBIComponent extends SearchComponent implements SolrCoreAware {
   public static final String QUERY_ID = "query_id";
   public static final String QUERY_ATTRIBUTES = "query_attributes";
   public static final String USER_QUERY = "user_query";
-  public static final String UBI_QUERY_JSONL_LOG = "ubi_queries.jsonl";
 
   protected PluginInfo info = PluginInfo.EMPTY_INFO;
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-  private final CharArr charArr = new CharArr(1024 * 2);
-  JSONWriter jsonWriter = new JSONWriter(charArr, -1);
-  private Writer writer;
-  OutputStream fos;
 
   private StreamFactory streamFactory;
   private StreamExpression streamExpression;
+  private TupleStream stream;
 
   protected SolrParams initArgs;
   private SolrClientCache solrClientCache;
@@ -152,37 +141,21 @@ public void init(NamedList args) {
   }
 
   @Override
-  @SuppressWarnings({"rawtypes"})
   public void inform(SolrCore core) {
-    List children = info.getChildren("ubi");
-    String defaultZkhost = null;
     CoreContainer coreContainer = core.getCoreContainer();
     this.solrClientCache = coreContainer.getSolrClientCache();
-    if (coreContainer.isZooKeeperAware()) {
-      defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress();
-    }
-
-    // we should provide a way to specify your own file.
-    if (children.isEmpty()) {
-      String ubiQueryJSONLLog = EnvUtils.getProperty("solr.log.dir") + "/" + UBI_QUERY_JSONL_LOG;
-      try {
-        fos = new BufferedOutputStream(new FileOutputStream(ubiQueryJSONLLog));
-      } catch (FileNotFoundException exception) {
-        throw new SolrException(
-            SolrException.ErrorCode.SERVER_ERROR,
-            "Error creating file  " + ubiQueryJSONLLog,
-            exception);
-      }
-      writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
-    }
 
     // do I need this check?
     if (initArgs != null) {
       log.info("Initializing UBIComponent");
       if (coreContainer.isZooKeeperAware()) {
+        String defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress();
         String streamQueriesExpressionFile = initArgs.get("streamQueriesExpressionFile");
 
-        if (streamQueriesExpressionFile != null) {
+        if (streamQueriesExpressionFile == null) {
+          log.info(
+              "You must provide a streamQueriesExpressionFile to enable recording UBI information.");
+        } else {
 
           LineNumberReader bufferedReader;
 
@@ -202,11 +175,10 @@ public void inform(SolrCore core) {
             streamFactory = new StreamFactory();
 
             streamFactory.withDefaultZkHost(defaultZkhost);
-            // streamFactory.withCollectionZkHost("ubi", zkHost);
 
             Lang.register(streamFactory);
 
-            TupleStream stream = constructStream(streamFactory, streamExpression);
+            stream = constructStream(streamFactory, streamExpression);
 
             // not sure if I need this?  Except maybe, we assume let?
             // Map params = validateLetAndGetParams(stream, expr);
@@ -219,7 +191,7 @@ public void inform(SolrCore core) {
           }
         }
       } else {
-        log.info("Streaming UBI query data collection is only available in SolrCloud mode.");
+        log.info("UBI query data collection is only available in SolrCloud mode.");
       }
     }
   }
@@ -275,7 +247,6 @@ protected void processIds(
       IndexSchema schema,
       SolrIndexSearcher searcher)
       throws IOException {
-    charArr.reset();
     StringBuilder sb = new StringBuilder();
 
     Set fields = Collections.singleton(schema.getUniqueKeyField().getName());
@@ -295,27 +266,8 @@ protected void processIds(
     ubiQueryLogInfo.add(QUERY_ATTRIBUTES, ubiQuery.getQueryAttributes());
     ubiQueryLogInfo.add("doc_ids", docIds);
 
-    if (writer != null) {
-      jsonWriter.write(ubiQueryLogInfo);
-      writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());
-      writer.append('\n');
-      writer.flush();
-    }
-
-    if (streamFactory != null) {
-      // streamFactory.withFunctionName("stdin", StandardInStream.class);
-      StreamContext streamContext = new StreamContext();
-      streamContext.setSolrClientCache(solrClientCache);
-      TupleStream stream;
-      // PushBackStream pushBackStream = null;
-      // stream = constructStream(streamFactory, streamExpression);
-      stream = streamFactory.constructStream(streamExpression);
-      stream.setStreamContext(streamContext);
-
-      // Map params = validateLetAndGetParams(stream, expr);
-
-      // pushBackStream = new PushBackStream(stream);
-
+    // pushBackStream = new PushBackStream(stream);
+    if (stream != null) {
       List tuples = getTuples(stream);
     }
   }
@@ -342,6 +294,7 @@ protected Tuple getTuple(TupleStream tupleStream) throws IOException {
     return t;
   }
 
+  // this should be a shared utility method
   public static String readExpression(LineNumberReader bufferedReader, String[] args)
       throws IOException {
 
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index 1ac4b1815b9..c2e30457a8a 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.solr.handler.component;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URLDecoder;
 import java.net.URLEncoder;
@@ -25,6 +26,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import org.apache.commons.io.input.ReversedLinesFileReader;
 import org.apache.lucene.util.IOUtils;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.io.Lang;
@@ -68,7 +70,6 @@ public class UBIComponentStreamingQueriesTest extends SolrCloudTestCase {
 
   @BeforeClass
   public static void setupCluster() throws Exception {
-    System.setProperty("solr.log.dir", createTempDir("solr_logs").toString());
 
     final int numShards = usually() ? 2 : 1;
     final int numReplicas = usually() ? 2 : 1;
@@ -174,11 +175,11 @@ public void testUsingStreamingExpressionDirectly() throws Exception {
     assertEquals("1", tuple.getString("totalIndexed"));
 
     // Check the UBI collection
-    final JsonQueryRequest requestUBI = new JsonQueryRequest().setQuery("id:4.0").setLimit(1);
+    final JsonQueryRequest requestFromUBICollection = new JsonQueryRequest().setQuery("id:4.0").setLimit(1);
 
     // Randomly grab a client, it shouldn't matter which is used to check UBI event.
     SolrClient client = getRandClient();
-    final QueryResponse responseUBI = requestUBI.process(client, UBI_COLLECTION);
+    final QueryResponse responseUBI = requestFromUBICollection.process(client, UBI_COLLECTION);
     try {
       assertEquals(0, responseUBI.getStatus());
       assertEquals(1, responseUBI.getResults().getNumFound());
@@ -205,7 +206,8 @@ private List getTuples(TupleStream tupleStream) throws IOException {
 
   private static String getClause(UBIQuery ubiQuery) {
     String clause = "commit(ubi,update(ubi,tuple(id=4.0," + ubiQuery.toTuple() + ")))";
-    clause = URLEncoder.encode(clause, StandardCharsets.UTF_8);
+    //String clause = "commit(ubi,update(ubi,tuple(id=4.0)))";
+    //clause = URLEncoder.encode(clause, StandardCharsets.UTF_8);
     return clause;
   }
 
@@ -317,4 +319,11 @@ public void randomDocs() throws Exception {
   private static SolrClient getRandClient() {
     return CLIENTS.get(random().nextInt(CLIENTS.size()));
   }
+
+  private static String readLastLineOfFile(File file) throws IOException {
+    try (ReversedLinesFileReader reader =
+        ReversedLinesFileReader.builder().setFile(file).setCharset(StandardCharsets.UTF_8).get()) {
+      return reader.readLine();
+    }
+  }
 }
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
index a2dad6e8b91..bdfaa513835 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
@@ -16,38 +16,23 @@
  */
 package org.apache.solr.handler.component;
 
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import org.apache.commons.io.input.ReversedLinesFileReader;
 import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.util.EnvUtils;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class UBIComponentTest extends SolrTestCaseJ4 {
 
-  private static File ubiQueriesLog;
-
   @BeforeClass
   public static void beforeTest() throws Exception {
-
-    System.setProperty("solr.log.dir", createTempDir("solr_logs").toString());
-
     initCore("solrconfig-ubi-local-component.xml", "schema12.xml");
     assertNull(h.validateUpdate(adoc("id", "1", "subject", "aa")));
     assertNull(h.validateUpdate(adoc("id", "two", "subject", "aa")));
     assertNull(h.validateUpdate(adoc("id", "3", "subject", "aa")));
     assertU(commit());
-
-    ubiQueriesLog =
-        new File(EnvUtils.getProperty("solr.log.dir") + "/" + UBIComponent.UBI_QUERY_JSONL_LOG);
-    assertTrue(ubiQueriesLog.exists());
   }
 
   @Test
   public void testToLogIds() {
-
     assertQ(
         "Make sure we generate a query id",
         req("qt", "/withubi", "q", "aa", "rows", "2", "ubi", "true"),
@@ -100,12 +85,6 @@ public void testJSONQuerySyntax() throws Exception {
         "response/numFound==3",
         "ubi/query_id=='1234'");
 
-    String lastLine = readLastLineOfFile(ubiQueriesLog);
-
-    String jsonlLogLine =
-        "{\"query_id\":\"1234\",\"user_query\":null,\"query_attributes\":null,\"doc_ids\":\"1,two\"}";
-    assertJSONEquals(jsonlLogLine, lastLine);
-
     assertJQ(
         req(
             "qt",
@@ -129,12 +108,6 @@ public void testJSONQuerySyntax() throws Exception {
                 + "}"),
         "response/numFound==3",
         "ubi/query_id=='xjy-42-1rj'");
-
-    lastLine = readLastLineOfFile(ubiQueriesLog);
-
-    jsonlLogLine =
-        "{\"query_id\":\"xjy-42-1rj\",\"user_query\":\"aa\",\"query_attributes\":{\"page\":2,\"filter\":\"inStock:true\"},\"doc_ids\":\"1,two\"}";
-    assertJSONEquals(jsonlLogLine, lastLine);
   }
 
   @Test
@@ -152,11 +125,4 @@ public void testDisabling() {
         req("qt", "/withubi", "q", "aa", "ubi", "false"),
         "count(//lst[@name='ubi'])=0");
   }
-
-  private static String readLastLineOfFile(File file) throws IOException {
-    try (ReversedLinesFileReader reader =
-        ReversedLinesFileReader.builder().setFile(file).setCharset(StandardCharsets.UTF_8).get()) {
-      return reader.readLine();
-    }
-  }
 }

From af8d3f08c2998b1da79c691a7015a84a4719591e Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 19 Aug 2024 10:09:59 -0500
Subject: [PATCH 033/103] Properly deal with solrClientCache and object
 tracking.  Cleaning up warnings.

---
 .../solr/handler/component/UBIComponent.java  | 25 +++++++++++--------
 .../conf/solrconfig-ubi-local-component.xml   |  4 +--
 .../ubi-enabled/conf/ubi-query-pipeline.expr  | 22 ++++++++--------
 .../UBIComponentStreamingQueriesTest.java     | 25 +++++--------------
 .../handler/component/UBIComponentTest.java   | 24 +++++++++---------
 5 files changed, 46 insertions(+), 54 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index b42d3efcd97..3d7842775ae 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -29,6 +29,7 @@
 import org.apache.solr.client.solrj.io.Lang;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.StreamContext;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
@@ -128,12 +129,9 @@ public class UBIComponent extends SearchComponent implements SolrCoreAware {
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-  private StreamFactory streamFactory;
-  private StreamExpression streamExpression;
   private TupleStream stream;
 
   protected SolrParams initArgs;
-  private SolrClientCache solrClientCache;
 
   @Override
   public void init(NamedList args) {
@@ -143,13 +141,13 @@ public void init(NamedList args) {
   @Override
   public void inform(SolrCore core) {
     CoreContainer coreContainer = core.getCoreContainer();
-    this.solrClientCache = coreContainer.getSolrClientCache();
+    SolrClientCache solrClientCache = coreContainer.getSolrClientCache();
 
     // do I need this check?
     if (initArgs != null) {
       log.info("Initializing UBIComponent");
       if (coreContainer.isZooKeeperAware()) {
-        String defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress();
+        String defaultZkHost = core.getCoreContainer().getZkController().getZkServerAddress();
         String streamQueriesExpressionFile = initArgs.get("streamQueriesExpressionFile");
 
         if (streamQueriesExpressionFile == null) {
@@ -171,14 +169,19 @@ public void inform(SolrCore core) {
 
             bufferedReader.close();
 
-            streamExpression = StreamExpressionParser.parse(expr);
-            streamFactory = new StreamFactory();
+            StreamContext streamContext = new StreamContext();
 
-            streamFactory.withDefaultZkHost(defaultZkhost);
+            streamContext.setSolrClientCache(solrClientCache);
+
+            StreamExpression streamExpression = StreamExpressionParser.parse(expr);
+            StreamFactory streamFactory = new StreamFactory();
+
+            streamFactory.withDefaultZkHost(defaultZkHost);
 
             Lang.register(streamFactory);
 
             stream = constructStream(streamFactory, streamExpression);
+            stream.setStreamContext(streamContext);
 
             // not sure if I need this?  Except maybe, we assume let?
             // Map params = validateLetAndGetParams(stream, expr);
@@ -251,10 +254,11 @@ protected void processIds(
 
     Set fields = Collections.singleton(schema.getUniqueKeyField().getName());
     for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {
-
-      sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(',');
+      sb.append(schema.printableUniqueKey(searcher.getDocFetcher().doc(iter.nextDoc(), fields)))
+          .append(',');
     }
     String docIds = sb.length() > 0 ? sb.substring(0, sb.length() - 1) : "";
+
     SimpleOrderedMap ubiResponseInfo = new SimpleOrderedMap<>();
     SimpleOrderedMap ubiQueryLogInfo = new SimpleOrderedMap<>();
     ubiResponseInfo.add(QUERY_ID, ubiQuery.getQueryId());
@@ -268,6 +272,7 @@ protected void processIds(
 
     // pushBackStream = new PushBackStream(stream);
     if (stream != null) {
+      // getTuples invokes the streaming expression.
       List tuples = getTuples(stream);
     }
   }
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml
index ddedb35856f..59a94af99dd 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml
@@ -39,7 +39,7 @@
   
 
   
-  
+  
     
       dismax
     
@@ -54,7 +54,7 @@
     
   
 
-  
+  
     
       text
     
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
index 58337e63315..801650bbced 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
@@ -13,15 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#commit(ubi,
-#  update(ubi,
-#    tuple(id=49,a_i=1,b_i=5)
-#  )
-#)
-
-let(cli-zkhost="localhost:9983",    
-   commit(ubi,
-    update(ubi,
-           tuple(id=49,a_i=1,b_i=5))
-           )
+commit(ubi,
+  update(ubi,
+    tuple(id=49,a_i=1,b_i=5)
+  )
 )
+
+#let(cli-zkhost="localhost:9983",    
+#   commit(ubi,
+#    update(ubi,
+#           tuple(id=49,a_i=1,b_i=5))
+#           )
+#)
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index c2e30457a8a..eb5afed8350 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -19,8 +19,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.net.URLDecoder;
-import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -131,7 +129,7 @@ public void clearCollection() throws Exception {
     assertEquals("commit failed", 0, cluster.getSolrClient().commit(UBI_COLLECTION).getStatus());
   }
 
-  public void testWritingStreamingExpression() {
+  public void testCreatingStreamingExpression() {
     UBIQuery ubiQuery = new UBIQuery("5678");
     ubiQuery.setUserQuery("Apple Memory");
 
@@ -139,10 +137,6 @@ public void testWritingStreamingExpression() {
     assertEquals(
         "Check the decoded version for ease of comparison",
         "commit(ubi,update(ubi,tuple(id=4.0,query_id=5678,user_query=Apple Memory)))",
-        URLDecoder.decode(clause, StandardCharsets.UTF_8));
-    assertEquals(
-        "Verify the encoded version",
-        "commit%28ubi%2Cupdate%28ubi%2Ctuple%28id%3D4.0%2Cquery_id%3D5678%2Cuser_query%3DApple+Memory%29%29%29",
         clause);
   }
 
@@ -175,7 +169,8 @@ public void testUsingStreamingExpressionDirectly() throws Exception {
     assertEquals("1", tuple.getString("totalIndexed"));
 
     // Check the UBI collection
-    final JsonQueryRequest requestFromUBICollection = new JsonQueryRequest().setQuery("id:4.0").setLimit(1);
+    final JsonQueryRequest requestFromUBICollection =
+        new JsonQueryRequest().setQuery("id:4.0").setLimit(1);
 
     // Randomly grab a client, it shouldn't matter which is used to check UBI event.
     SolrClient client = getRandClient();
@@ -205,16 +200,7 @@ private List getTuples(TupleStream tupleStream) throws IOException {
   }
 
   private static String getClause(UBIQuery ubiQuery) {
-    String clause = "commit(ubi,update(ubi,tuple(id=4.0," + ubiQuery.toTuple() + ")))";
-    //String clause = "commit(ubi,update(ubi,tuple(id=4.0)))";
-    //clause = URLEncoder.encode(clause, StandardCharsets.UTF_8);
-    return clause;
-  }
-
-  private static String getClause() {
-
-    String clause = "commit(ubi,update(ubi,tuple(id=add(1,3), name_s=bob)))";
-    return clause;
+    return "commit(ubi,update(ubi,tuple(id=4.0," + ubiQuery.toTuple() + ")))";
   }
 
   @SuppressWarnings({"rawtypes", "unchecked"})
@@ -254,7 +240,8 @@ public void testRandomDocs() throws Exception {
     // Check the UBI collection
     final JsonQueryRequest requestUBI = new JsonQueryRequest().setQuery("id:49").setLimit(1);
 
-    // Randomly grab a client, it shouldn't matter which is used to check UBI event.
+    // Randomly grab a client, it shouldn't matter which is used, to check UBI event was actually
+    // tracked.
     client = getRandClient();
     final QueryResponse responseUBI = requestUBI.process(client, UBI_COLLECTION);
     try {
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
index bdfaa513835..49d645a2cf6 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
@@ -32,27 +32,27 @@ public static void beforeTest() throws Exception {
   }
 
   @Test
-  public void testToLogIds() {
+  public void testGeneratingAQueryId() {
     assertQ(
         "Make sure we generate a query id",
-        req("qt", "/withubi", "q", "aa", "rows", "2", "ubi", "true"),
+        req("qt", "/with_ubi", "q", "aa", "rows", "2", "ubi", "true"),
         "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
   }
 
   @Test
-  public void testZeroResults() {
+  public void testZeroResultsGeneratesQueryId() {
     assertQ(
         "Make sure we generate a query id even when no results are returned",
-        req("qt", "/withubi", "q", "abcdefgxyz", "rows", "0", "ubi", "true"),
+        req("qt", "/with_ubi", "q", "abcdefgxyz", "rows", "0", "ubi", "true"),
         "//*[@numFound='0']",
         "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
   }
 
   @Test
-  public void testPassedInQueryId() {
+  public void testPassedInQueryIdIsUsed() {
     assertQ(
         "Make sure we reuse a passed in query id",
-        req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true", "query_id", "123abc"),
+        req("qt", "/with_ubi", "q", "aa", "rows", "0", "ubi", "true", "query_id", "123abc"),
         "//lst[@name='ubi']/str[@name='query_id'][.='123abc']");
   }
 
@@ -60,7 +60,7 @@ public void testPassedInQueryId() {
   public void testGenerateQueryId() {
     assertQ(
         "Make sure we generate a query id if one is not passed in",
-        req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true"),
+        req("qt", "/with_ubi", "q", "aa", "rows", "0", "ubi", "true"),
         "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
   }
 
@@ -69,7 +69,7 @@ public void testJSONQuerySyntax() throws Exception {
     assertJQ(
         req(
             "qt",
-            "/withubi",
+            "/with_ubi",
             "json",
             "{\n"
                 + "    'query': 'aa',\n"
@@ -78,7 +78,7 @@ public void testJSONQuerySyntax() throws Exception {
                 + "    'limit': 2,\n"
                 + "    'params': {\n"
                 + "    'df': 'subject',\n"
-                + "    'qt': '/withubi',\n"
+                + "    'qt': '/with_ubi',\n"
                 + "    'ubi': 'true'\n"
                 + "   }\n"
                 + "}"),
@@ -88,7 +88,7 @@ public void testJSONQuerySyntax() throws Exception {
     assertJQ(
         req(
             "qt",
-            "/withubi",
+            "/with_ubi",
             "json",
             "{\n"
                 + "    'query': 'aa',\n"
@@ -114,7 +114,7 @@ public void testJSONQuerySyntax() throws Exception {
   public void testTrackingOfUserQuery() {
     assertQ(
         "Make sure we generate a query id",
-        req("qt", "/withubi", "q", "aa", "rows", "0", "ubi", "true", "user_query", "fresh air"),
+        req("qt", "/with_ubi", "q", "aa", "rows", "0", "ubi", "true", "user_query", "fresh air"),
         "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
   }
 
@@ -122,7 +122,7 @@ public void testTrackingOfUserQuery() {
   public void testDisabling() {
     assertQ(
         "Make sure we don't generate a query_id",
-        req("qt", "/withubi", "q", "aa", "ubi", "false"),
+        req("qt", "/with_ubi", "q", "aa", "ubi", "false"),
         "count(//lst[@name='ubi'])=0");
   }
 }

From 9b47b0ba98b0a0c8e5859df2ecfee41a0ba838a4 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Tue, 8 Oct 2024 15:07:57 -0600
Subject: [PATCH 034/103] lets use streaming for local

---
 .../solr/handler/component/UBIComponent.java  | 47 +++++++++++--------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 3d7842775ae..bfc20d8104a 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -148,11 +148,14 @@ public void inform(SolrCore core) {
       log.info("Initializing UBIComponent");
       if (coreContainer.isZooKeeperAware()) {
         String defaultZkHost = core.getCoreContainer().getZkController().getZkServerAddress();
-        String streamQueriesExpressionFile = initArgs.get("streamQueriesExpressionFile");
+        String ubiQueryProcessingExpression = initArgs.get("ubiQueryProcessingExpression");
 
-        if (streamQueriesExpressionFile == null) {
+        String expr = null;
+        if (ubiQueryProcessingExpression == null) {
           log.info(
-              "You must provide a streamQueriesExpressionFile to enable recording UBI information.");
+              "You should provide a ubiQueryProcessingExpression to control how UBI query information is persisted.");
+          log.info("Writing out UBI query information to local log file ubi_queries.log instead.");
+
         } else {
 
           LineNumberReader bufferedReader;
@@ -161,38 +164,42 @@ public void inform(SolrCore core) {
             bufferedReader =
                 new LineNumberReader(
                     new InputStreamReader(
-                        core.getResourceLoader().openResource(streamQueriesExpressionFile),
+                        core.getResourceLoader().openResource(ubiQueryProcessingExpression),
                         StandardCharsets.UTF_8));
 
             String[] args = {}; // maybe we have variables?
-            String expr = readExpression(bufferedReader, args);
+            expr = readExpression(bufferedReader, args);
 
             bufferedReader.close();
 
-            StreamContext streamContext = new StreamContext();
-
-            streamContext.setSolrClientCache(solrClientCache);
-
-            StreamExpression streamExpression = StreamExpressionParser.parse(expr);
-            StreamFactory streamFactory = new StreamFactory();
-
-            streamFactory.withDefaultZkHost(defaultZkHost);
-
-            Lang.register(streamFactory);
-
-            stream = constructStream(streamFactory, streamExpression);
-            stream.setStreamContext(streamContext);
-
             // not sure if I need this?  Except maybe, we assume let?
             // Map params = validateLetAndGetParams(stream, expr);
 
           } catch (IOException ioe) {
             throw new SolrException(
                 SolrException.ErrorCode.SERVER_ERROR,
-                "Error reading file " + streamQueriesExpressionFile,
+                "Error reading file " + ubiQueryProcessingExpression,
                 ioe);
           }
         }
+        StreamContext streamContext = new StreamContext();
+
+        streamContext.setSolrClientCache(solrClientCache);
+
+        StreamExpression streamExpression = StreamExpressionParser.parse(expr);
+        StreamFactory streamFactory = new StreamFactory();
+
+        streamFactory.withDefaultZkHost(defaultZkHost);
+
+        Lang.register(streamFactory);
+
+        try {
+          stream = constructStream(streamFactory, streamExpression);
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+        stream.setStreamContext(streamContext);
+
       } else {
         log.info("UBI query data collection is only available in SolrCloud mode.");
       }

From 7277248394ce92fe5dc513761614e75c12ddec2f Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Thu, 10 Oct 2024 09:52:01 -0600
Subject: [PATCH 035/103] Track progress so I dont lose it

---
 .../solr/handler/component/LogStream.java     | 442 ++++++++++++++++++
 .../UBIComponentLocalLoggingTest.java         | 105 +++++
 2 files changed, 547 insertions(+)
 create mode 100644 solr/core/src/java/org/apache/solr/handler/component/LogStream.java
 create mode 100644 solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LogStream.java b/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
new file mode 100644
index 00000000000..a9a5f08a29c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.PushBackStream;
+import org.apache.solr.client.solrj.io.stream.StreamContext;
+import org.apache.solr.client.solrj.io.stream.TupleStream;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.core.SolrCore;
+import org.noggit.CharArr;
+import org.noggit.JSONWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Sends tuples emitted by a wrapped {@link TupleStream} as writes to a log file.
+ * I really want to call this the DogStream, as it matches the CatStream.
+ *
+ * @since 9.8.0
+ */
+public class LogStream extends TupleStream implements Expressible {
+    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    // field name in summary tuple for #docs updated in batch
+    public static String BATCH_LOGGED_FIELD_NAME = "batchLogged";
+
+    private StreamContext context;
+    private Path chroot;
+
+    /**
+     * The name of the log file that should be written to.  This will be in the same directory that the CatStream is allowed to write to.
+     */
+    private String filepath;
+    private int updateBatchSize;
+
+
+
+    private int batchNumber;
+    private long totalDocsIndex;
+    private PushBackStream tupleSource;
+    private List documentBatch = new ArrayList<>();
+
+    private OutputStream fos;
+    private final CharArr charArr = new CharArr(1024 * 2);
+    JSONWriter jsonWriter = new JSONWriter(charArr, -1);
+    private Writer writer;
+
+
+    public LogStream(StreamExpression expression, StreamFactory factory) throws IOException {
+
+
+        filepath = factory.getValueOperand(expression, 0);
+        if (filepath == null) {
+            throw new IllegalArgumentException("No filepath provided to log stream to");
+        }
+        final String filepathWithoutSurroundingQuotes =
+                stripSurroundingQuotesIfTheyExist(filepath);
+        if (StrUtils.isNullOrEmpty(filepathWithoutSurroundingQuotes)) {
+            throw new IllegalArgumentException("No filepath provided to stream");
+        }
+
+        this.filepath = filepathWithoutSurroundingQuotes;
+
+        // Extract underlying TupleStream.
+        List streamExpressions =
+                factory.getExpressionOperandsRepresentingTypes(
+                        expression, Expressible.class, TupleStream.class);
+        if (1 != streamExpressions.size()) {
+            throw new IOException(
+                    String.format(
+                            Locale.ROOT,
+                            "Invalid expression %s - expecting a single stream but found %d",
+                            expression,
+                            streamExpressions.size()));
+        }
+        StreamExpression sourceStreamExpression = streamExpressions.get(0);
+        init(filepathWithoutSurroundingQuotes, factory.constructStream(sourceStreamExpression));
+    }
+
+    public LogStream(String commaDelimitedFilepaths) {
+
+    }
+
+    public LogStream(
+            String collectionName, TupleStream tupleSource)
+            throws IOException {
+
+        init(collectionName, tupleSource);
+    }
+
+
+    private void init(
+            String filepaths, TupleStream tupleSource) {
+        this.filepath = filepaths;
+        this.tupleSource = new PushBackStream(tupleSource);
+    }
+
+    /** The name of the file being updated */
+    protected String getFilePath() {
+        return filepath;
+    }
+
+    @Override
+    public void open() throws IOException {
+        Path filePath = chroot.resolve(filepath).normalize();
+        if (!filePath.startsWith(chroot)) {
+            throw new SolrException(
+                    SolrException.ErrorCode.BAD_REQUEST,
+                    "file to log to must be under " + chroot);
+        }
+
+//        if (!Files.exists(filePath)) {
+//
+//            throw new SolrException(
+//                    SolrException.ErrorCode.BAD_REQUEST,
+//                    "file/directory to stream doesn't exist: " + crawlRootStr);
+//        }
+        fos = new FileOutputStream(filePath.toFile());
+        writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
+
+
+
+        tupleSource.open();
+    }
+
+    @Override
+    public Tuple read() throws IOException {
+
+        Tuple tuple = tupleSource.read();
+        if (tuple.EOF) {
+
+            return tuple;
+        } else {
+            tupleSource.pushBack(tuple);
+            uploadBatchToCollection(tuple);
+            // return createBatchSummaryTuple(b);
+        }
+
+
+
+        //uploadBatchToCollection(documentBatch);
+        //int b = documentBatch.size();
+        //documentBatch.clear();
+        int b = 0;
+        return createBatchSummaryTuple(b);
+    }
+
+    @Override
+    public void close() throws IOException {
+        writer.flush();
+        fos.flush();
+        fos.close();
+        tupleSource.close();
+    }
+
+    @Override
+    public StreamComparator getStreamSort() {
+        return tupleSource.getStreamSort();
+    }
+
+    @Override
+    public List children() {
+        ArrayList sourceList = new ArrayList<>(1);
+        sourceList.add(tupleSource);
+        return sourceList;
+    }
+
+    @Override
+    public StreamExpression toExpression(StreamFactory factory) throws IOException {
+        return toExpression(factory, true);
+    }
+
+    private StreamExpression toExpression(StreamFactory factory, boolean includeStreams)
+            throws IOException {
+        StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
+        expression.addParameter(filepath);
+        //expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost));
+        //expression.addParameter(
+        //        new StreamExpressionNamedParameter("batchSize", Integer.toString(updateBatchSize)));
+
+        if (includeStreams) {
+            if (tupleSource != null) {
+                expression.addParameter(((Expressible) tupleSource).toExpression(factory));
+            } else {
+                throw new IOException(
+                        "This LogStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+            }
+        } else {
+            expression.addParameter("");
+        }
+
+        return expression;
+    }
+
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+        // An update stream is backward wrt the order in the explanation. This stream is the "child"
+        // while the collection we're updating is the parent.
+
+        StreamExplanation explanation = new StreamExplanation(getStreamNodeId() + "-datastore");
+
+        explanation.setFunctionName(String.format(Locale.ROOT, "log (%s)", filepath));
+        explanation.setImplementingClass("Solr/Lucene");
+        explanation.setExpressionType(ExpressionType.DATASTORE);
+        explanation.setExpression("Log into " + filepath);
+
+        // child is a datastore so add it at this point
+        StreamExplanation child = new StreamExplanation(getStreamNodeId().toString());
+        child.setFunctionName(String.format(Locale.ROOT, factory.getFunctionName(getClass())));
+        child.setImplementingClass(getClass().getName());
+        child.setExpressionType(ExpressionType.STREAM_DECORATOR);
+        child.setExpression(toExpression(factory, false).toString());
+        child.addChild(tupleSource.toExplanation(factory));
+
+        explanation.addChild(child);
+
+        return explanation;
+    }
+
+    @Override
+    public void setStreamContext(StreamContext context) {
+        this.context = context;
+        Object solrCoreObj = context.get("solr-core");
+        if (solrCoreObj == null || !(solrCoreObj instanceof SolrCore)) {
+            throw new SolrException(
+                    SolrException.ErrorCode.INVALID_STATE,
+                    "StreamContext must have SolrCore in solr-core key");
+        }
+        final SolrCore core = (SolrCore) context.get("solr-core");
+
+        this.chroot = core.getCoreContainer().getUserFilesPath();
+        if (!Files.exists(chroot)) {
+            throw new IllegalStateException(
+                    chroot + " directory used to load files must exist but could not be found!");
+        }
+    }
+
+    private void verifyCollectionName(String collectionName, StreamExpression expression)
+            throws IOException {
+        if (null == collectionName) {
+            throw new IOException(
+                    String.format(
+                            Locale.ROOT,
+                            "invalid expression %s - collectionName expected as first operand",
+                            expression));
+        }
+    }
+
+    private String findZkHost(
+            StreamFactory factory, String collectionName, StreamExpression expression) {
+        StreamExpressionNamedParameter zkHostExpression = factory.getNamedOperand(expression, "zkHost");
+        if (null == zkHostExpression) {
+            String zkHost = factory.getCollectionZkHost(collectionName);
+            if (zkHost == null) {
+                return factory.getDefaultZkHost();
+            } else {
+                return zkHost;
+            }
+        } else if (zkHostExpression.getParameter() instanceof StreamExpressionValue) {
+            return ((StreamExpressionValue) zkHostExpression.getParameter()).getValue();
+        }
+
+        return null;
+    }
+
+    private void verifyZkHost(String zkHost, String collectionName, StreamExpression expression)
+            throws IOException {
+        if (null == zkHost) {
+            throw new IOException(
+                    String.format(
+                            Locale.ROOT,
+                            "invalid expression %s - zkHost not found for collection '%s'",
+                            expression,
+                            collectionName));
+        }
+    }
+
+    private int extractBatchSize(StreamExpression expression, StreamFactory factory)
+            throws IOException {
+        StreamExpressionNamedParameter batchSizeParam =
+                factory.getNamedOperand(expression, "batchSize");
+        if (batchSizeParam == null) {
+            // Sensible default batch size
+            return 250;
+        }
+        String batchSizeStr = ((StreamExpressionValue) batchSizeParam.getParameter()).getValue();
+        return parseBatchSize(batchSizeStr, expression);
+    }
+
+    private int parseBatchSize(String batchSizeStr, StreamExpression expression) throws IOException {
+        try {
+            int batchSize = Integer.parseInt(batchSizeStr);
+            if (batchSize <= 0) {
+                throw new IOException(
+                        String.format(
+                                Locale.ROOT,
+                                "invalid expression %s - batchSize '%d' must be greater than 0.",
+                                expression,
+                                batchSize));
+            }
+            return batchSize;
+        } catch (NumberFormatException e) {
+            throw new IOException(
+                    String.format(
+                            Locale.ROOT,
+                            "invalid expression %s - batchSize '%s' is not a valid integer.",
+                            expression,
+                            batchSizeStr));
+        }
+    }
+
+    /**
+     * Used during initialization to specify the default value for the "pruneVersionField"
+     *  option. {@link org.apache.solr.client.solrj.io.stream.UpdateStream} returns true for backcompat and to simplify
+     * slurping of data from one collection to another.
+     */
+    protected boolean defaultPruneVersionField() {
+        return true;
+    }
+
+//    private SolrInputDocument convertTupleTJson(Tuple tuple) {
+//        SolrInputDocument doc = new SolrInputDocument();
+//        for (String field : tuple.getFields().keySet()) {
+//
+//            if (!(field.equals(CommonParams.VERSION_FIELD) )) {
+//                Object value = tuple.get(field);
+//                if (value instanceof List) {
+//                    addMultivaluedField(doc, field, (List) value);
+//                } else {
+//                    doc.addField(field, value);
+//                }
+//            }
+//        }
+//        log.debug("Tuple [{}] was converted into SolrInputDocument [{}].", tuple, doc);
+//        jsonWriter
+//        return doc;
+//    }
+
+    private void addMultivaluedField(SolrInputDocument doc, String fieldName, List values) {
+        for (Object value : values) {
+            doc.addField(fieldName, value);
+        }
+    }
+
+    /**
+     * This method will be called on every batch of tuples comsumed, after converting each tuple in
+     * that batch to a Solr Input Document.
+     */
+    protected void uploadBatchToCollection(Tuple doc) throws IOException {
+        charArr.reset();
+//        doc.toMap()
+//        Map m =doc.toMap()
+//        doc.forEach(
+//                (s, field) -> {
+//                    if (s.equals("_version_") || s.equals("_roor_")) return;
+//                    if (field instanceof List) {
+//                        if (((List) field).size() == 1) {
+//                            field = ((List) field).get(0);
+//                        }
+//                    }
+//                    field = constructDateStr(field);
+//                    if (field instanceof List) {
+//                        List list = (List) field;
+//                        if (hasdate(list)) {
+//                            ArrayList listCopy = new ArrayList<>(list.size());
+//                            for (Object o : list) listCopy.add(constructDateStr(o));
+//                            field = listCopy;
+//                        }
+//                    }
+//                    m.put(s, field);
+//                });
+        //jsonWriter.write(m);
+        jsonWriter.write(doc);
+        writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());
+        writer.append('\n');
+    }
+
+    private Tuple createBatchSummaryTuple(int batchSize) {
+        assert batchSize > 0;
+        Tuple tuple = new Tuple();
+        this.totalDocsIndex += batchSize;
+        ++batchNumber;
+        tuple.put(BATCH_LOGGED_FIELD_NAME, batchSize);
+        tuple.put("totalIndexed", this.totalDocsIndex);
+        tuple.put("batchNumber", batchNumber);
+       // if (coreName != null) {
+       //     tuple.put("worker", coreName);
+        //}
+        return tuple;
+    }
+
+    private String stripSurroundingQuotesIfTheyExist(String value) {
+        if (value.length() < 2) return value;
+        if ((value.startsWith("\"") && value.endsWith("\""))
+                || (value.startsWith("'") && value.endsWith("'"))) {
+            return value.substring(1, value.length() - 1);
+        }
+
+        return value;
+    }
+}
+
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
new file mode 100644
index 00000000000..6f94c11a837
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -0,0 +1,105 @@
+package org.apache.solr.handler.component;
+
+import org.apache.solr.client.solrj.io.Lang;
+import org.apache.solr.client.solrj.io.SolrClientCache;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.StreamContext;
+import org.apache.solr.client.solrj.io.stream.TupleStream;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class UBIComponentLocalLoggingTest extends SolrCloudTestCase {
+
+
+    @Test
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    public void testLocalCatStream() throws Exception {
+
+        File localFile = File.createTempFile("topLevel1", ".txt");
+
+        TupleStream stream;
+        List tuples;
+        StreamContext streamContext = new StreamContext();
+        SolrClientCache solrClientCache = new SolrClientCache();
+
+        streamContext.setSolrClientCache(solrClientCache);
+
+        StreamFactory streamFactory = new StreamFactory();
+
+
+        // LocalCatStream extends CatStream and disables the Solr cluster specific
+        // logic about where to read data from.
+        streamFactory.withFunctionName("logging", LogStream.class);
+
+
+        Lang.register(streamFactory);
+
+        String clause = "logging(bob.txt,echo(\"bob\"))";
+        stream = streamFactory.constructStream(clause);
+        stream.setStreamContext(streamContext);
+        tuples = getTuples(stream);
+        stream.close();
+        solrClientCache.close();
+
+
+        //populateFileWithData(localFile.toPath());
+
+
+        Tuple tuple = new Tuple(new HashMap());
+        tuple.put("field1", "blah");
+        tuple.put("field2", "blah");
+        tuple.put("field3", "blah");
+
+       LogStream logStream =
+                new LogStream(localFile.getAbsolutePath());
+        List tuples2 = new ArrayList();
+        try {
+            logStream.open();
+
+
+
+//            while (true) {
+//                Tuple tuple = logStream.read();
+//                if (tuple.EOF) {
+//                    break;
+//                } else {
+//                    tuples.add(tuple);
+//                }
+//            }
+
+        } finally {
+            logStream.close();
+        }
+
+        assertEquals(4, tuples.size());
+
+        for (int i = 0; i < 4; i++) {
+            Tuple t = tuples.get(i);
+            assertEquals(localFile.getName() + " line " + (i + 1), t.get("line"));
+            assertEquals(localFile.getAbsolutePath(), t.get("file"));
+        }
+    }
+
+    private List getTuples(TupleStream tupleStream) throws IOException {
+        tupleStream.open();
+        List tuples = new ArrayList<>();
+        for (; ; ) {
+            Tuple t = tupleStream.read();
+            // log.info(" ... {}", t.fields);
+            if (t.EOF) {
+                break;
+            } else {
+                tuples.add(t);
+            }
+        }
+        tupleStream.close();
+        return tuples;
+    }
+}

From 64948b93c62d77390120081021c29e0460338969 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 08:56:38 -0600
Subject: [PATCH 036/103] Track changes

---
 .../solr/handler/component/LogStream.java     | 83 ++-----------------
 .../UBIComponentLocalLoggingTest.java         | 77 +++++++++++++++--
 2 files changed, 77 insertions(+), 83 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LogStream.java b/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
index a9a5f08a29c..2618aad5115 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
@@ -76,7 +76,8 @@ public class LogStream extends TupleStream implements Expressible {
 
     private int batchNumber;
     private long totalDocsIndex;
-    private PushBackStream tupleSource;
+    //private PushBackStream tupleSource;
+    private TupleStream tupleSource;
     private List documentBatch = new ArrayList<>();
 
     private OutputStream fos;
@@ -116,9 +117,6 @@ public LogStream(StreamExpression expression, StreamFactory factory) throws IOEx
         init(filepathWithoutSurroundingQuotes, factory.constructStream(sourceStreamExpression));
     }
 
-    public LogStream(String commaDelimitedFilepaths) {
-
-    }
 
     public LogStream(
             String collectionName, TupleStream tupleSource)
@@ -131,7 +129,7 @@ public LogStream(
     private void init(
             String filepaths, TupleStream tupleSource) {
         this.filepath = filepaths;
-        this.tupleSource = new PushBackStream(tupleSource);
+        this.tupleSource = tupleSource;//new PushBackStream(tupleSource);
     }
 
     /** The name of the file being updated */
@@ -170,7 +168,7 @@ public Tuple read() throws IOException {
 
             return tuple;
         } else {
-            tupleSource.pushBack(tuple);
+            //tupleSource.pushBack(tuple);
             uploadBatchToCollection(tuple);
             // return createBatchSummaryTuple(b);
         }
@@ -180,7 +178,7 @@ public Tuple read() throws IOException {
         //uploadBatchToCollection(documentBatch);
         //int b = documentBatch.size();
         //documentBatch.clear();
-        int b = 0;
+        int b = 1;
         return createBatchSummaryTuple(b);
     }
 
@@ -286,77 +284,6 @@ private void verifyCollectionName(String collectionName, StreamExpression expres
         }
     }
 
-    private String findZkHost(
-            StreamFactory factory, String collectionName, StreamExpression expression) {
-        StreamExpressionNamedParameter zkHostExpression = factory.getNamedOperand(expression, "zkHost");
-        if (null == zkHostExpression) {
-            String zkHost = factory.getCollectionZkHost(collectionName);
-            if (zkHost == null) {
-                return factory.getDefaultZkHost();
-            } else {
-                return zkHost;
-            }
-        } else if (zkHostExpression.getParameter() instanceof StreamExpressionValue) {
-            return ((StreamExpressionValue) zkHostExpression.getParameter()).getValue();
-        }
-
-        return null;
-    }
-
-    private void verifyZkHost(String zkHost, String collectionName, StreamExpression expression)
-            throws IOException {
-        if (null == zkHost) {
-            throw new IOException(
-                    String.format(
-                            Locale.ROOT,
-                            "invalid expression %s - zkHost not found for collection '%s'",
-                            expression,
-                            collectionName));
-        }
-    }
-
-    private int extractBatchSize(StreamExpression expression, StreamFactory factory)
-            throws IOException {
-        StreamExpressionNamedParameter batchSizeParam =
-                factory.getNamedOperand(expression, "batchSize");
-        if (batchSizeParam == null) {
-            // Sensible default batch size
-            return 250;
-        }
-        String batchSizeStr = ((StreamExpressionValue) batchSizeParam.getParameter()).getValue();
-        return parseBatchSize(batchSizeStr, expression);
-    }
-
-    private int parseBatchSize(String batchSizeStr, StreamExpression expression) throws IOException {
-        try {
-            int batchSize = Integer.parseInt(batchSizeStr);
-            if (batchSize <= 0) {
-                throw new IOException(
-                        String.format(
-                                Locale.ROOT,
-                                "invalid expression %s - batchSize '%d' must be greater than 0.",
-                                expression,
-                                batchSize));
-            }
-            return batchSize;
-        } catch (NumberFormatException e) {
-            throw new IOException(
-                    String.format(
-                            Locale.ROOT,
-                            "invalid expression %s - batchSize '%s' is not a valid integer.",
-                            expression,
-                            batchSizeStr));
-        }
-    }
-
-    /**
-     * Used during initialization to specify the default value for the "pruneVersionField"
-     *  option. {@link org.apache.solr.client.solrj.io.stream.UpdateStream} returns true for backcompat and to simplify
-     * slurping of data from one collection to another.
-     */
-    protected boolean defaultPruneVersionField() {
-        return true;
-    }
 
 //    private SolrInputDocument convertTupleTJson(Tuple tuple) {
 //        SolrInputDocument doc = new SolrInputDocument();
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index 6f94c11a837..4b1619e002d 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -6,27 +6,81 @@
 import org.apache.solr.client.solrj.io.stream.StreamContext;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 
 public class UBIComponentLocalLoggingTest extends SolrCloudTestCase {
 
+    private static final String COLLECTION = "collection1";
+
+    @BeforeClass
+    public static void setupCluster() throws Exception {
+        configureCluster(1)
+                .addConfig(
+                        "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+                .configure();
+    }
 
     @Test
     @SuppressWarnings({"unchecked", "rawtypes"})
     public void testLocalCatStream() throws Exception {
 
+        CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1, 1, 0)
+                .process(cluster.getSolrClient());
+        cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
+
         File localFile = File.createTempFile("topLevel1", ".txt");
 
         TupleStream stream;
         List tuples;
         StreamContext streamContext = new StreamContext();
+        //Replica rr = zkStateReader.getCollection(coll).getReplicas().get(0);
+
+        //cluster.getJettySolrRunner(0).getCoreContainer().getCore()
+//        Replica replica =
+  //              getRandomReplica(
+    //                    shard, (r) -> (r.getState() == Replica.State.ACTIVE && !r.equals(shard.getLeader())));
+
+        SolrCore solrCoreToLoad = null;
+        for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
+            for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
+                if (solrCore != null){
+                    solrCoreToLoad = solrCore;
+                }
+                System.out.println(solrCore);
+            }
+        }
+
+        final Path dataDir = findUserFilesDataDir();
+        Files.createDirectories(dataDir);
+        //populateFileStreamData(dataDir);
+
+        //JettySolrRunner replicaJetty = cluster.getReplicaJetty(replica);
+        //cluster.getJettySolrRunner(0).getCoreContainer().getr
+
+
+        //SolrQueryRequest req = req("q", "*:*");
+CoreContainer cc = cluster.getJettySolrRunner(0).getCoreContainer();
+
+        var l = cc.getAllCoreNames();
+SolrCore core = cc.getCore(l.get(0));
+        streamContext.put("solr-core", core);
         SolrClientCache solrClientCache = new SolrClientCache();
 
         streamContext.setSolrClientCache(solrClientCache);
@@ -57,11 +111,13 @@ public void testLocalCatStream() throws Exception {
         tuple.put("field2", "blah");
         tuple.put("field3", "blah");
 
-       LogStream logStream =
-                new LogStream(localFile.getAbsolutePath());
+       //LogStream logStream =
+         ///    //   new LogStream(localFile.getAbsolutePath());
+       // LogStream logStream =
+         //              new LogStream();
         List tuples2 = new ArrayList();
         try {
-            logStream.open();
+           // logStream.open();
 
 
 
@@ -75,7 +131,7 @@ public void testLocalCatStream() throws Exception {
 //            }
 
         } finally {
-            logStream.close();
+         //   logStream.close();
         }
 
         assertEquals(4, tuples.size());
@@ -87,6 +143,18 @@ public void testLocalCatStream() throws Exception {
         }
     }
 
+    private static Path findUserFilesDataDir() {
+        for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+            for (CoreDescriptor coreDescriptor : jetty.getCoreContainer().getCoreDescriptors()) {
+                if (coreDescriptor.getCollectionName().equals(COLLECTION)) {
+                    return jetty.getCoreContainer().getUserFilesPath();
+                }
+            }
+        }
+
+        throw new IllegalStateException("Unable to determine data-dir for: " + COLLECTION);
+    }
+
     private List getTuples(TupleStream tupleStream) throws IOException {
         tupleStream.open();
         List tuples = new ArrayList<>();
@@ -97,7 +165,6 @@ private List getTuples(TupleStream tupleStream) throws IOException {
                 break;
             } else {
                 tuples.add(t);
-            }
         }
         tupleStream.close();
         return tuples;

From a646b68fb70d304aeba0696b8e8c7c84366a4524 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 10:20:06 -0600
Subject: [PATCH 037/103] Introduce better test of the LogStream

---
 .../solr/handler/component/LogStream.java     | 551 +++++++++---------
 .../solr/handler/component/LogStreamTest.java | 126 ++++
 .../UBIComponentLocalLoggingTest.java         | 252 ++++----
 3 files changed, 518 insertions(+), 411 deletions(-)
 create mode 100644 solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LogStream.java b/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
index 2618aad5115..a43d2cdeccd 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
@@ -16,7 +16,6 @@
  */
 package org.apache.solr.handler.component;
 
-
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -31,7 +30,6 @@
 import java.util.Locale;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
-import org.apache.solr.client.solrj.io.stream.PushBackStream;
 import org.apache.solr.client.solrj.io.stream.StreamContext;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
 import org.apache.solr.client.solrj.io.stream.expr.Explanation;
@@ -39,8 +37,6 @@
 import org.apache.solr.client.solrj.io.stream.expr.Expressible;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
@@ -52,318 +48,311 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Sends tuples emitted by a wrapped {@link TupleStream} as writes to a log file.
- * I really want to call this the DogStream, as it matches the CatStream.
+ * Sends tuples emitted by a wrapped {@link TupleStream} as writes to a log file. The log file will
+ * be created in the "userfiles" directory.
+ *
+ * 
I really want to call this the DogStream, as it matches the CatStream.
+ *
+ * 
WriterStream? LoggingStream? FileoutputStream? JsonOutputStream? LoggingStream??
  *
  * @since 9.8.0
  */
 public class LogStream extends TupleStream implements Expressible {
-    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
-    // field name in summary tuple for #docs updated in batch
-    public static String BATCH_LOGGED_FIELD_NAME = "batchLogged";
-
-    private StreamContext context;
-    private Path chroot;
-
-    /**
-     * The name of the log file that should be written to.  This will be in the same directory that the CatStream is allowed to write to.
-     */
-    private String filepath;
-    private int updateBatchSize;
-
-
-
-    private int batchNumber;
-    private long totalDocsIndex;
-    //private PushBackStream tupleSource;
-    private TupleStream tupleSource;
-    private List documentBatch = new ArrayList<>();
-
-    private OutputStream fos;
-    private final CharArr charArr = new CharArr(1024 * 2);
-    JSONWriter jsonWriter = new JSONWriter(charArr, -1);
-    private Writer writer;
-
-
-    public LogStream(StreamExpression expression, StreamFactory factory) throws IOException {
-
-
-        filepath = factory.getValueOperand(expression, 0);
-        if (filepath == null) {
-            throw new IllegalArgumentException("No filepath provided to log stream to");
-        }
-        final String filepathWithoutSurroundingQuotes =
-                stripSurroundingQuotesIfTheyExist(filepath);
-        if (StrUtils.isNullOrEmpty(filepathWithoutSurroundingQuotes)) {
-            throw new IllegalArgumentException("No filepath provided to stream");
-        }
-
-        this.filepath = filepathWithoutSurroundingQuotes;
-
-        // Extract underlying TupleStream.
-        List streamExpressions =
-                factory.getExpressionOperandsRepresentingTypes(
-                        expression, Expressible.class, TupleStream.class);
-        if (1 != streamExpressions.size()) {
-            throw new IOException(
-                    String.format(
-                            Locale.ROOT,
-                            "Invalid expression %s - expecting a single stream but found %d",
-                            expression,
-                            streamExpressions.size()));
-        }
-        StreamExpression sourceStreamExpression = streamExpressions.get(0);
-        init(filepathWithoutSurroundingQuotes, factory.constructStream(sourceStreamExpression));
-    }
-
-
-    public LogStream(
-            String collectionName, TupleStream tupleSource)
-            throws IOException {
-
-        init(collectionName, tupleSource);
-    }
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
+  // field name in summary tuple for #docs updated in batch
+  public static String BATCH_LOGGED_FIELD_NAME = "batchLogged";
 
-    private void init(
-            String filepaths, TupleStream tupleSource) {
-        this.filepath = filepaths;
-        this.tupleSource = tupleSource;//new PushBackStream(tupleSource);
-    }
+  private StreamContext context;
+  private Path chroot;
 
-    /** The name of the file being updated */
-    protected String getFilePath() {
-        return filepath;
-    }
+  /**
+   * The name of the log file that should be written to. This will be in the same directory that the
+   * CatStream is allowed to write to.
+   */
+  private String filepath;
 
-    @Override
-    public void open() throws IOException {
-        Path filePath = chroot.resolve(filepath).normalize();
-        if (!filePath.startsWith(chroot)) {
-            throw new SolrException(
-                    SolrException.ErrorCode.BAD_REQUEST,
-                    "file to log to must be under " + chroot);
-        }
+  private int updateBatchSize;
 
-//        if (!Files.exists(filePath)) {
-//
-//            throw new SolrException(
-//                    SolrException.ErrorCode.BAD_REQUEST,
-//                    "file/directory to stream doesn't exist: " + crawlRootStr);
-//        }
-        fos = new FileOutputStream(filePath.toFile());
-        writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
+  private int batchNumber;
+  private long totalDocsIndex;
+  // private PushBackStream tupleSource;
+  private TupleStream tupleSource;
+  private List documentBatch = new ArrayList<>();
 
+  private OutputStream fos;
+  private final CharArr charArr = new CharArr(1024 * 2);
+  JSONWriter jsonWriter = new JSONWriter(charArr, -1);
+  private Writer writer;
 
+  public LogStream(StreamExpression expression, StreamFactory factory) throws IOException {
 
-        tupleSource.open();
+    filepath = factory.getValueOperand(expression, 0);
+    if (filepath == null) {
+      throw new IllegalArgumentException("No filepath provided to log stream to");
     }
-
-    @Override
-    public Tuple read() throws IOException {
-
-        Tuple tuple = tupleSource.read();
-        if (tuple.EOF) {
-
-            return tuple;
-        } else {
-            //tupleSource.pushBack(tuple);
-            uploadBatchToCollection(tuple);
-            // return createBatchSummaryTuple(b);
-        }
-
-
-
-        //uploadBatchToCollection(documentBatch);
-        //int b = documentBatch.size();
-        //documentBatch.clear();
-        int b = 1;
-        return createBatchSummaryTuple(b);
+    final String filepathWithoutSurroundingQuotes = stripSurroundingQuotesIfTheyExist(filepath);
+    if (StrUtils.isNullOrEmpty(filepathWithoutSurroundingQuotes)) {
+      throw new IllegalArgumentException("No filepath provided to stream");
     }
 
-    @Override
-    public void close() throws IOException {
-        writer.flush();
-        fos.flush();
-        fos.close();
-        tupleSource.close();
+    this.filepath = filepathWithoutSurroundingQuotes;
+
+    // Extract underlying TupleStream.
+    List streamExpressions =
+        factory.getExpressionOperandsRepresentingTypes(
+            expression, Expressible.class, TupleStream.class);
+    if (1 != streamExpressions.size()) {
+      throw new IOException(
+          String.format(
+              Locale.ROOT,
+              "Invalid expression %s - expecting a single stream but found %d",
+              expression,
+              streamExpressions.size()));
     }
-
-    @Override
-    public StreamComparator getStreamSort() {
-        return tupleSource.getStreamSort();
+    StreamExpression sourceStreamExpression = streamExpressions.get(0);
+    init(filepathWithoutSurroundingQuotes, factory.constructStream(sourceStreamExpression));
+  }
+
+  public LogStream(String filepath, TupleStream tupleSource) throws IOException {
+
+    init(filepath, tupleSource);
+  }
+
+  private void init(String filepath, TupleStream tupleSource) {
+    this.filepath = filepath;
+    this.tupleSource = tupleSource;
+  }
+
+  /** The name of the file being updated */
+  protected String getFilePath() {
+    return filepath;
+  }
+
+  @Override
+  public void open() throws IOException {
+    Path filePath = chroot.resolve(filepath).normalize();
+    if (!filePath.startsWith(chroot)) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST, "file to log to must be under " + chroot);
     }
 
-    @Override
-    public List children() {
-        ArrayList sourceList = new ArrayList<>(1);
-        sourceList.add(tupleSource);
-        return sourceList;
+    //        if (!Files.exists(filePath)) {
+    //
+    //            throw new SolrException(
+    //                    SolrException.ErrorCode.BAD_REQUEST,
+    //                    "file/directory to stream doesn't exist: " + crawlRootStr);
+    //        }
+    fos = new FileOutputStream(filePath.toFile());
+    writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
+
+    tupleSource.open();
+  }
+
+  @Override
+  public Tuple read() throws IOException {
+
+    Tuple tuple = tupleSource.read();
+    if (tuple.EOF) {
+
+      return tuple;
+    } else {
+      // tupleSource.pushBack(tuple);
+      uploadBatchToCollection(tuple);
+      // return createBatchSummaryTuple(b);
     }
 
-    @Override
-    public StreamExpression toExpression(StreamFactory factory) throws IOException {
-        return toExpression(factory, true);
+    // uploadBatchToCollection(documentBatch);
+    // int b = documentBatch.size();
+    // documentBatch.clear();
+    int b = 1;
+    return createBatchSummaryTuple(b);
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (writer != null) {
+      writer.flush();
     }
-
-    private StreamExpression toExpression(StreamFactory factory, boolean includeStreams)
-            throws IOException {
-        StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
-        expression.addParameter(filepath);
-        //expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost));
-        //expression.addParameter(
-        //        new StreamExpressionNamedParameter("batchSize", Integer.toString(updateBatchSize)));
-
-        if (includeStreams) {
-            if (tupleSource != null) {
-                expression.addParameter(((Expressible) tupleSource).toExpression(factory));
-            } else {
-                throw new IOException(
-                        "This LogStream contains a non-expressible TupleStream - it cannot be converted to an expression");
-            }
-        } else {
-            expression.addParameter("");
-        }
-
-        return expression;
+    if (fos != null) {
+      fos.flush();
+      fos.close();
+    }
+    tupleSource.close();
+  }
+
+  @Override
+  public StreamComparator getStreamSort() {
+    return tupleSource.getStreamSort();
+  }
+
+  @Override
+  public List children() {
+    ArrayList sourceList = new ArrayList<>(1);
+    sourceList.add(tupleSource);
+    return sourceList;
+  }
+
+  @Override
+  public StreamExpression toExpression(StreamFactory factory) throws IOException {
+    return toExpression(factory, true);
+  }
+
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams)
+      throws IOException {
+    StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
+    expression.addParameter(filepath);
+    // expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost));
+    // expression.addParameter(
+    //        new StreamExpressionNamedParameter("batchSize", Integer.toString(updateBatchSize)));
+
+    if (includeStreams) {
+      if (tupleSource != null) {
+        expression.addParameter(((Expressible) tupleSource).toExpression(factory));
+      } else {
+        throw new IOException(
+            "This LogStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+      }
+    } else {
+      expression.addParameter("");
     }
 
-    @Override
-    public Explanation toExplanation(StreamFactory factory) throws IOException {
+    return expression;
+  }
 
-        // An update stream is backward wrt the order in the explanation. This stream is the "child"
-        // while the collection we're updating is the parent.
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
 
-        StreamExplanation explanation = new StreamExplanation(getStreamNodeId() + "-datastore");
+    // An update stream is backward wrt the order in the explanation. This stream is the "child"
+    // while the collection we're updating is the parent.
 
-        explanation.setFunctionName(String.format(Locale.ROOT, "log (%s)", filepath));
-        explanation.setImplementingClass("Solr/Lucene");
-        explanation.setExpressionType(ExpressionType.DATASTORE);
-        explanation.setExpression("Log into " + filepath);
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId() + "-datastore");
 
-        // child is a datastore so add it at this point
-        StreamExplanation child = new StreamExplanation(getStreamNodeId().toString());
-        child.setFunctionName(String.format(Locale.ROOT, factory.getFunctionName(getClass())));
-        child.setImplementingClass(getClass().getName());
-        child.setExpressionType(ExpressionType.STREAM_DECORATOR);
-        child.setExpression(toExpression(factory, false).toString());
-        child.addChild(tupleSource.toExplanation(factory));
+    explanation.setFunctionName(String.format(Locale.ROOT, "log (%s)", filepath));
+    explanation.setImplementingClass("Solr/Lucene");
+    explanation.setExpressionType(ExpressionType.DATASTORE);
+    explanation.setExpression("Log into " + filepath);
 
-        explanation.addChild(child);
+    // child is a datastore so add it at this point
+    StreamExplanation child = new StreamExplanation(getStreamNodeId().toString());
+    child.setFunctionName(String.format(Locale.ROOT, factory.getFunctionName(getClass())));
+    child.setImplementingClass(getClass().getName());
+    child.setExpressionType(ExpressionType.STREAM_DECORATOR);
+    child.setExpression(toExpression(factory, false).toString());
+    child.addChild(tupleSource.toExplanation(factory));
 
-        return explanation;
-    }
+    explanation.addChild(child);
 
-    @Override
-    public void setStreamContext(StreamContext context) {
-        this.context = context;
-        Object solrCoreObj = context.get("solr-core");
-        if (solrCoreObj == null || !(solrCoreObj instanceof SolrCore)) {
-            throw new SolrException(
-                    SolrException.ErrorCode.INVALID_STATE,
-                    "StreamContext must have SolrCore in solr-core key");
-        }
-        final SolrCore core = (SolrCore) context.get("solr-core");
-
-        this.chroot = core.getCoreContainer().getUserFilesPath();
-        if (!Files.exists(chroot)) {
-            throw new IllegalStateException(
-                    chroot + " directory used to load files must exist but could not be found!");
-        }
-    }
+    return explanation;
+  }
 
-    private void verifyCollectionName(String collectionName, StreamExpression expression)
-            throws IOException {
-        if (null == collectionName) {
-            throw new IOException(
-                    String.format(
-                            Locale.ROOT,
-                            "invalid expression %s - collectionName expected as first operand",
-                            expression));
-        }
+  @Override
+  public void setStreamContext(StreamContext context) {
+    this.context = context;
+    Object solrCoreObj = context.get("solr-core");
+    if (solrCoreObj == null || !(solrCoreObj instanceof SolrCore)) {
+      throw new SolrException(
+          SolrException.ErrorCode.INVALID_STATE,
+          "StreamContext must have SolrCore in solr-core key");
     }
+    final SolrCore core = (SolrCore) context.get("solr-core");
 
-
-//    private SolrInputDocument convertTupleTJson(Tuple tuple) {
-//        SolrInputDocument doc = new SolrInputDocument();
-//        for (String field : tuple.getFields().keySet()) {
-//
-//            if (!(field.equals(CommonParams.VERSION_FIELD) )) {
-//                Object value = tuple.get(field);
-//                if (value instanceof List) {
-//                    addMultivaluedField(doc, field, (List) value);
-//                } else {
-//                    doc.addField(field, value);
-//                }
-//            }
-//        }
-//        log.debug("Tuple [{}] was converted into SolrInputDocument [{}].", tuple, doc);
-//        jsonWriter
-//        return doc;
-//    }
-
-    private void addMultivaluedField(SolrInputDocument doc, String fieldName, List values) {
-        for (Object value : values) {
-            doc.addField(fieldName, value);
-        }
+    this.chroot = core.getCoreContainer().getUserFilesPath();
+    if (!Files.exists(chroot)) {
+      throw new IllegalStateException(
+          chroot + " directory used to load files must exist but could not be found!");
     }
-
-    /**
-     * This method will be called on every batch of tuples comsumed, after converting each tuple in
-     * that batch to a Solr Input Document.
-     */
-    protected void uploadBatchToCollection(Tuple doc) throws IOException {
-        charArr.reset();
-//        doc.toMap()
-//        Map m =doc.toMap()
-//        doc.forEach(
-//                (s, field) -> {
-//                    if (s.equals("_version_") || s.equals("_roor_")) return;
-//                    if (field instanceof List) {
-//                        if (((List) field).size() == 1) {
-//                            field = ((List) field).get(0);
-//                        }
-//                    }
-//                    field = constructDateStr(field);
-//                    if (field instanceof List) {
-//                        List list = (List) field;
-//                        if (hasdate(list)) {
-//                            ArrayList listCopy = new ArrayList<>(list.size());
-//                            for (Object o : list) listCopy.add(constructDateStr(o));
-//                            field = listCopy;
-//                        }
-//                    }
-//                    m.put(s, field);
-//                });
-        //jsonWriter.write(m);
-        jsonWriter.write(doc);
-        writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());
-        writer.append('\n');
+  }
+
+  private void verifyCollectionName(String collectionName, StreamExpression expression)
+      throws IOException {
+    if (null == collectionName) {
+      throw new IOException(
+          String.format(
+              Locale.ROOT,
+              "invalid expression %s - collectionName expected as first operand",
+              expression));
     }
-
-    private Tuple createBatchSummaryTuple(int batchSize) {
-        assert batchSize > 0;
-        Tuple tuple = new Tuple();
-        this.totalDocsIndex += batchSize;
-        ++batchNumber;
-        tuple.put(BATCH_LOGGED_FIELD_NAME, batchSize);
-        tuple.put("totalIndexed", this.totalDocsIndex);
-        tuple.put("batchNumber", batchNumber);
-       // if (coreName != null) {
-       //     tuple.put("worker", coreName);
-        //}
-        return tuple;
+  }
+
+  //    private SolrInputDocument convertTupleTJson(Tuple tuple) {
+  //        SolrInputDocument doc = new SolrInputDocument();
+  //        for (String field : tuple.getFields().keySet()) {
+  //
+  //            if (!(field.equals(CommonParams.VERSION_FIELD) )) {
+  //                Object value = tuple.get(field);
+  //                if (value instanceof List) {
+  //                    addMultivaluedField(doc, field, (List) value);
+  //                } else {
+  //                    doc.addField(field, value);
+  //                }
+  //            }
+  //        }
+  //        log.debug("Tuple [{}] was converted into SolrInputDocument [{}].", tuple, doc);
+  //        jsonWriter
+  //        return doc;
+  //    }
+
+  private void addMultivaluedField(SolrInputDocument doc, String fieldName, List values) {
+    for (Object value : values) {
+      doc.addField(fieldName, value);
     }
-
-    private String stripSurroundingQuotesIfTheyExist(String value) {
-        if (value.length() < 2) return value;
-        if ((value.startsWith("\"") && value.endsWith("\""))
-                || (value.startsWith("'") && value.endsWith("'"))) {
-            return value.substring(1, value.length() - 1);
-        }
-
-        return value;
+  }
+
+  /**
+   * This method will be called on every batch of tuples comsumed, after converting each tuple in
+   * that batch to a Solr Input Document.
+   */
+  protected void uploadBatchToCollection(Tuple doc) throws IOException {
+    charArr.reset();
+    //        doc.toMap()
+    //        Map m =doc.toMap()
+    //        doc.forEach(
+    //                (s, field) -> {
+    //                    if (s.equals("_version_") || s.equals("_roor_")) return;
+    //                    if (field instanceof List) {
+    //                        if (((List) field).size() == 1) {
+    //                            field = ((List) field).get(0);
+    //                        }
+    //                    }
+    //                    field = constructDateStr(field);
+    //                    if (field instanceof List) {
+    //                        List list = (List) field;
+    //                        if (hasdate(list)) {
+    //                            ArrayList listCopy = new ArrayList<>(list.size());
+    //                            for (Object o : list) listCopy.add(constructDateStr(o));
+    //                            field = listCopy;
+    //                        }
+    //                    }
+    //                    m.put(s, field);
+    //                });
+    // jsonWriter.write(m);
+    jsonWriter.write(doc);
+    writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());
+    writer.append('\n');
+  }
+
+  private Tuple createBatchSummaryTuple(int batchSize) {
+    assert batchSize > 0;
+    Tuple tuple = new Tuple();
+    this.totalDocsIndex += batchSize;
+    ++batchNumber;
+    tuple.put(BATCH_LOGGED_FIELD_NAME, batchSize);
+    tuple.put("totalIndexed", this.totalDocsIndex);
+    tuple.put("batchNumber", batchNumber);
+    // if (coreName != null) {
+    //     tuple.put("worker", coreName);
+    // }
+    return tuple;
+  }
+
+  private String stripSurroundingQuotesIfTheyExist(String value) {
+    if (value.length() < 2) return value;
+    if ((value.startsWith("\"") && value.endsWith("\""))
+        || (value.startsWith("'") && value.endsWith("'"))) {
+      return value.substring(1, value.length() - 1);
     }
-}
 
+    return value;
+  }
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java b/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java
new file mode 100644
index 00000000000..701b21d85e2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java
@@ -0,0 +1,126 @@
+package org.apache.solr.handler.component;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.io.SolrClientCache;
+import org.apache.solr.client.solrj.io.stream.EchoStream;
+import org.apache.solr.client.solrj.io.stream.StreamContext;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+@SolrTestCaseJ4.SuppressSSL
+public class LogStreamTest extends SolrCloudTestCase {
+  private static StreamFactory factory;
+  private static StreamContext context;
+  private static final String COLLECTION = "streams";
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    configureCluster(1)
+        .addConfig(
+            "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+        .configure();
+
+    CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1, 1, 0)
+        .process(cluster.getSolrClient());
+    cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
+
+    String zkHost = cluster.getZkServer().getZkAddress();
+    factory =
+        new StreamFactory()
+            .withCollectionZkHost(COLLECTION, zkHost)
+            .withFunctionName("logging", LogStream.class)
+            .withFunctionName("echo", EchoStream.class);
+
+    final Path dataDir = findUserFilesDataDir();
+    Files.createDirectories(dataDir);
+
+    context = new StreamContext();
+    context.put("solr-core", findSolrCore());
+    SolrClientCache solrClientCache = new SolrClientCache();
+
+    context.setSolrClientCache(solrClientCache);
+  }
+
+  @Test
+  public void testLogStreamExpressionToExpression() throws Exception {
+    String expressionString;
+
+    // Basic test
+    try (LogStream stream =
+        new LogStream(StreamExpressionParser.parse("logging(bob.txt,echo(\"bob\"))"), factory)) {
+      expressionString = stream.toExpression(factory).toString();
+      assertTrue(expressionString.contains("logging(bob.txt,"));
+      assertTrue(expressionString.contains("echo(\"bob"));
+    }
+
+    // Unwrap double quotes around file name test
+    try (LogStream stream =
+        new LogStream(
+            StreamExpressionParser.parse("logging(\"outputs/bob.txt\",echo(\"bob\"))"), factory)) {
+      expressionString = stream.toExpression(factory).toString();
+      assertTrue(expressionString.contains("logging(outputs/bob.txt,"));
+      assertTrue(expressionString.contains("echo(\"bob"));
+    }
+  }
+
+  @Test
+  public void testFileOutputDirectoryPermissions() throws Exception {
+
+    LogStream stream =
+        new LogStream(StreamExpressionParser.parse("logging(/tmp/bob.txt,echo(\"bob\"))"), factory);
+    stream.setStreamContext(context);
+
+    LogStream finalStream1 = stream;
+    SolrException thrown =
+        assertThrows(
+            "Attempting to write to /tmp should be prevented",
+            SolrException.class,
+            () -> finalStream1.open());
+    assertTrue(thrown.getMessage().startsWith("file to log to must be under "));
+
+    stream =
+        new LogStream(StreamExpressionParser.parse("logging(../bob.txt,echo(\"bob\"))"), factory);
+    stream.setStreamContext(context);
+
+    LogStream finalStream2 = stream;
+    thrown =
+        assertThrows(
+            "Attempting to escape the userfiles directory should be prevented",
+            SolrException.class,
+            () -> finalStream2.open());
+    assertTrue(thrown.getMessage().startsWith("file to log to must be under "));
+  }
+
+  private static Path findUserFilesDataDir() {
+    for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+      for (CoreDescriptor coreDescriptor : jetty.getCoreContainer().getCoreDescriptors()) {
+        if (coreDescriptor.getCollectionName().equals(COLLECTION)) {
+          return jetty.getCoreContainer().getUserFilesPath();
+        }
+      }
+    }
+
+    throw new IllegalStateException("Unable to determine data-dir for: " + COLLECTION);
+  }
+
+  private static SolrCore findSolrCore() {
+    for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
+      for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
+        if (solrCore != null) {
+          return solrCore;
+        }
+      }
+    }
+    throw new RuntimeException("Didn't find any valid cores.");
+  }
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index 4b1619e002d..89e53db3fc0 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -1,5 +1,12 @@
 package org.apache.solr.handler.component;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
 import org.apache.solr.client.solrj.io.Lang;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
@@ -8,165 +15,150 @@
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.request.SolrQueryRequest;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
 public class UBIComponentLocalLoggingTest extends SolrCloudTestCase {
 
-    private static final String COLLECTION = "collection1";
-
-    @BeforeClass
-    public static void setupCluster() throws Exception {
-        configureCluster(1)
-                .addConfig(
-                        "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
-                .configure();
-    }
-
-    @Test
-    @SuppressWarnings({"unchecked", "rawtypes"})
-    public void testLocalCatStream() throws Exception {
-
-        CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1, 1, 0)
-                .process(cluster.getSolrClient());
-        cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
-
-        File localFile = File.createTempFile("topLevel1", ".txt");
-
-        TupleStream stream;
-        List tuples;
-        StreamContext streamContext = new StreamContext();
-        //Replica rr = zkStateReader.getCollection(coll).getReplicas().get(0);
-
-        //cluster.getJettySolrRunner(0).getCoreContainer().getCore()
-//        Replica replica =
-  //              getRandomReplica(
-    //                    shard, (r) -> (r.getState() == Replica.State.ACTIVE && !r.equals(shard.getLeader())));
-
-        SolrCore solrCoreToLoad = null;
-        for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
-            for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
-                if (solrCore != null){
-                    solrCoreToLoad = solrCore;
-                }
-                System.out.println(solrCore);
-            }
+  private static final String COLLECTION = "collection1";
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    configureCluster(1)
+        .addConfig(
+            "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+        .configure();
+  }
+
+  @Test
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  public void testLocalCatStream() throws Exception {
+
+    CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1, 1, 0)
+        .process(cluster.getSolrClient());
+    cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
+
+    File localFile = File.createTempFile("topLevel1", ".txt");
+
+    TupleStream stream;
+    List tuples;
+    StreamContext streamContext = new StreamContext();
+    // Replica rr = zkStateReader.getCollection(coll).getReplicas().get(0);
+
+    // cluster.getJettySolrRunner(0).getCoreContainer().getCore()
+    //        Replica replica =
+    //              getRandomReplica(
+    //                    shard, (r) -> (r.getState() == Replica.State.ACTIVE &&
+    // !r.equals(shard.getLeader())));
+
+    SolrCore solrCoreToLoad = null;
+    for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
+      for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
+        if (solrCore != null) {
+          solrCoreToLoad = solrCore;
         }
+        System.out.println(solrCore);
+      }
+    }
 
-        final Path dataDir = findUserFilesDataDir();
-        Files.createDirectories(dataDir);
-        //populateFileStreamData(dataDir);
-
-        //JettySolrRunner replicaJetty = cluster.getReplicaJetty(replica);
-        //cluster.getJettySolrRunner(0).getCoreContainer().getr
-
-
-        //SolrQueryRequest req = req("q", "*:*");
-CoreContainer cc = cluster.getJettySolrRunner(0).getCoreContainer();
-
-        var l = cc.getAllCoreNames();
-SolrCore core = cc.getCore(l.get(0));
-        streamContext.put("solr-core", core);
-        SolrClientCache solrClientCache = new SolrClientCache();
-
-        streamContext.setSolrClientCache(solrClientCache);
-
-        StreamFactory streamFactory = new StreamFactory();
-
+    final Path dataDir = findUserFilesDataDir();
+    Files.createDirectories(dataDir);
+    // populateFileStreamData(dataDir);
 
-        // LocalCatStream extends CatStream and disables the Solr cluster specific
-        // logic about where to read data from.
-        streamFactory.withFunctionName("logging", LogStream.class);
+    // JettySolrRunner replicaJetty = cluster.getReplicaJetty(replica);
+    // cluster.getJettySolrRunner(0).getCoreContainer().getr
 
+    // SolrQueryRequest req = req("q", "*:*");
+    CoreContainer cc = cluster.getJettySolrRunner(0).getCoreContainer();
 
-        Lang.register(streamFactory);
+    var l = cc.getAllCoreNames();
+    SolrCore core = cc.getCore(l.get(0));
+    streamContext.put("solr-core", core);
+    SolrClientCache solrClientCache = new SolrClientCache();
 
-        String clause = "logging(bob.txt,echo(\"bob\"))";
-        stream = streamFactory.constructStream(clause);
-        stream.setStreamContext(streamContext);
-        tuples = getTuples(stream);
-        stream.close();
-        solrClientCache.close();
+    streamContext.setSolrClientCache(solrClientCache);
 
+    StreamFactory streamFactory = new StreamFactory();
 
-        //populateFileWithData(localFile.toPath());
+    // LocalCatStream extends CatStream and disables the Solr cluster specific
+    // logic about where to read data from.
+    streamFactory.withFunctionName("logging", LogStream.class);
 
+    Lang.register(streamFactory);
 
-        Tuple tuple = new Tuple(new HashMap());
-        tuple.put("field1", "blah");
-        tuple.put("field2", "blah");
-        tuple.put("field3", "blah");
+    String clause = "logging(bob.txt,echo(\"bob\"))";
+    stream = streamFactory.constructStream(clause);
+    stream.setStreamContext(streamContext);
+    tuples = getTuples(stream);
+    stream.close();
+    solrClientCache.close();
 
-       //LogStream logStream =
-         ///    //   new LogStream(localFile.getAbsolutePath());
-       // LogStream logStream =
-         //              new LogStream();
-        List tuples2 = new ArrayList();
-        try {
-           // logStream.open();
+    // populateFileWithData(localFile.toPath());
 
+    Tuple tuple = new Tuple(new HashMap());
+    tuple.put("field1", "blah");
+    tuple.put("field2", "blah");
+    tuple.put("field3", "blah");
 
+    // LogStream logStream =
+    ///    //   new LogStream(localFile.getAbsolutePath());
+    // LogStream logStream =
+    //              new LogStream();
+    List tuples2 = new ArrayList();
+    try {
+      // logStream.open();
 
-//            while (true) {
-//                Tuple tuple = logStream.read();
-//                if (tuple.EOF) {
-//                    break;
-//                } else {
-//                    tuples.add(tuple);
-//                }
-//            }
+      //            while (true) {
+      //                Tuple tuple = logStream.read();
+      //                if (tuple.EOF) {
+      //                    break;
+      //                } else {
+      //                    tuples.add(tuple);
+      //                }
+      //            }
 
-        } finally {
-         //   logStream.close();
-        }
+    } finally {
+      //   logStream.close();
+    }
 
-        assertEquals(4, tuples.size());
+    assertEquals(1, tuples.size());
 
-        for (int i = 0; i < 4; i++) {
-            Tuple t = tuples.get(i);
-            assertEquals(localFile.getName() + " line " + (i + 1), t.get("line"));
-            assertEquals(localFile.getAbsolutePath(), t.get("file"));
-        }
-    }
+    //    for (int i = 0; i < 1; i++) {
+    //      Tuple t = tuples.get(i);
+    //      assertEquals(localFile.getName() + " line " + (i + 1), t.get("line"));
+    //      assertEquals(localFile.getAbsolutePath(), t.get("file"));
+    //    }
+  }
 
-    private static Path findUserFilesDataDir() {
-        for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
-            for (CoreDescriptor coreDescriptor : jetty.getCoreContainer().getCoreDescriptors()) {
-                if (coreDescriptor.getCollectionName().equals(COLLECTION)) {
-                    return jetty.getCoreContainer().getUserFilesPath();
-                }
-            }
+  private static Path findUserFilesDataDir() {
+    for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+      for (CoreDescriptor coreDescriptor : jetty.getCoreContainer().getCoreDescriptors()) {
+        if (coreDescriptor.getCollectionName().equals(COLLECTION)) {
+          return jetty.getCoreContainer().getUserFilesPath();
         }
-
-        throw new IllegalStateException("Unable to determine data-dir for: " + COLLECTION);
+      }
     }
 
-    private List getTuples(TupleStream tupleStream) throws IOException {
-        tupleStream.open();
-        List tuples = new ArrayList<>();
-        for (; ; ) {
-            Tuple t = tupleStream.read();
-            // log.info(" ... {}", t.fields);
-            if (t.EOF) {
-                break;
-            } else {
-                tuples.add(t);
-        }
-        tupleStream.close();
-        return tuples;
+    throw new IllegalStateException("Unable to determine data-dir for: " + COLLECTION);
+  }
+
+  private List getTuples(TupleStream tupleStream) throws IOException {
+    tupleStream.open();
+    List tuples = new ArrayList<>();
+    for (; ; ) {
+      Tuple t = tupleStream.read();
+      // log.info(" ... {}", t.fields);
+      if (t.EOF) {
+        break;
+      } else {
+        tuples.add(t);
+      }
     }
+    tupleStream.close();
+    return tuples;
+  }
 }

From 9c98ab655689f25977000ce3db86d41746064338 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 11:45:06 -0600
Subject: [PATCH 038/103] Polishing up the rendering of the expression

---
 .../apache/solr/handler/component/LogStream.java    | 11 ++++-------
 .../solr/handler/component/LogStreamTest.java       | 13 +++++++++++++
 .../solr/client/solrj/io/stream/UpdateStream.java   |  2 +-
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LogStream.java b/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
index a43d2cdeccd..ab3a50aacbc 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
@@ -223,21 +223,18 @@ private StreamExpression toExpression(StreamFactory factory, boolean includeStre
   @Override
   public Explanation toExplanation(StreamFactory factory) throws IOException {
 
-    // An update stream is backward wrt the order in the explanation. This stream is the "child"
-    // while the collection we're updating is the parent.
-
     StreamExplanation explanation = new StreamExplanation(getStreamNodeId() + "-datastore");
 
-    explanation.setFunctionName(String.format(Locale.ROOT, "log (%s)", filepath));
-    explanation.setImplementingClass("Solr/Lucene");
+    explanation.setFunctionName(String.format(Locale.ROOT, "logging (%s)", filepath));
+    explanation.setImplementingClass(this.getClass().getName());
     explanation.setExpressionType(ExpressionType.DATASTORE);
-    explanation.setExpression("Log into " + filepath);
+    explanation.setExpression("Log tuples into " + filepath);
 
     // child is a datastore so add it at this point
     StreamExplanation child = new StreamExplanation(getStreamNodeId().toString());
     child.setFunctionName(String.format(Locale.ROOT, factory.getFunctionName(getClass())));
     child.setImplementingClass(getClass().getName());
-    child.setExpressionType(ExpressionType.STREAM_DECORATOR);
+    child.setExpressionType(ExpressionType.DATASTORE);
     child.setExpression(toExpression(factory, false).toString());
     child.addChild(tupleSource.toExplanation(factory));
 
diff --git a/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java b/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java
index 701b21d85e2..d63f1375cc0 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java
@@ -6,6 +6,7 @@
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.stream.EchoStream;
 import org.apache.solr.client.solrj.io.stream.StreamContext;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -73,6 +74,18 @@ public void testLogStreamExpressionToExpression() throws Exception {
     }
   }
 
+  @Test
+  public void testLogStreamExpressionToExplanation() throws Exception {
+
+    try (LogStream stream =
+        new LogStream(StreamExpressionParser.parse("logging(bob.txt,echo(\"bob\"))"), factory)) {
+      String expressionString = stream.toExpression(factory).toString();
+      Explanation explanation = stream.toExplanation(factory);
+      assertEquals("logging (bob.txt)", explanation.getFunctionName());
+      assertEquals(LogStream.class.getName(), explanation.getImplementingClass());
+    }
+  }
+
   @Test
   public void testFileOutputDirectoryPermissions() throws Exception {
 
diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java
index 8c3ba20fc8b..7c698e2e339 100644
--- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java
+++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java
@@ -211,7 +211,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException {
     StreamExplanation explanation = new StreamExplanation(getStreamNodeId() + "-datastore");
 
     explanation.setFunctionName(String.format(Locale.ROOT, "solr (%s)", collection));
-    explanation.setImplementingClass("Solr/Lucene");
+    explanation.setImplementingClass(this.getClass().getName());
     explanation.setExpressionType(ExpressionType.DATASTORE);
     explanation.setExpression("Update into " + collection);
 

From 17f05ed99d1107c9f5347b85f6fd9f13d3a2f401 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 13:36:14 -0600
Subject: [PATCH 039/103] Handle exception when loading streams.

---
 .../org/apache/solr/handler/component/UBIComponent.java   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index bfc20d8104a..5ae1bc403e3 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -195,8 +195,12 @@ public void inform(SolrCore core) {
 
         try {
           stream = constructStream(streamFactory, streamExpression);
-        } catch (IOException e) {
-          e.printStackTrace();
+        } catch (IOException exception) {
+          throw new SolrException(
+              SolrException.ErrorCode.SERVER_ERROR,
+              "Error constructing stream for processing UBI data collection: "
+                  + UBIComponent.class.getSimpleName(),
+              exception);
         }
         stream.setStreamContext(streamContext);
 

From d198f7e5c305928706951370878753d8cda8d5f6 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 13:55:53 -0600
Subject: [PATCH 040/103] Log --> Logging to avoid confusion with "logarithmic"

---
 .../{LogStream.java => LoggingStream.java}    |  10 +-
 .../solr/handler/component/LogStreamTest.java | 139 -----------
 .../handler/component/LoggingStreamTest.java  | 222 ++++++++++++++++++
 .../UBIComponentLocalLoggingTest.java         |  10 +-
 4 files changed, 233 insertions(+), 148 deletions(-)
 rename solr/core/src/java/org/apache/solr/handler/component/{LogStream.java => LoggingStream.java} (96%)
 delete mode 100644 solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java
 create mode 100644 solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LogStream.java b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
similarity index 96%
rename from solr/core/src/java/org/apache/solr/handler/component/LogStream.java
rename to solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
index ab3a50aacbc..905097dc882 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LogStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
@@ -53,11 +53,13 @@
  *
  * 
I really want to call this the DogStream, as it matches the CatStream.
  *
+ * 
Is this generically useful to be added to the streaming jar and Lang?
+ *
  * 
WriterStream? LoggingStream? FileoutputStream? JsonOutputStream? LoggingStream??
  *
  * @since 9.8.0
  */
-public class LogStream extends TupleStream implements Expressible {
+public class LoggingStream extends TupleStream implements Expressible {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   // field name in summary tuple for #docs updated in batch
@@ -85,7 +87,7 @@ public class LogStream extends TupleStream implements Expressible {
   JSONWriter jsonWriter = new JSONWriter(charArr, -1);
   private Writer writer;
 
-  public LogStream(StreamExpression expression, StreamFactory factory) throws IOException {
+  public LoggingStream(StreamExpression expression, StreamFactory factory) throws IOException {
 
     filepath = factory.getValueOperand(expression, 0);
     if (filepath == null) {
@@ -114,7 +116,7 @@ public LogStream(StreamExpression expression, StreamFactory factory) throws IOEx
     init(filepathWithoutSurroundingQuotes, factory.constructStream(sourceStreamExpression));
   }
 
-  public LogStream(String filepath, TupleStream tupleSource) throws IOException {
+  public LoggingStream(String filepath, TupleStream tupleSource) throws IOException {
 
     init(filepath, tupleSource);
   }
@@ -211,7 +213,7 @@ private StreamExpression toExpression(StreamFactory factory, boolean includeStre
         expression.addParameter(((Expressible) tupleSource).toExpression(factory));
       } else {
         throw new IOException(
-            "This LogStream contains a non-expressible TupleStream - it cannot be converted to an expression");
+            "This LoggingStream contains a non-expressible TupleStream - it cannot be converted to an expression");
       }
     } else {
       expression.addParameter("");
diff --git a/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java b/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java
deleted file mode 100644
index d63f1375cc0..00000000000
--- a/solr/core/src/test/org/apache/solr/handler/component/LogStreamTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-package org.apache.solr.handler.component;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.io.SolrClientCache;
-import org.apache.solr.client.solrj.io.stream.EchoStream;
-import org.apache.solr.client.solrj.io.stream.StreamContext;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@SolrTestCaseJ4.SuppressSSL
-public class LogStreamTest extends SolrCloudTestCase {
-  private static StreamFactory factory;
-  private static StreamContext context;
-  private static final String COLLECTION = "streams";
-
-  @BeforeClass
-  public static void setupCluster() throws Exception {
-    configureCluster(1)
-        .addConfig(
-            "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
-        .configure();
-
-    CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1, 1, 0)
-        .process(cluster.getSolrClient());
-    cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
-
-    String zkHost = cluster.getZkServer().getZkAddress();
-    factory =
-        new StreamFactory()
-            .withCollectionZkHost(COLLECTION, zkHost)
-            .withFunctionName("logging", LogStream.class)
-            .withFunctionName("echo", EchoStream.class);
-
-    final Path dataDir = findUserFilesDataDir();
-    Files.createDirectories(dataDir);
-
-    context = new StreamContext();
-    context.put("solr-core", findSolrCore());
-    SolrClientCache solrClientCache = new SolrClientCache();
-
-    context.setSolrClientCache(solrClientCache);
-  }
-
-  @Test
-  public void testLogStreamExpressionToExpression() throws Exception {
-    String expressionString;
-
-    // Basic test
-    try (LogStream stream =
-        new LogStream(StreamExpressionParser.parse("logging(bob.txt,echo(\"bob\"))"), factory)) {
-      expressionString = stream.toExpression(factory).toString();
-      assertTrue(expressionString.contains("logging(bob.txt,"));
-      assertTrue(expressionString.contains("echo(\"bob"));
-    }
-
-    // Unwrap double quotes around file name test
-    try (LogStream stream =
-        new LogStream(
-            StreamExpressionParser.parse("logging(\"outputs/bob.txt\",echo(\"bob\"))"), factory)) {
-      expressionString = stream.toExpression(factory).toString();
-      assertTrue(expressionString.contains("logging(outputs/bob.txt,"));
-      assertTrue(expressionString.contains("echo(\"bob"));
-    }
-  }
-
-  @Test
-  public void testLogStreamExpressionToExplanation() throws Exception {
-
-    try (LogStream stream =
-        new LogStream(StreamExpressionParser.parse("logging(bob.txt,echo(\"bob\"))"), factory)) {
-      String expressionString = stream.toExpression(factory).toString();
-      Explanation explanation = stream.toExplanation(factory);
-      assertEquals("logging (bob.txt)", explanation.getFunctionName());
-      assertEquals(LogStream.class.getName(), explanation.getImplementingClass());
-    }
-  }
-
-  @Test
-  public void testFileOutputDirectoryPermissions() throws Exception {
-
-    LogStream stream =
-        new LogStream(StreamExpressionParser.parse("logging(/tmp/bob.txt,echo(\"bob\"))"), factory);
-    stream.setStreamContext(context);
-
-    LogStream finalStream1 = stream;
-    SolrException thrown =
-        assertThrows(
-            "Attempting to write to /tmp should be prevented",
-            SolrException.class,
-            () -> finalStream1.open());
-    assertTrue(thrown.getMessage().startsWith("file to log to must be under "));
-
-    stream =
-        new LogStream(StreamExpressionParser.parse("logging(../bob.txt,echo(\"bob\"))"), factory);
-    stream.setStreamContext(context);
-
-    LogStream finalStream2 = stream;
-    thrown =
-        assertThrows(
-            "Attempting to escape the userfiles directory should be prevented",
-            SolrException.class,
-            () -> finalStream2.open());
-    assertTrue(thrown.getMessage().startsWith("file to log to must be under "));
-  }
-
-  private static Path findUserFilesDataDir() {
-    for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
-      for (CoreDescriptor coreDescriptor : jetty.getCoreContainer().getCoreDescriptors()) {
-        if (coreDescriptor.getCollectionName().equals(COLLECTION)) {
-          return jetty.getCoreContainer().getUserFilesPath();
-        }
-      }
-    }
-
-    throw new IllegalStateException("Unable to determine data-dir for: " + COLLECTION);
-  }
-
-  private static SolrCore findSolrCore() {
-    for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
-      for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
-        if (solrCore != null) {
-          return solrCore;
-        }
-      }
-    }
-    throw new RuntimeException("Didn't find any valid cores.");
-  }
-}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java b/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
new file mode 100644
index 00000000000..b6664bc1c22
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.io.SolrClientCache;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.CsvStream;
+import org.apache.solr.client.solrj.io.stream.EchoStream;
+import org.apache.solr.client.solrj.io.stream.ListStream;
+import org.apache.solr.client.solrj.io.stream.SolrStream;
+import org.apache.solr.client.solrj.io.stream.StreamContext;
+import org.apache.solr.client.solrj.io.stream.TupStream;
+import org.apache.solr.client.solrj.io.stream.TupleStream;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+@SolrTestCaseJ4.SuppressSSL
+public class LoggingStreamTest extends SolrCloudTestCase {
+  private static StreamFactory factory;
+  private static StreamContext context;
+  private static final String COLLECTION = "streams";
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    configureCluster(1)
+        .addConfig(
+            "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+        .configure();
+
+    CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1, 1, 0)
+        .process(cluster.getSolrClient());
+    cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
+
+    String zkHost = cluster.getZkServer().getZkAddress();
+    factory =
+        new StreamFactory()
+            .withCollectionZkHost(COLLECTION, zkHost)
+            .withFunctionName("logging", LoggingStream.class)
+            .withFunctionName("echo", EchoStream.class)
+            .withFunctionName("parseCSV", CsvStream.class)
+            .withFunctionName("list", ListStream.class)
+            .withFunctionName("tuple", TupStream.class);
+
+    final Path dataDir = findUserFilesDataDir();
+    Files.createDirectories(dataDir);
+
+    context = new StreamContext();
+    context.put("solr-core", findSolrCore());
+    SolrClientCache solrClientCache = new SolrClientCache();
+
+    context.setSolrClientCache(solrClientCache);
+  }
+
+  @Test
+  public void testLogStreamExpressionToExpression() throws Exception {
+    String expressionString;
+
+    // Basic test
+    try (LoggingStream stream =
+        new LoggingStream(
+            StreamExpressionParser.parse("logging(bob.txt,echo(\"bob\"))"), factory)) {
+      expressionString = stream.toExpression(factory).toString();
+      assertTrue(expressionString.contains("logging(bob.txt,"));
+      assertTrue(expressionString.contains("echo(\"bob"));
+    }
+
+    // Unwrap double quotes around file name test
+    try (LoggingStream stream =
+        new LoggingStream(
+            StreamExpressionParser.parse("logging(\"outputs/bob.txt\",echo(\"bob\"))"), factory)) {
+      expressionString = stream.toExpression(factory).toString();
+      assertTrue(expressionString.contains("logging(outputs/bob.txt,"));
+      assertTrue(expressionString.contains("echo(\"bob"));
+    }
+  }
+
+  @Test
+  public void testLogStreamExpressionToExplanation() throws Exception {
+
+    try (LoggingStream stream =
+        new LoggingStream(
+            StreamExpressionParser.parse("logging(bob.txt,echo(\"bob\"))"), factory)) {
+      Explanation explanation = stream.toExplanation(factory);
+      assertEquals("logging (bob.txt)", explanation.getFunctionName());
+      assertEquals(LoggingStream.class.getName(), explanation.getImplementingClass());
+    }
+  }
+
+  @Test
+  public void testFileOutputDirectoryPermissions() throws Exception {
+
+    LoggingStream stream =
+        new LoggingStream(
+            StreamExpressionParser.parse("logging(/tmp/bob.txt,echo(\"bob\"))"), factory);
+    stream.setStreamContext(context);
+
+    LoggingStream finalStream1 = stream;
+    SolrException thrown =
+        assertThrows(
+            "Attempting to write to /tmp should be prevented",
+            SolrException.class,
+            () -> finalStream1.open());
+    assertTrue(thrown.getMessage().startsWith("file to log to must be under "));
+
+    stream =
+        new LoggingStream(
+            StreamExpressionParser.parse("logging(../bob.txt,echo(\"bob\"))"), factory);
+    stream.setStreamContext(context);
+
+    LoggingStream finalStream2 = stream;
+    thrown =
+        assertThrows(
+            "Attempting to escape the userfiles directory should be prevented",
+            SolrException.class,
+            () -> finalStream2.open());
+    assertTrue(thrown.getMessage().startsWith("file to log to must be under "));
+  }
+
+  @Test
+  public void testLoggingStreamCombinedWithCatAndJsonStream() throws Exception {
+    String expr =
+        "logging(parsed_csv_output.log,"
+            + "parseCSV(list(tuple(file=\"file1\", line=\"a,b,c\"), "
+            + "                        tuple(file=\"file1\", line=\"1,2,3\"),"
+            + "                        tuple(file=\"file1\", line=\"\\\"hello, world\\\",9000,20\"),"
+            + "                        tuple(file=\"file2\", line=\"field_1,field_2,field_3\"), "
+            + "                        tuple(file=\"file2\", line=\"8,9,\")))"
+            + ")";
+
+    try (LoggingStream stream = new LoggingStream(StreamExpressionParser.parse(expr), factory)) {
+      List tuples = getTuples(stream);
+    }
+
+    ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+    paramsLoc.set("expr", expr);
+    paramsLoc.set("qt", "/stream");
+
+    String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" + COLLECTION;
+    TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+    StreamContext context = new StreamContext();
+    solrStream.setStreamContext(context);
+    List tuples = getTuples(solrStream);
+    assertEquals(tuples.size(), 3);
+    assertEquals(tuples.get(0).getString("a"), "1");
+    assertEquals(tuples.get(0).getString("b"), "2");
+    assertEquals(tuples.get(0).getString("c"), "3");
+
+    assertEquals(tuples.get(1).getString("a"), "hello, world");
+    assertEquals(tuples.get(1).getString("b"), "9000");
+    assertEquals(tuples.get(1).getString("c"), "20");
+
+    assertEquals(tuples.get(2).getString("field_1"), "8");
+    assertEquals(tuples.get(2).getString("field_2"), "9");
+    assertNull(tuples.get(2).get("field_3"));
+  }
+
+  private static Path findUserFilesDataDir() {
+    for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+      for (CoreDescriptor coreDescriptor : jetty.getCoreContainer().getCoreDescriptors()) {
+        if (coreDescriptor.getCollectionName().equals(COLLECTION)) {
+          return jetty.getCoreContainer().getUserFilesPath();
+        }
+      }
+    }
+
+    throw new IllegalStateException("Unable to determine data-dir for: " + COLLECTION);
+  }
+
+  private static SolrCore findSolrCore() {
+    for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
+      for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
+        if (solrCore != null) {
+          return solrCore;
+        }
+      }
+    }
+    throw new RuntimeException("Didn't find any valid cores.");
+  }
+
+  protected List getTuples(TupleStream tupleStream) throws IOException {
+    List tuples = new ArrayList<>();
+
+    try (tupleStream) {
+      tupleStream.open();
+      for (Tuple t = tupleStream.read(); !t.EOF; t = tupleStream.read()) {
+        tuples.add(t);
+      }
+    }
+    return tuples;
+  }
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index 89e53db3fc0..760acb188b2 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -86,7 +86,7 @@ public void testLocalCatStream() throws Exception {
 
     // LocalCatStream extends CatStream and disables the Solr cluster specific
     // logic about where to read data from.
-    streamFactory.withFunctionName("logging", LogStream.class);
+    streamFactory.withFunctionName("logging", LoggingStream.class);
 
     Lang.register(streamFactory);
 
@@ -104,10 +104,10 @@ public void testLocalCatStream() throws Exception {
     tuple.put("field2", "blah");
     tuple.put("field3", "blah");
 
-    // LogStream logStream =
-    ///    //   new LogStream(localFile.getAbsolutePath());
-    // LogStream logStream =
-    //              new LogStream();
+    // LoggingStream logStream =
+    ///    //   new LoggingStream(localFile.getAbsolutePath());
+    // LoggingStream logStream =
+    //              new LoggingStream();
     List tuples2 = new ArrayList();
     try {
       // logStream.open();

From b93457fc52b6ee4113c36089996aebf1096c93a3 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 14:05:27 -0600
Subject: [PATCH 041/103] Lint

---
 .../component/UBIComponentLocalLoggingTest.java  | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index 760acb188b2..f7b1062d793 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.solr.handler.component;
 
 import java.io.File;

From e0a59bf09023f87c433c367b785b05f1b966e4bf Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 14:26:28 -0600
Subject: [PATCH 042/103] Manage streamcontexts better

---
 .../solr/handler/component/LoggingStream.java |  5 ++-
 .../handler/component/LoggingStreamTest.java  | 40 +++++++------------
 2 files changed, 18 insertions(+), 27 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
index 905097dc882..530c86ffd98 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
@@ -49,7 +49,7 @@
 
 /**
  * Sends tuples emitted by a wrapped {@link TupleStream} as writes to a log file. The log file will
- * be created in the "userfiles" directory.
+ * be created in the "userfiles" directory and formatted in the JSON w/ Lines format.
  *
  * 
I really want to call this the DogStream, as it matches the CatStream.
  *
@@ -261,6 +261,9 @@ public void setStreamContext(StreamContext context) {
       throw new IllegalStateException(
           chroot + " directory used to load files must exist but could not be found!");
     }
+
+    // Pass down the stream context.
+    this.tupleSource.setStreamContext(context);
   }
 
   private void verifyCollectionName(String collectionName, StreamExpression expression)
diff --git a/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java b/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
index b6664bc1c22..aafe807e4bf 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
@@ -27,7 +27,6 @@
 import org.apache.solr.client.solrj.io.stream.CsvStream;
 import org.apache.solr.client.solrj.io.stream.EchoStream;
 import org.apache.solr.client.solrj.io.stream.ListStream;
-import org.apache.solr.client.solrj.io.stream.SolrStream;
 import org.apache.solr.client.solrj.io.stream.StreamContext;
 import org.apache.solr.client.solrj.io.stream.TupStream;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
@@ -37,7 +36,6 @@
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.cloud.SolrCloudTestCase;
 import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.embedded.JettySolrRunner;
@@ -149,7 +147,7 @@ public void testFileOutputDirectoryPermissions() throws Exception {
   @Test
   public void testLoggingStreamCombinedWithCatAndJsonStream() throws Exception {
     String expr =
-        "logging(parsed_csv_output.log,"
+        "logging(parsed_csv_output.jsonl,"
             + "parseCSV(list(tuple(file=\"file1\", line=\"a,b,c\"), "
             + "                        tuple(file=\"file1\", line=\"1,2,3\"),"
             + "                        tuple(file=\"file1\", line=\"\\\"hello, world\\\",9000,20\"),"
@@ -158,31 +156,21 @@ public void testLoggingStreamCombinedWithCatAndJsonStream() throws Exception {
             + ")";
 
     try (LoggingStream stream = new LoggingStream(StreamExpressionParser.parse(expr), factory)) {
+      stream.setStreamContext(context);
       List tuples = getTuples(stream);
+      assertEquals(tuples.size(), 3);
+      assertEquals(tuples.get(0).getString("totalIndexed"), "1");
+      assertEquals(tuples.get(0).getString("batchLogged"), "1");
+      assertEquals(tuples.get(0).getString("batchNumber"), "1");
+
+      assertEquals(tuples.get(1).getString("totalIndexed"), "2");
+      assertEquals(tuples.get(1).getString("batchLogged"), "1");
+      assertEquals(tuples.get(1).getString("batchNumber"), "2");
+
+      assertEquals(tuples.get(2).getString("totalIndexed"), "3");
+      assertEquals(tuples.get(2).getString("batchLogged"), "1");
+      assertEquals(tuples.get(2).getString("batchNumber"), "3");
     }
-
-    ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
-    paramsLoc.set("expr", expr);
-    paramsLoc.set("qt", "/stream");
-
-    String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" + COLLECTION;
-    TupleStream solrStream = new SolrStream(url, paramsLoc);
-
-    StreamContext context = new StreamContext();
-    solrStream.setStreamContext(context);
-    List tuples = getTuples(solrStream);
-    assertEquals(tuples.size(), 3);
-    assertEquals(tuples.get(0).getString("a"), "1");
-    assertEquals(tuples.get(0).getString("b"), "2");
-    assertEquals(tuples.get(0).getString("c"), "3");
-
-    assertEquals(tuples.get(1).getString("a"), "hello, world");
-    assertEquals(tuples.get(1).getString("b"), "9000");
-    assertEquals(tuples.get(1).getString("c"), "20");
-
-    assertEquals(tuples.get(2).getString("field_1"), "8");
-    assertEquals(tuples.get(2).getString("field_2"), "9");
-    assertNull(tuples.get(2).get("field_3"));
   }
 
   private static Path findUserFilesDataDir() {

From aaa3b06451897f3eada33942420b76b947108135 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 16:43:12 -0600
Subject: [PATCH 043/103] Clean up unused code

---
 .../UBIComponentLocalLoggingTest.java         | 49 -------------------
 1 file changed, 49 deletions(-)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index f7b1062d793..f116c90bfdf 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -16,12 +16,10 @@
  */
 package org.apache.solr.handler.component;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import org.apache.solr.client.solrj.io.Lang;
 import org.apache.solr.client.solrj.io.SolrClientCache;
@@ -51,25 +49,17 @@ public static void setupCluster() throws Exception {
   }
 
   @Test
-  @SuppressWarnings({"unchecked", "rawtypes"})
   public void testLocalCatStream() throws Exception {
 
     CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1, 1, 0)
         .process(cluster.getSolrClient());
     cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
 
-    File localFile = File.createTempFile("topLevel1", ".txt");
 
     TupleStream stream;
     List tuples;
     StreamContext streamContext = new StreamContext();
-    // Replica rr = zkStateReader.getCollection(coll).getReplicas().get(0);
 
-    // cluster.getJettySolrRunner(0).getCoreContainer().getCore()
-    //        Replica replica =
-    //              getRandomReplica(
-    //                    shard, (r) -> (r.getState() == Replica.State.ACTIVE &&
-    // !r.equals(shard.getLeader())));
 
     SolrCore solrCoreToLoad = null;
     for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
@@ -77,7 +67,6 @@ public void testLocalCatStream() throws Exception {
         if (solrCore != null) {
           solrCoreToLoad = solrCore;
         }
-        System.out.println(solrCore);
       }
     }
 
@@ -85,10 +74,6 @@ public void testLocalCatStream() throws Exception {
     Files.createDirectories(dataDir);
     // populateFileStreamData(dataDir);
 
-    // JettySolrRunner replicaJetty = cluster.getReplicaJetty(replica);
-    // cluster.getJettySolrRunner(0).getCoreContainer().getr
-
-    // SolrQueryRequest req = req("q", "*:*");
     CoreContainer cc = cluster.getJettySolrRunner(0).getCoreContainer();
 
     var l = cc.getAllCoreNames();
@@ -113,41 +98,7 @@ public void testLocalCatStream() throws Exception {
     stream.close();
     solrClientCache.close();
 
-    // populateFileWithData(localFile.toPath());
-
-    Tuple tuple = new Tuple(new HashMap());
-    tuple.put("field1", "blah");
-    tuple.put("field2", "blah");
-    tuple.put("field3", "blah");
-
-    // LoggingStream logStream =
-    ///    //   new LoggingStream(localFile.getAbsolutePath());
-    // LoggingStream logStream =
-    //              new LoggingStream();
-    List tuples2 = new ArrayList();
-    try {
-      // logStream.open();
-
-      //            while (true) {
-      //                Tuple tuple = logStream.read();
-      //                if (tuple.EOF) {
-      //                    break;
-      //                } else {
-      //                    tuples.add(tuple);
-      //                }
-      //            }
-
-    } finally {
-      //   logStream.close();
-    }
-
-    assertEquals(1, tuples.size());
 
-    //    for (int i = 0; i < 1; i++) {
-    //      Tuple t = tuples.get(i);
-    //      assertEquals(localFile.getName() + " line " + (i + 1), t.get("line"));
-    //      assertEquals(localFile.getAbsolutePath(), t.get("file"));
-    //    }
   }
 
   private static Path findUserFilesDataDir() {

From 133c21340072c838c7579ae43bf9e14b60e26a54 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 16:43:21 -0600
Subject: [PATCH 044/103] Language improvement

---
 .../org/apache/solr/client/solrj/io/stream/TupStream.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TupStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TupStream.java
index bf982011f6a..0f1f1161bca 100644
--- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TupStream.java
+++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TupStream.java
@@ -67,8 +67,8 @@ public TupStream(StreamExpression expression, StreamFactory factory) throws IOEx
       fieldLabels.put(name, name);
       StreamExpressionParameter param = np.getParameter();
 
-      // we're going to split these up here so we only make the choice once
-      // order of these in read() doesn't matter
+      // We're going to split these up here, so we only make the choice once.
+      // The order of these in read() doesn't matter.
       if (param instanceof StreamExpressionValue) {
         stringParams.put(name, ((StreamExpressionValue) param).getValue());
       } else if (factory.isEvaluator((StreamExpression) param)) {
@@ -140,7 +140,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException {
   public void setStreamContext(StreamContext context) {
     this.streamContext = context;
 
-    // also set in evalators and streams
+    // also set in evaluators and streams
     for (StreamEvaluator evaluator : evaluatorParams.values()) {
       evaluator.setStreamContext(context);
     }

From c1dd697eff75222d6036a46255556986597140f4 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 17:14:06 -0600
Subject: [PATCH 045/103] lint

---
 .../solr/handler/component/UBIComponentLocalLoggingTest.java  | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index f116c90bfdf..fd0f2411c68 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -55,12 +55,10 @@ public void testLocalCatStream() throws Exception {
         .process(cluster.getSolrClient());
     cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
 
-
     TupleStream stream;
     List tuples;
     StreamContext streamContext = new StreamContext();
 
-
     SolrCore solrCoreToLoad = null;
     for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
       for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
@@ -97,8 +95,6 @@ public void testLocalCatStream() throws Exception {
     tuples = getTuples(stream);
     stream.close();
     solrClientCache.close();
-
-
   }
 
   private static Path findUserFilesDataDir() {

From ef90a7d6daaf1910483b4344f31585186182d650 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 19:06:49 -0600
Subject: [PATCH 046/103] Rollback the idea of logging via log4j, it's too
 awkward...

---
 .../solr/handler/component/LoggingStream.java |  9 +++++--
 .../solr/handler/component/UBIComponent.java  | 27 ++++++++++---------
 solr/packaging/test/test_ubi.bats             |  2 +-
 solr/server/resources/log4j2.xml              | 19 -------------
 .../solrj/io/stream/StreamDecoratorTest.java  |  2 +-
 5 files changed, 24 insertions(+), 35 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
index 530c86ffd98..34209e6deb3 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
@@ -258,8 +258,13 @@ public void setStreamContext(StreamContext context) {
 
     this.chroot = core.getCoreContainer().getUserFilesPath();
     if (!Files.exists(chroot)) {
-      throw new IllegalStateException(
-          chroot + " directory used to load files must exist but could not be found!");
+      try {
+        Files.createDirectories(this.chroot);
+      } catch (IOException ioe) {
+        throw new SolrException(
+            SolrException.ErrorCode.INVALID_STATE,
+            chroot + " directory used to load files must exist but and couldn't be created!");
+      }
     }
 
     // Pass down the stream context.
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 5ae1bc403e3..82242072b2a 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -148,23 +148,25 @@ public void inform(SolrCore core) {
       log.info("Initializing UBIComponent");
       if (coreContainer.isZooKeeperAware()) {
         String defaultZkHost = core.getCoreContainer().getZkController().getZkServerAddress();
-        String ubiQueryProcessingExpression = initArgs.get("ubiQueryProcessingExpression");
+        String ubiQueryStreamProcessingExpression =
+            initArgs.get("ubiQueryStreamProcessingExpression");
 
-        String expr = null;
-        if (ubiQueryProcessingExpression == null) {
-          log.info(
-              "You should provide a ubiQueryProcessingExpression to control how UBI query information is persisted.");
-          log.info("Writing out UBI query information to local log file ubi_queries.log instead.");
+        String expr;
 
+        if (ubiQueryStreamProcessingExpression == null) {
+          log.info(
+              "You should provide a ubiQueryStreamProcessingExpression to control how UBI query information is persisted.");
+          log.info(
+              "Writing out UBI query information to local log file ubi_queries.jsonl instead.");
+          expr = "logging(ubi_queries.jsonl," + "tuple(id=49,a_i=1,b_i=5)" + ")";
         } else {
-
           LineNumberReader bufferedReader;
 
           try {
             bufferedReader =
                 new LineNumberReader(
                     new InputStreamReader(
-                        core.getResourceLoader().openResource(ubiQueryProcessingExpression),
+                        core.getResourceLoader().openResource(ubiQueryStreamProcessingExpression),
                         StandardCharsets.UTF_8));
 
             String[] args = {}; // maybe we have variables?
@@ -178,16 +180,18 @@ public void inform(SolrCore core) {
           } catch (IOException ioe) {
             throw new SolrException(
                 SolrException.ErrorCode.SERVER_ERROR,
-                "Error reading file " + ubiQueryProcessingExpression,
+                "Error reading file " + ubiQueryStreamProcessingExpression,
                 ioe);
           }
         }
-        StreamContext streamContext = new StreamContext();
 
+        StreamContext streamContext = new StreamContext();
+        streamContext.put("solr-core", core);
         streamContext.setSolrClientCache(solrClientCache);
 
         StreamExpression streamExpression = StreamExpressionParser.parse(expr);
         StreamFactory streamFactory = new StreamFactory();
+        streamFactory.withFunctionName("logging", LoggingStream.class);
 
         streamFactory.withDefaultZkHost(defaultZkHost);
 
@@ -281,10 +285,9 @@ protected void processIds(
     ubiQueryLogInfo.add(QUERY_ATTRIBUTES, ubiQuery.getQueryAttributes());
     ubiQueryLogInfo.add("doc_ids", docIds);
 
-    // pushBackStream = new PushBackStream(stream);
     if (stream != null) {
-      // getTuples invokes the streaming expression.
       List tuples = getTuples(stream);
+      log.error("Here are the tuples (" + tuples.size() + "):" + tuples);
     }
   }
 
diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats
index b9fb7016205..bc52b8e51fe 100644
--- a/solr/packaging/test/test_ubi.bats
+++ b/solr/packaging/test/test_ubi.bats
@@ -30,7 +30,7 @@ teardown() {
 }
 
 @test "Run set up process" {
-  solr start -c -e techproducts
+  solr start -e techproducts
   
   run solr healthcheck -c techproducts
   refute_output --partial 'error'
diff --git a/solr/server/resources/log4j2.xml b/solr/server/resources/log4j2.xml
index e94a2c68bb9..24325525713 100644
--- a/solr/server/resources/log4j2.xml
+++ b/solr/server/resources/log4j2.xml
@@ -59,22 +59,6 @@
       
       
     
-    
-    
-      
-        
-          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
-        
-      
-      
-        
-        
-      
-      
-    
 
   
   
@@ -89,9 +73,6 @@
     
     
       
-    
-    
-      
         
     
     
diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java
index 2b19742ae6f..d293b5f66d3 100644
--- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java
+++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java
@@ -1264,7 +1264,7 @@ public void testDaemonStream() throws Exception {
           .add(id, "10", "a_s", "hello0", "a_i", "1", "a_f", "10")
           .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
 
-      // Now lets clear the existing docs in the queue 9, plus 3 more to get passed the run that was
+      // Now lets clear the existing docs in the queue 9, plus 3 more to get pass the run that was
       // blocked. The next run should
       // have the tuples with the updated count.
       for (int i = 0; i < 12; i++) {

From 228bb996f0ad06a34ea34ed5a1e869af04cd29a1 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 19:10:31 -0600
Subject: [PATCH 047/103] Remove whitespace

---
 solr/server/resources/log4j2.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/solr/server/resources/log4j2.xml b/solr/server/resources/log4j2.xml
index 24325525713..c4dd7c891ee 100644
--- a/solr/server/resources/log4j2.xml
+++ b/solr/server/resources/log4j2.xml
@@ -73,7 +73,7 @@
     
     
       
-        
+    
     
     
     

From 2e639c52bcf6ad410bb48a7d2b68c4ede2663617 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 12 Oct 2024 22:34:18 -0600
Subject: [PATCH 048/103] track the changes

---
 .../solr/handler/component/UBIComponent.java  |  2 +-
 .../ubi-enabled/conf/solrconfig.xml           |  2 +-
 .../ubi-enabled/conf/ubi-query-pipeline.expr  |  4 +-
 .../UBIComponentLocalLoggingTest.java         |  1 +
 .../UBIComponentStreamingQueriesTest.java     | 66 +++++++++++--------
 5 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 82242072b2a..2aef862bc0d 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -155,7 +155,7 @@ public void inform(SolrCore core) {
 
         if (ubiQueryStreamProcessingExpression == null) {
           log.info(
-              "You should provide a ubiQueryStreamProcessingExpression to control how UBI query information is persisted.");
+              "No 'ubiQueryStreamProcessingExpression' file provided to describe processing of UBI query information.");
           log.info(
               "Writing out UBI query information to local log file ubi_queries.jsonl instead.");
           expr = "logging(ubi_queries.jsonl," + "tuple(id=49,a_i=1,b_i=5)" + ")";
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml
index 62cdc180c57..2b493e0259b 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml
@@ -37,7 +37,7 @@
   
   
   
-    ubi-query-pipeline.expr
+    ubi-query-pipeline.expr
   
   
   
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
index 801650bbced..208287abc69 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-commit(ubi,
-  update(ubi,
+commit(ubi_queries,
+  update(ubi_queries,
     tuple(id=49,a_i=1,b_i=5)
   )
 )
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index fd0f2411c68..6670234f181 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -64,6 +64,7 @@ public void testLocalCatStream() throws Exception {
       for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
         if (solrCore != null) {
           solrCoreToLoad = solrCore;
+          break;
         }
       }
     }
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index eb5afed8350..7333cdf92bd 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -52,8 +52,9 @@
  * index using Streaming Expressions.
  */
 public class UBIComponentStreamingQueriesTest extends SolrCloudTestCase {
-  public static final String COLLECTION = "conf2_col";
-  public static final String UBI_COLLECTION = "ubi";
+  public static final String COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION = "collection_stream_ubi_queries_to_ubi_collection";
+  public static final String COLLECTION_STREAM_UBI_QUERIES_TO_LOG = "collection_stream_ubi_queries_to_log";
+  public static final String UBI_QUERIES_COLLECTION = "ubi_queries";
 
   /** One client per node */
   private static final List NODE_CLIENTS = new ArrayList<>(7);
@@ -78,7 +79,8 @@ public static void setupCluster() throws Exception {
     // The ubi collection itself just depends on the typical _default configset.
     configureCluster(numNodes)
         .addConfig("ubi-enabled", configset("ubi-enabled"))
-        .addConfig("ubi", configset("_default"))
+            .addConfig(
+                    "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
         .configure();
 
     zkHost = cluster.getZkServer().getZkAddress();
@@ -91,22 +93,32 @@ public static void setupCluster() throws Exception {
     }
 
     assertEquals(
-        "failed to create collection",
+        "failed to create collection " + COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION,
         0,
-        CollectionAdminRequest.createCollection(COLLECTION, "ubi-enabled", numShards, numReplicas)
+        CollectionAdminRequest.createCollection(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, "ubi-enabled", numShards, numReplicas)
             .process(cluster.getSolrClient())
             .getStatus());
 
-    cluster.waitForActiveCollection(COLLECTION, numShards, numShards * numReplicas);
+    cluster.waitForActiveCollection(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, numShards, numShards * numReplicas);
 
     assertEquals(
-        "failed to create UBI collection",
+            "failed to create collection " + COLLECTION_STREAM_UBI_QUERIES_TO_LOG,
+            0,
+            CollectionAdminRequest.createCollection(COLLECTION_STREAM_UBI_QUERIES_TO_LOG, "config", numShards, numReplicas)
+                    .process(cluster.getSolrClient())
+                    .getStatus());
+
+    cluster.waitForActiveCollection(COLLECTION_STREAM_UBI_QUERIES_TO_LOG, numShards, numShards * numReplicas);
+
+
+    assertEquals(
+        "failed to create UBI queries collection",
         0,
-        CollectionAdminRequest.createCollection(UBI_COLLECTION, "_default", numShards, numReplicas)
+        CollectionAdminRequest.createCollection(UBI_QUERIES_COLLECTION, "_default", numShards, numReplicas)
             .process(cluster.getSolrClient())
             .getStatus());
 
-    cluster.waitForActiveCollection(UBI_COLLECTION, numShards, numShards * numReplicas);
+    cluster.waitForActiveCollection(UBI_QUERIES_COLLECTION, numShards, numShards * numReplicas);
   }
 
   @AfterClass
@@ -122,11 +134,11 @@ public static void closeClients() throws Exception {
   @After
   public void clearCollection() throws Exception {
     assertEquals(
-        "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(COLLECTION, "*:*").getStatus());
-    assertEquals("commit failed", 0, cluster.getSolrClient().commit(COLLECTION).getStatus());
+        "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, "*:*").getStatus());
+    assertEquals("commit failed", 0, cluster.getSolrClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
     assertEquals(
-        "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(UBI_COLLECTION, "*:*").getStatus());
-    assertEquals("commit failed", 0, cluster.getSolrClient().commit(UBI_COLLECTION).getStatus());
+        "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(UBI_QUERIES_COLLECTION, "*:*").getStatus());
+    assertEquals("commit failed", 0, cluster.getSolrClient().commit(UBI_QUERIES_COLLECTION).getStatus());
   }
 
   public void testCreatingStreamingExpression() {
@@ -152,7 +164,7 @@ public void testUsingStreamingExpressionDirectly() throws Exception {
 
     streamContext.setSolrClientCache(solrClientCache);
 
-    StreamFactory streamFactory = new StreamFactory().withCollectionZkHost("ubi", zkHost);
+    StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(UBI_QUERIES_COLLECTION, zkHost);
 
     Lang.register(streamFactory);
 
@@ -174,7 +186,7 @@ public void testUsingStreamingExpressionDirectly() throws Exception {
 
     // Randomly grab a client, it shouldn't matter which is used to check UBI event.
     SolrClient client = getRandClient();
-    final QueryResponse responseUBI = requestFromUBICollection.process(client, UBI_COLLECTION);
+    final QueryResponse responseUBI = requestFromUBICollection.process(client, UBI_QUERIES_COLLECTION);
     try {
       assertEquals(0, responseUBI.getStatus());
       assertEquals(1, responseUBI.getResults().getNumFound());
@@ -200,7 +212,7 @@ private List getTuples(TupleStream tupleStream) throws IOException {
   }
 
   private static String getClause(UBIQuery ubiQuery) {
-    return "commit(ubi,update(ubi,tuple(id=4.0," + ubiQuery.toTuple() + ")))";
+    return "commit(" + UBI_QUERIES_COLLECTION +",update("+UBI_QUERIES_COLLECTION+",tuple(id=4.0," + ubiQuery.toTuple() + ")))";
   }
 
   @SuppressWarnings({"rawtypes", "unchecked"})
@@ -209,8 +221,8 @@ public void testRandomDocs() throws Exception {
     final UpdateRequest ureq = new UpdateRequest();
 
     ureq.add(sdoc("id", 1, "data_s", "data_1"));
-    assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION).getStatus());
-    assertEquals("commit failed", 0, getRandClient().commit(COLLECTION).getStatus());
+    assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
+    assertEquals("commit failed", 0, getRandClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
 
     // query our collection to generate a UBI event and then confirm it was recorded.
 
@@ -229,7 +241,7 @@ public void testRandomDocs() throws Exception {
 
     // Randomly grab a client, it shouldn't matter which is used to generate the query event.
     SolrClient client = getRandClient();
-    final QueryResponse rsp = req.process(client, COLLECTION);
+    final QueryResponse rsp = req.process(client, COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION);
     try {
       assertEquals(0, rsp.getStatus());
       assertEquals(1, rsp.getResults().getNumFound());
@@ -243,13 +255,13 @@ public void testRandomDocs() throws Exception {
     // Randomly grab a client, it shouldn't matter which is used, to check UBI event was actually
     // tracked.
     client = getRandClient();
-    final QueryResponse responseUBI = requestUBI.process(client, UBI_COLLECTION);
-    try {
+    final QueryResponse responseUBI = requestUBI.process(client, UBI_QUERIES_COLLECTION);
+    //try {
       assertEquals(0, responseUBI.getStatus());
       assertEquals(1, responseUBI.getResults().getNumFound());
-    } catch (AssertionError e) {
-      throw new AssertionError(responseUBI + " + " + client + " => " + e.getMessage(), e);
-    }
+    //} catch (AssertionError e) {
+    //  throw new AssertionError(responseUBI + " + " + client + " => " + e.getMessage(), e);
+    //}
   }
 
   public void randomDocs() throws Exception {
@@ -268,9 +280,9 @@ public void randomDocs() throws Exception {
             sdoc( // NOTE: No 'id' field, SignatureUpdateProcessor fills it in for us
                 "data_s", (docCounter % uniqueMod)));
       }
-      assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION).getStatus());
+      assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
     }
-    assertEquals("commit failed", 0, getRandClient().commit(COLLECTION).getStatus());
+    assertEquals("commit failed", 0, getRandClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
 
     assertTrue(docCounter > uniqueMod);
 
@@ -282,7 +294,7 @@ public void randomDocs() throws Exception {
             .setLimit(0)
             .withFacet("data_facet", new TermsFacetMap("data_s").setLimit(uniqueMod + 1));
     for (SolrClient client : CLIENTS) {
-      final QueryResponse rsp = req.process(client, COLLECTION);
+      final QueryResponse rsp = req.process(client, COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION);
       try {
         assertEquals(0, rsp.getStatus());
         assertEquals(uniqueMod, rsp.getResults().getNumFound());

From d1e215eaa5a9ef854eb1530875e42b403a502220 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 13 Nov 2024 15:39:12 -0500
Subject: [PATCH 049/103] Lint

---
 .../UBIComponentStreamingQueriesTest.java     | 107 +++++++++++++-----
 1 file changed, 76 insertions(+), 31 deletions(-)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index 7333cdf92bd..1696c5c1f17 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -52,8 +52,10 @@
  * index using Streaming Expressions.
  */
 public class UBIComponentStreamingQueriesTest extends SolrCloudTestCase {
-  public static final String COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION = "collection_stream_ubi_queries_to_ubi_collection";
-  public static final String COLLECTION_STREAM_UBI_QUERIES_TO_LOG = "collection_stream_ubi_queries_to_log";
+  public static final String COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION =
+      "collection_stream_ubi_queries_to_ubi_collection";
+  public static final String COLLECTION_STREAM_UBI_QUERIES_TO_LOG =
+      "collection_stream_ubi_queries_to_log";
   public static final String UBI_QUERIES_COLLECTION = "ubi_queries";
 
   /** One client per node */
@@ -79,8 +81,8 @@ public static void setupCluster() throws Exception {
     // The ubi collection itself just depends on the typical _default configset.
     configureCluster(numNodes)
         .addConfig("ubi-enabled", configset("ubi-enabled"))
-            .addConfig(
-                    "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+        .addConfig(
+            "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
         .configure();
 
     zkHost = cluster.getZkServer().getZkAddress();
@@ -95,26 +97,33 @@ public static void setupCluster() throws Exception {
     assertEquals(
         "failed to create collection " + COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION,
         0,
-        CollectionAdminRequest.createCollection(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, "ubi-enabled", numShards, numReplicas)
+        CollectionAdminRequest.createCollection(
+                COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION,
+                "ubi-enabled",
+                numShards,
+                numReplicas)
             .process(cluster.getSolrClient())
             .getStatus());
 
-    cluster.waitForActiveCollection(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, numShards, numShards * numReplicas);
+    cluster.waitForActiveCollection(
+        COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, numShards, numShards * numReplicas);
 
     assertEquals(
-            "failed to create collection " + COLLECTION_STREAM_UBI_QUERIES_TO_LOG,
-            0,
-            CollectionAdminRequest.createCollection(COLLECTION_STREAM_UBI_QUERIES_TO_LOG, "config", numShards, numReplicas)
-                    .process(cluster.getSolrClient())
-                    .getStatus());
-
-    cluster.waitForActiveCollection(COLLECTION_STREAM_UBI_QUERIES_TO_LOG, numShards, numShards * numReplicas);
+        "failed to create collection " + COLLECTION_STREAM_UBI_QUERIES_TO_LOG,
+        0,
+        CollectionAdminRequest.createCollection(
+                COLLECTION_STREAM_UBI_QUERIES_TO_LOG, "config", numShards, numReplicas)
+            .process(cluster.getSolrClient())
+            .getStatus());
 
+    cluster.waitForActiveCollection(
+        COLLECTION_STREAM_UBI_QUERIES_TO_LOG, numShards, numShards * numReplicas);
 
     assertEquals(
         "failed to create UBI queries collection",
         0,
-        CollectionAdminRequest.createCollection(UBI_QUERIES_COLLECTION, "_default", numShards, numReplicas)
+        CollectionAdminRequest.createCollection(
+                UBI_QUERIES_COLLECTION, "_default", numShards, numReplicas)
             .process(cluster.getSolrClient())
             .getStatus());
 
@@ -134,11 +143,25 @@ public static void closeClients() throws Exception {
   @After
   public void clearCollection() throws Exception {
     assertEquals(
-        "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, "*:*").getStatus());
-    assertEquals("commit failed", 0, cluster.getSolrClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
+        "DBQ failed",
+        0,
+        cluster
+            .getSolrClient()
+            .deleteByQuery(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, "*:*")
+            .getStatus());
+    assertEquals(
+        "commit failed",
+        0,
+        cluster
+            .getSolrClient()
+            .commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION)
+            .getStatus());
     assertEquals(
-        "DBQ failed", 0, cluster.getSolrClient().deleteByQuery(UBI_QUERIES_COLLECTION, "*:*").getStatus());
-    assertEquals("commit failed", 0, cluster.getSolrClient().commit(UBI_QUERIES_COLLECTION).getStatus());
+        "DBQ failed",
+        0,
+        cluster.getSolrClient().deleteByQuery(UBI_QUERIES_COLLECTION, "*:*").getStatus());
+    assertEquals(
+        "commit failed", 0, cluster.getSolrClient().commit(UBI_QUERIES_COLLECTION).getStatus());
   }
 
   public void testCreatingStreamingExpression() {
@@ -164,7 +187,8 @@ public void testUsingStreamingExpressionDirectly() throws Exception {
 
     streamContext.setSolrClientCache(solrClientCache);
 
-    StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(UBI_QUERIES_COLLECTION, zkHost);
+    StreamFactory streamFactory =
+        new StreamFactory().withCollectionZkHost(UBI_QUERIES_COLLECTION, zkHost);
 
     Lang.register(streamFactory);
 
@@ -186,7 +210,8 @@ public void testUsingStreamingExpressionDirectly() throws Exception {
 
     // Randomly grab a client, it shouldn't matter which is used to check UBI event.
     SolrClient client = getRandClient();
-    final QueryResponse responseUBI = requestFromUBICollection.process(client, UBI_QUERIES_COLLECTION);
+    final QueryResponse responseUBI =
+        requestFromUBICollection.process(client, UBI_QUERIES_COLLECTION);
     try {
       assertEquals(0, responseUBI.getStatus());
       assertEquals(1, responseUBI.getResults().getNumFound());
@@ -212,7 +237,13 @@ private List getTuples(TupleStream tupleStream) throws IOException {
   }
 
   private static String getClause(UBIQuery ubiQuery) {
-    return "commit(" + UBI_QUERIES_COLLECTION +",update("+UBI_QUERIES_COLLECTION+",tuple(id=4.0," + ubiQuery.toTuple() + ")))";
+    return "commit("
+        + UBI_QUERIES_COLLECTION
+        + ",update("
+        + UBI_QUERIES_COLLECTION
+        + ",tuple(id=4.0,"
+        + ubiQuery.toTuple()
+        + ")))";
   }
 
   @SuppressWarnings({"rawtypes", "unchecked"})
@@ -221,8 +252,14 @@ public void testRandomDocs() throws Exception {
     final UpdateRequest ureq = new UpdateRequest();
 
     ureq.add(sdoc("id", 1, "data_s", "data_1"));
-    assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
-    assertEquals("commit failed", 0, getRandClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
+    assertEquals(
+        "add failed",
+        0,
+        ureq.process(getRandClient(), COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
+    assertEquals(
+        "commit failed",
+        0,
+        getRandClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
 
     // query our collection to generate a UBI event and then confirm it was recorded.
 
@@ -256,12 +293,12 @@ public void testRandomDocs() throws Exception {
     // tracked.
     client = getRandClient();
     final QueryResponse responseUBI = requestUBI.process(client, UBI_QUERIES_COLLECTION);
-    //try {
-      assertEquals(0, responseUBI.getStatus());
-      assertEquals(1, responseUBI.getResults().getNumFound());
-    //} catch (AssertionError e) {
+    // try {
+    assertEquals(0, responseUBI.getStatus());
+    assertEquals(1, responseUBI.getResults().getNumFound());
+    // } catch (AssertionError e) {
     //  throw new AssertionError(responseUBI + " + " + client + " => " + e.getMessage(), e);
-    //}
+    // }
   }
 
   public void randomDocs() throws Exception {
@@ -280,9 +317,16 @@ public void randomDocs() throws Exception {
             sdoc( // NOTE: No 'id' field, SignatureUpdateProcessor fills it in for us
                 "data_s", (docCounter % uniqueMod)));
       }
-      assertEquals("add failed", 0, ureq.process(getRandClient(), COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
+      assertEquals(
+          "add failed",
+          0,
+          ureq.process(getRandClient(), COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION)
+              .getStatus());
     }
-    assertEquals("commit failed", 0, getRandClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
+    assertEquals(
+        "commit failed",
+        0,
+        getRandClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
 
     assertTrue(docCounter > uniqueMod);
 
@@ -294,7 +338,8 @@ public void randomDocs() throws Exception {
             .setLimit(0)
             .withFacet("data_facet", new TermsFacetMap("data_s").setLimit(uniqueMod + 1));
     for (SolrClient client : CLIENTS) {
-      final QueryResponse rsp = req.process(client, COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION);
+      final QueryResponse rsp =
+          req.process(client, COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION);
       try {
         assertEquals(0, rsp.getStatus());
         assertEquals(uniqueMod, rsp.getResults().getNumFound());

From 47d57ead5ee61e9b9cfb0dbee83b3a9c645f8b85 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 13 Nov 2024 17:33:39 -0500
Subject: [PATCH 050/103] Integration test of simple UBI setup.

---
 .../solr/handler/component/LoggingStream.java | 23 +-----------
 .../solr/handler/component/UBIComponent.java  |  6 ++--
 solr/packaging/test/test_ubi.bats             | 36 +++++--------------
 3 files changed, 13 insertions(+), 52 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
index 34209e6deb3..fdeb3dd9d03 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
@@ -65,7 +65,6 @@ public class LoggingStream extends TupleStream implements Expressible {
   // field name in summary tuple for #docs updated in batch
   public static String BATCH_LOGGED_FIELD_NAME = "batchLogged";
 
-  private StreamContext context;
   private Path chroot;
 
   /**
@@ -139,12 +138,6 @@ public void open() throws IOException {
           SolrException.ErrorCode.BAD_REQUEST, "file to log to must be under " + chroot);
     }
 
-    //        if (!Files.exists(filePath)) {
-    //
-    //            throw new SolrException(
-    //                    SolrException.ErrorCode.BAD_REQUEST,
-    //                    "file/directory to stream doesn't exist: " + crawlRootStr);
-    //        }
     fos = new FileOutputStream(filePath.toFile());
     writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
 
@@ -204,9 +197,6 @@ private StreamExpression toExpression(StreamFactory factory, boolean includeStre
       throws IOException {
     StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
     expression.addParameter(filepath);
-    // expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost));
-    // expression.addParameter(
-    //        new StreamExpressionNamedParameter("batchSize", Integer.toString(updateBatchSize)));
 
     if (includeStreams) {
       if (tupleSource != null) {
@@ -247,7 +237,6 @@ public Explanation toExplanation(StreamFactory factory) throws IOException {
 
   @Override
   public void setStreamContext(StreamContext context) {
-    this.context = context;
     Object solrCoreObj = context.get("solr-core");
     if (solrCoreObj == null || !(solrCoreObj instanceof SolrCore)) {
       throw new SolrException(
@@ -271,16 +260,6 @@ public void setStreamContext(StreamContext context) {
     this.tupleSource.setStreamContext(context);
   }
 
-  private void verifyCollectionName(String collectionName, StreamExpression expression)
-      throws IOException {
-    if (null == collectionName) {
-      throw new IOException(
-          String.format(
-              Locale.ROOT,
-              "invalid expression %s - collectionName expected as first operand",
-              expression));
-    }
-  }
 
   //    private SolrInputDocument convertTupleTJson(Tuple tuple) {
   //        SolrInputDocument doc = new SolrInputDocument();
@@ -307,7 +286,7 @@ private void addMultivaluedField(SolrInputDocument doc, String fieldName, List tuples = getTuples(stream);
       log.error("Here are the tuples (" + tuples.size() + "):" + tuples);
     }
+    else {
+      log.error("UBI Query Stream is null, can't log query information.");
+    }
   }
 
   protected List getTuples(TupleStream tupleStream) throws IOException {
diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats
index bc52b8e51fe..392d09b5615 100644
--- a/solr/packaging/test/test_ubi.bats
+++ b/solr/packaging/test/test_ubi.bats
@@ -26,7 +26,7 @@ teardown() {
   save_home_on_failure
 
   delete_all_collections
-  SOLR_STOP_WAIT=1 solr stop -all >/dev/null 2>&1
+  SOLR_STOP_WAIT=1 solr stop --all >/dev/null 2>&1
 }
 
 @test "Run set up process" {
@@ -48,7 +48,7 @@ teardown() {
 
   assert_output --partial '"status":0'
   
-  curl -X POST -H 'Content-type:application/json' -d '{
+  run curl -X POST -H 'Content-type:application/json' -d '{
     "update-requesthandler": {
       "name": "/select",
       "class": "solr.SearchHandler",
@@ -58,7 +58,7 @@ teardown() {
   
   assert_output --partial '"status":0'
   
-  curl -X POST -H 'Content-type:application/json' -d '{
+  run curl -X POST -H 'Content-type:application/json' -d '{
     "update-requesthandler": {
       "name": "/query",
       "class": "solr.SearchHandler",
@@ -72,31 +72,11 @@ teardown() {
   run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=3&ubi=true"
   assert_output --partial '"status":0'
   assert_output --partial '"query_id":"1234'
-  
-  
-  # Rich UBI user query tracking enabled query
-  run curl -X POST -H 'Content-type:application/json' -d '{
-    "query" : "ram OR memory",
-    "filter": [
-        "inStock:true"
-    ],
-    "limit":2,
-    "params": {
-      "ubi": "true",
-      "query_id": "xyz890",
-      "user_query": {
-        "query": "RAM memory",
-        "experiment": "supersecret",
-        "page": 1,
-        "filter": "productStatus:available"
-      }            
-    }
-  }' "http://localhost:${SOLR_PORT}/solr/techproducts/query" 
-  assert_output --partial '"query_id":"xyz890"'
+   
   
   # No luck on getting the logs to read. 
-  #run cat "${SOLR_LOGS_DIR}/solr.log"
-  #run tail -n 1 "${SOLR_LOGS_DIR}/ubi_queries.jsonl"
-  #assert_output --partial "inStock:false"
-  #assert_file_contains "${SOLR_LOGS_DIR}/ubi_queries.jsonl" 'eric'
+  assert_file_exist ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl
+  run tail -n 1 "${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl"
+  assert_output --partial '{"id":"49","a_i":"1","b_i":"5"}'
+  assert_file_contains "${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl" '{"id":"49","a_i":"1","b_i":"5"}'
 }

From 9e045e6368e242e77f40ec180aa2760e26a7f17a Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 13 Nov 2024 17:59:44 -0500
Subject: [PATCH 051/103] Auditing this test to try and remember what it is for

---
 .../solr/handler/component/UBIComponentLocalLoggingTest.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index 6670234f181..4c00f3342e7 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -36,6 +36,9 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+/**
+ * This test demonstrates, well, not totally sure.  It may be we could delete this?
+ */
 public class UBIComponentLocalLoggingTest extends SolrCloudTestCase {
 
   private static final String COLLECTION = "collection1";
@@ -49,7 +52,7 @@ public static void setupCluster() throws Exception {
   }
 
   @Test
-  public void testLocalCatStream() throws Exception {
+  public void testLocalLoggingStream() throws Exception {
 
     CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1, 1, 0)
         .process(cluster.getSolrClient());

From 96da29c2af8d5b480171833768cba2f651bb463b Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 13 Nov 2024 18:00:16 -0500
Subject: [PATCH 052/103] Lint

---
 .../java/org/apache/solr/handler/component/LoggingStream.java  | 1 -
 .../java/org/apache/solr/handler/component/UBIComponent.java   | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
index fdeb3dd9d03..59a35324c0f 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
@@ -260,7 +260,6 @@ public void setStreamContext(StreamContext context) {
     this.tupleSource.setStreamContext(context);
   }
 
-
   //    private SolrInputDocument convertTupleTJson(Tuple tuple) {
   //        SolrInputDocument doc = new SolrInputDocument();
   //        for (String field : tuple.getFields().keySet()) {
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index f1910486927..1146d1dd732 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -287,8 +287,7 @@ protected void processIds(
     if (stream != null) {
       List tuples = getTuples(stream);
       log.error("Here are the tuples (" + tuples.size() + "):" + tuples);
-    }
-    else {
+    } else {
       log.error("UBI Query Stream is null, can't log query information.");
     }
   }

From 8d928fa6bea13d17735b5023bea84754e28269d0 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 13 Nov 2024 18:00:43 -0500
Subject: [PATCH 053/103] Simplify naming

---
 ...local-component.xml => solrconfig-ubi.xml} | 25 +++++--------------
 .../handler/component/UBIComponentTest.java   | 24 +++++-------------
 2 files changed, 12 insertions(+), 37 deletions(-)
 rename solr/core/src/test-files/solr/collection1/conf/{solrconfig-ubi-local-component.xml => solrconfig-ubi.xml} (77%)

diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi.xml
similarity index 77%
rename from solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml
rename to solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi.xml
index 59a94af99dd..a2381f4a74c 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi-local-component.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-ubi.xml
@@ -20,8 +20,11 @@
 
   ${tests.luceneMatchVersion:LATEST}
   
-  
+       
+     
+      ubi
+    
+  
 
   
   
 
-  
-  
-    
-      dismax
-    
-    
-      ubi
-    
-  
-  
-  
-    
-      dismax
-    
-  
-
-  
+  
     
       text
     
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
index 49d645a2cf6..33f3ec40ddd 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
@@ -24,7 +24,7 @@ public class UBIComponentTest extends SolrTestCaseJ4 {
 
   @BeforeClass
   public static void beforeTest() throws Exception {
-    initCore("solrconfig-ubi-local-component.xml", "schema12.xml");
+    initCore("solrconfig-ubi.xml", "schema12.xml");
     assertNull(h.validateUpdate(adoc("id", "1", "subject", "aa")));
     assertNull(h.validateUpdate(adoc("id", "two", "subject", "aa")));
     assertNull(h.validateUpdate(adoc("id", "3", "subject", "aa")));
@@ -35,7 +35,7 @@ public static void beforeTest() throws Exception {
   public void testGeneratingAQueryId() {
     assertQ(
         "Make sure we generate a query id",
-        req("qt", "/with_ubi", "q", "aa", "rows", "2", "ubi", "true"),
+        req("q", "aa", "rows", "2", "ubi", "true"),
         "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
   }
 
@@ -43,7 +43,7 @@ public void testGeneratingAQueryId() {
   public void testZeroResultsGeneratesQueryId() {
     assertQ(
         "Make sure we generate a query id even when no results are returned",
-        req("qt", "/with_ubi", "q", "abcdefgxyz", "rows", "0", "ubi", "true"),
+        req("q", "abcdefgxyz", "rows", "0", "ubi", "true"),
         "//*[@numFound='0']",
         "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
   }
@@ -52,7 +52,7 @@ public void testZeroResultsGeneratesQueryId() {
   public void testPassedInQueryIdIsUsed() {
     assertQ(
         "Make sure we reuse a passed in query id",
-        req("qt", "/with_ubi", "q", "aa", "rows", "0", "ubi", "true", "query_id", "123abc"),
+        req("q", "aa", "rows", "0", "ubi", "true", "query_id", "123abc"),
         "//lst[@name='ubi']/str[@name='query_id'][.='123abc']");
   }
 
@@ -60,7 +60,7 @@ public void testPassedInQueryIdIsUsed() {
   public void testGenerateQueryId() {
     assertQ(
         "Make sure we generate a query id if one is not passed in",
-        req("qt", "/with_ubi", "q", "aa", "rows", "0", "ubi", "true"),
+        req("q", "aa", "rows", "0", "ubi", "true"),
         "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
   }
 
@@ -68,8 +68,6 @@ public void testGenerateQueryId() {
   public void testJSONQuerySyntax() throws Exception {
     assertJQ(
         req(
-            "qt",
-            "/with_ubi",
             "json",
             "{\n"
                 + "    'query': 'aa',\n"
@@ -87,8 +85,6 @@ public void testJSONQuerySyntax() throws Exception {
 
     assertJQ(
         req(
-            "qt",
-            "/with_ubi",
             "json",
             "{\n"
                 + "    'query': 'aa',\n"
@@ -110,19 +106,11 @@ public void testJSONQuerySyntax() throws Exception {
         "ubi/query_id=='xjy-42-1rj'");
   }
 
-  @Test
-  public void testTrackingOfUserQuery() {
-    assertQ(
-        "Make sure we generate a query id",
-        req("qt", "/with_ubi", "q", "aa", "rows", "0", "ubi", "true", "user_query", "fresh air"),
-        "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
-  }
-
   @Test
   public void testDisabling() {
     assertQ(
         "Make sure we don't generate a query_id",
-        req("qt", "/with_ubi", "q", "aa", "ubi", "false"),
+        req("q", "aa", "ubi", "false"),
         "count(//lst[@name='ubi'])=0");
   }
 }

From 5e691ae7fc8b49df427e194ca8dd6fbba8338522 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 13 Nov 2024 18:00:51 -0500
Subject: [PATCH 054/103] Lint

---
 .../solr/handler/component/UBIComponentLocalLoggingTest.java  | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index 4c00f3342e7..85a5041ca65 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -36,9 +36,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-/**
- * This test demonstrates, well, not totally sure.  It may be we could delete this?
- */
+/** This test demonstrates, well, not totally sure. It may be we could delete this? */
 public class UBIComponentLocalLoggingTest extends SolrCloudTestCase {
 
   private static final String COLLECTION = "collection1";

From d555b40fdcd95eb0f898cd051c7420c9199be043 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 13 Nov 2024 21:41:04 -0500
Subject: [PATCH 055/103] Now able to stream updates

---
 .../solr/handler/component/LoggingStream.java |   2 +-
 .../solr/handler/component/UBIComponent.java  | 105 +++++++++-------
 .../solr/handler/component/UBIQuery.java      |  39 +++++-
 .../component/UBIQueryTupleStream.java        | 112 ++++++++++++++++++
 solr/packaging/test/test_ubi.bats             |  36 ++++--
 5 files changed, 245 insertions(+), 49 deletions(-)
 create mode 100644 solr/core/src/java/org/apache/solr/handler/component/UBIQueryTupleStream.java

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
index 59a35324c0f..b0b0f014863 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
@@ -138,7 +138,7 @@ public void open() throws IOException {
           SolrException.ErrorCode.BAD_REQUEST, "file to log to must be under " + chroot);
     }
 
-    fos = new FileOutputStream(filePath.toFile());
+    fos = new FileOutputStream(filePath.toFile(), true);
     writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
 
     tupleSource.open();
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 1146d1dd732..73a58f7fe22 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -124,12 +124,15 @@ public class UBIComponent extends SearchComponent implements SolrCoreAware {
   public static final String QUERY_ID = "query_id";
   public static final String QUERY_ATTRIBUTES = "query_attributes";
   public static final String USER_QUERY = "user_query";
+  public static final String APPLICATION = "application";
 
   protected PluginInfo info = PluginInfo.EMPTY_INFO;
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-  private TupleStream stream;
+  private StreamContext streamContext;
+  private StreamExpression streamExpression;
+  private StreamFactory streamFactory;
 
   protected SolrParams initArgs;
 
@@ -157,7 +160,8 @@ public void inform(SolrCore core) {
               "No 'ubiQueryStreamProcessingExpression' file provided to describe processing of UBI query information.");
           log.info(
               "Writing out UBI query information to local $SOLR_HOME/userfiles/ubi_queries.jsonl file instead.");
-          expr = "logging(ubi_queries.jsonl," + "tuple(id=49,a_i=1,b_i=5)" + ")";
+          // expr = "logging(ubi_queries.jsonl," + "tuple(id=49,a_i=1,b_i=5)" + ")";
+          expr = "logging(ubi_queries.jsonl," + "ubiQueryTuple()" + ")";
         } else {
           LineNumberReader bufferedReader;
 
@@ -184,28 +188,29 @@ public void inform(SolrCore core) {
           }
         }
 
-        StreamContext streamContext = new StreamContext();
+        streamContext = new StreamContext();
         streamContext.put("solr-core", core);
         streamContext.setSolrClientCache(solrClientCache);
 
-        StreamExpression streamExpression = StreamExpressionParser.parse(expr);
-        StreamFactory streamFactory = new StreamFactory();
+        streamExpression = StreamExpressionParser.parse(expr);
+        streamFactory = new StreamFactory();
         streamFactory.withFunctionName("logging", LoggingStream.class);
+        streamFactory.withFunctionName("ubiQueryTuple", UBIQueryTupleStream.class);
 
         streamFactory.withDefaultZkHost(defaultZkHost);
 
         Lang.register(streamFactory);
 
-        try {
-          stream = constructStream(streamFactory, streamExpression);
-        } catch (IOException exception) {
-          throw new SolrException(
-              SolrException.ErrorCode.SERVER_ERROR,
-              "Error constructing stream for processing UBI data collection: "
-                  + UBIComponent.class.getSimpleName(),
-              exception);
-        }
-        stream.setStreamContext(streamContext);
+        //        try {
+        //          stream = constructStream(streamFactory, streamExpression);
+        //        } catch (IOException exception) {
+        //          throw new SolrException(
+        //              SolrException.ErrorCode.SERVER_ERROR,
+        //              "Error constructing stream for processing UBI data collection: "
+        //                  + UBIComponent.class.getSimpleName(),
+        //              exception);
+        //        }
+        //        stream.setStreamContext(streamContext);
 
       } else {
         log.info("UBI query data collection is only available in SolrCloud mode.");
@@ -232,6 +237,7 @@ public void process(ResponseBuilder rb) throws IOException {
     UBIQuery ubiQuery = new UBIQuery(params.get(QUERY_ID));
 
     ubiQuery.setUserQuery(params.get(USER_QUERY));
+    ubiQuery.setApplication(params.get(APPLICATION));
 
     Object queryAttributes = params.get(QUERY_ATTRIBUTES);
 
@@ -251,38 +257,36 @@ public void process(ResponseBuilder rb) throws IOException {
     }
 
     ResultContext rc = (ResultContext) rb.rsp.getResponse();
-
     DocList docs = rc.getDocList();
 
-    processIds(rb, docs, ubiQuery, schema, searcher);
-  }
+    String docIds = extractDocIds(docs, schema, searcher);
+    ubiQuery.setDocIds(docIds);
 
-  protected void processIds(
-      ResponseBuilder rb,
-      DocList dl,
-      UBIQuery ubiQuery,
-      IndexSchema schema,
-      SolrIndexSearcher searcher)
-      throws IOException {
-    StringBuilder sb = new StringBuilder();
-
-    Set fields = Collections.singleton(schema.getUniqueKeyField().getName());
-    for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {
-      sb.append(schema.printableUniqueKey(searcher.getDocFetcher().doc(iter.nextDoc(), fields)))
-          .append(',');
-    }
-    String docIds = sb.length() > 0 ? sb.substring(0, sb.length() - 1) : "";
+    addUBIClauseToResponse(ubiQuery, rb);
+    recordUBIData(ubiQuery);
+  }
 
-    SimpleOrderedMap ubiResponseInfo = new SimpleOrderedMap<>();
+  private void recordUBIData(UBIQuery ubiQuery) throws IOException {
     SimpleOrderedMap ubiQueryLogInfo = new SimpleOrderedMap<>();
-    ubiResponseInfo.add(QUERY_ID, ubiQuery.getQueryId());
-    rb.rsp.add("ubi", ubiResponseInfo);
 
     // Maybe ubiQueryLogInfo should be a ubiQuery?  But what about the doc_ids?
-    ubiQueryLogInfo.add(QUERY_ID, ubiQuery.getQueryId());
-    ubiQueryLogInfo.add(USER_QUERY, ubiQuery.getUserQuery());
-    ubiQueryLogInfo.add(QUERY_ATTRIBUTES, ubiQuery.getQueryAttributes());
-    ubiQueryLogInfo.add("doc_ids", docIds);
+    // ubiQueryLogInfo.add(QUERY_ID, ubiQuery.getQueryId());
+    // ubiQueryLogInfo.add(USER_QUERY, ubiQuery.getUserQuery());
+    // ubiQueryLogInfo.add(QUERY_ATTRIBUTES, ubiQuery.getQueryAttributes());
+    // ubiQueryLogInfo.add("doc_ids", ubiQuery.getDocIds());
+
+    TupleStream stream;
+    try {
+      stream = constructStream(streamFactory, streamExpression);
+    } catch (IOException exception) {
+      throw new SolrException(
+          SolrException.ErrorCode.SERVER_ERROR,
+          "Error constructing stream for processing UBI data collection: "
+              + UBIComponent.class.getSimpleName(),
+          exception);
+    }
+    streamContext.put("ubi-query", ubiQuery);
+    stream.setStreamContext(streamContext);
 
     if (stream != null) {
       List tuples = getTuples(stream);
@@ -292,6 +296,27 @@ protected void processIds(
     }
   }
 
+  private void addUBIClauseToResponse(UBIQuery ubiQuery, ResponseBuilder rb) {
+    SimpleOrderedMap ubiResponseInfo = new SimpleOrderedMap<>();
+
+    ubiResponseInfo.add(QUERY_ID, ubiQuery.getQueryId());
+    rb.rsp.add("ubi", ubiResponseInfo);
+  }
+
+  protected String extractDocIds(DocList dl, IndexSchema schema, SolrIndexSearcher searcher)
+      throws IOException {
+    StringBuilder sb = new StringBuilder();
+
+    Set fields = Collections.singleton(schema.getUniqueKeyField().getName());
+    for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {
+      sb.append(schema.printableUniqueKey(searcher.getDocFetcher().doc(iter.nextDoc(), fields)))
+          .append(',');
+    }
+    String docIds = sb.length() > 0 ? sb.substring(0, sb.length() - 1) : "";
+
+    return docIds;
+  }
+
   protected List getTuples(TupleStream tupleStream) throws IOException {
     tupleStream.open();
     List tuples = new ArrayList<>();
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
index 086a1ef22da..b6be2db30d9 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
@@ -16,6 +16,9 @@
  */
 package org.apache.solr.handler.component;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * Handles all the data required for tracking a query using User Behavior Insights.
  *
@@ -24,9 +27,11 @@
  */
 public class UBIQuery {
 
+  private String application;
   private String queryId;
   private String userQuery;
   private Object queryAttributes;
+  private String docIds;
 
   public UBIQuery(String queryId) {
 
@@ -36,6 +41,14 @@ public UBIQuery(String queryId) {
     this.queryId = queryId;
   }
 
+  public void setApplication(String application) {
+    this.application = application;
+  }
+
+  public String getApplication() {
+    return this.application;
+  }
+
   public String getQueryId() {
     return queryId;
   }
@@ -60,6 +73,14 @@ public void setQueryAttributes(Object queryAttributes) {
     this.queryAttributes = queryAttributes;
   }
 
+  public String getDocIds() {
+    return docIds;
+  }
+
+  public void setDocIds(String docIds) {
+    this.docIds = docIds;
+  }
+
   /**
    * Convert the UBIQuery into the format consumed by a streaming expression tuple()
    *
@@ -72,6 +93,22 @@ public String toTuple() {
         + ","
         + UBIComponent.USER_QUERY
         + "="
-        + this.userQuery;
+        + this.userQuery
+        + ","
+        + UBIComponent.APPLICATION
+        + "="
+        + this.application;
+  }
+
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  public Map toMap() {
+    @SuppressWarnings({"rawtypes", "unchecked"})
+    Map map = new HashMap();
+    map.put(UBIComponent.QUERY_ID, this.queryId);
+    map.put(UBIComponent.APPLICATION, this.application);
+    map.put(UBIComponent.USER_QUERY, this.userQuery);
+    // map.put(UBIComponent.QUERY_ATTRIBUTES, this.queryAttributes);
+
+    return map;
   }
 }
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQueryTupleStream.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQueryTupleStream.java
new file mode 100644
index 00000000000..906ddbed331
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQueryTupleStream.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.StreamContext;
+import org.apache.solr.client.solrj.io.stream.TupleStream;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+/**
+ * Converts a UBIQuery that is stored in the StreamContext under the key 'ubi-query' into a Tuple
+ * and returns it.
+ */
+public class UBIQueryTupleStream extends TupleStream implements Expressible {
+
+  private StreamContext streamContext;
+  private boolean finished;
+
+  public UBIQueryTupleStream(StreamExpression expression, StreamFactory factory)
+      throws IOException {}
+
+  @Override
+  public StreamExpression toExpression(StreamFactory factory) throws IOException {
+    return toExpression(factory, true);
+  }
+
+  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams)
+      throws IOException {
+    // function name
+    StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
+
+    return expression;
+  }
+
+  @Override
+  public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setExpressionType(ExpressionType.STREAM_SOURCE);
+    explanation.setExpression(toExpression(factory, false).toString());
+
+    return explanation;
+  }
+
+  @Override
+  public void setStreamContext(StreamContext context) {
+    this.streamContext = context;
+  }
+
+  @Override
+  public List children() {
+    List l = new ArrayList<>();
+    return l;
+  }
+
+  @Override
+  public void open() throws IOException {}
+
+  @Override
+  public void close() throws IOException {}
+
+  @SuppressWarnings({"unchecked"})
+  @Override
+  public Tuple read() throws IOException {
+
+    if (finished) {
+      return Tuple.EOF();
+    } else {
+      finished = true;
+
+      UBIQuery ubiQuery = (UBIQuery) streamContext.get("ubi-query");
+
+      return new Tuple(ubiQuery.toMap());
+    }
+  }
+
+  /** Return the stream sort - ie, the order in which records are returned */
+  @Override
+  public StreamComparator getStreamSort() {
+    return null;
+  }
+
+  @Override
+  public int getCost() {
+    return 0;
+  }
+}
diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats
index 392d09b5615..7ebdb08cb1c 100644
--- a/solr/packaging/test/test_ubi.bats
+++ b/solr/packaging/test/test_ubi.bats
@@ -69,14 +69,36 @@ teardown() {
   assert_output --partial '"status":0'
 
   # Simple ubi enabled query
-  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=3&ubi=true"
+  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=3&ubi=true&user_query=give%20me%20all&query_id=5678"
   assert_output --partial '"status":0'
-  assert_output --partial '"query_id":"1234'
+  assert_output --partial '"query_id":"5678'
    
-  
-  # No luck on getting the logs to read. 
+  # Check UBI query record was written out to default location
   assert_file_exist ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl
-  run tail -n 1 "${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl"
-  assert_output --partial '{"id":"49","a_i":"1","b_i":"5"}'
-  assert_file_contains "${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl" '{"id":"49","a_i":"1","b_i":"5"}'
+  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '{"query_id":"5678","application":null,"user_query":"give me all"}'
+  
+  # Rich UBI user query tracking enabled query
+  run curl -X POST -H 'Content-type:application/json' -d '{
+    "query" : "ram OR memory",
+    "filter": [
+        "inStock:true"
+    ],
+    "limit":2,
+    "params": {
+      "ubi": "true",
+      "application":"primary_search",
+      "query_id": "xyz890",
+      "user_query":"RAM memory",
+      "query_attributes": {
+        "experiment": "supersecret",
+        "page": 1,
+        "filter": "productStatus:available"
+      }            
+    }
+  }' "http://localhost:${SOLR_PORT}/solr/techproducts/query" 
+  assert_output --partial '"query_id":"xyz890"'
+  
+  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '{"query_id":"xyz890","application":"primary_search","user_query":"RAM memory"}'
+  
+  
 }

From e5396698d3a0fd9292f90de89486421dc38d5015 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Thu, 14 Nov 2024 10:57:09 -0500
Subject: [PATCH 056/103] Source UBI events from the component

---
 .../solr/handler/component/UBIComponent.java  | 98 ++++++++++---------
 .../solr/handler/component/UBIQuery.java      | 21 +---
 ...ryTupleStream.java => UBIQueryStream.java} |  9 +-
 .../handler/component/LoggingStreamTest.java  | 23 ++---
 .../UBIComponentLocalLoggingTest.java         |  5 +-
 .../UBIComponentStreamingQueriesTest.java     | 18 +++-
 solr/packaging/test/test_ubi.bats             |  4 +-
 7 files changed, 92 insertions(+), 86 deletions(-)
 rename solr/core/src/java/org/apache/solr/handler/component/{UBIQueryTupleStream.java => UBIQueryStream.java} (89%)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 73a58f7fe22..0388b84c94d 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -26,11 +26,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import org.apache.solr.client.solrj.io.Lang;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.stream.StreamContext;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
+import org.apache.solr.client.solrj.io.stream.expr.DefaultStreamFactory;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
@@ -160,8 +160,15 @@ public void inform(SolrCore core) {
               "No 'ubiQueryStreamProcessingExpression' file provided to describe processing of UBI query information.");
           log.info(
               "Writing out UBI query information to local $SOLR_HOME/userfiles/ubi_queries.jsonl file instead.");
+          // Most simplisitic version
           // expr = "logging(ubi_queries.jsonl," + "tuple(id=49,a_i=1,b_i=5)" + ")";
+
+          // The real version
           expr = "logging(ubi_queries.jsonl," + "ubiQueryTuple()" + ")";
+
+          // feels like stream or something should let me create a tuple out of something in the
+          // context.
+          // expr = "logging(ubi_queries.jsonl," + "get(ubi-core)" + ")";
         } else {
           LineNumberReader bufferedReader;
 
@@ -193,24 +200,20 @@ public void inform(SolrCore core) {
         streamContext.setSolrClientCache(solrClientCache);
 
         streamExpression = StreamExpressionParser.parse(expr);
-        streamFactory = new StreamFactory();
+        if (!streamExpression.toString().contains("ubiQueryTuple")) {
+          log.error(
+              "The streaming expression "
+                  + streamExpression
+                  + " must include the 'ubiQueryTuple()' to record UBI queries.");
+        }
+        System.out.println("streamExpression is " + streamExpression);
+        streamFactory = new DefaultStreamFactory();
         streamFactory.withFunctionName("logging", LoggingStream.class);
-        streamFactory.withFunctionName("ubiQueryTuple", UBIQueryTupleStream.class);
+        streamFactory.withFunctionName("ubiQueryTuple", UBIQueryStream.class);
 
         streamFactory.withDefaultZkHost(defaultZkHost);
 
-        Lang.register(streamFactory);
-
-        //        try {
-        //          stream = constructStream(streamFactory, streamExpression);
-        //        } catch (IOException exception) {
-        //          throw new SolrException(
-        //              SolrException.ErrorCode.SERVER_ERROR,
-        //              "Error constructing stream for processing UBI data collection: "
-        //                  + UBIComponent.class.getSimpleName(),
-        //              exception);
-        //        }
-        //        stream.setStreamContext(streamContext);
+        // Lang.register(streamFactory);
 
       } else {
         log.info("UBI query data collection is only available in SolrCloud mode.");
@@ -262,41 +265,36 @@ public void process(ResponseBuilder rb) throws IOException {
     String docIds = extractDocIds(docs, schema, searcher);
     ubiQuery.setDocIds(docIds);
 
-    addUBIClauseToResponse(ubiQuery, rb);
-    recordUBIData(ubiQuery);
+    addUserBehaviorInsightsToResponse(ubiQuery, rb);
+    recordQuery(ubiQuery);
   }
 
-  private void recordUBIData(UBIQuery ubiQuery) throws IOException {
-    SimpleOrderedMap ubiQueryLogInfo = new SimpleOrderedMap<>();
-
-    // Maybe ubiQueryLogInfo should be a ubiQuery?  But what about the doc_ids?
-    // ubiQueryLogInfo.add(QUERY_ID, ubiQuery.getQueryId());
-    // ubiQueryLogInfo.add(USER_QUERY, ubiQuery.getUserQuery());
-    // ubiQueryLogInfo.add(QUERY_ATTRIBUTES, ubiQuery.getQueryAttributes());
-    // ubiQueryLogInfo.add("doc_ids", ubiQuery.getDocIds());
-
+  private void recordQuery(UBIQuery ubiQuery) throws IOException {
     TupleStream stream;
-    try {
-      stream = constructStream(streamFactory, streamExpression);
-    } catch (IOException exception) {
-      throw new SolrException(
-          SolrException.ErrorCode.SERVER_ERROR,
-          "Error constructing stream for processing UBI data collection: "
-              + UBIComponent.class.getSimpleName(),
-          exception);
-    }
+    // try {
+    stream = constructStream(streamFactory, streamExpression);
+    // } catch (IOException exception) {
+    //   throw new SolrException(
+    //       SolrException.ErrorCode.SERVER_ERROR,
+    //       "Error constructing stream for processing UBI data collection: "
+    //           + UBIComponent.class.getSimpleName(),
+    //      exception);
+    // }
     streamContext.put("ubi-query", ubiQuery);
     stream.setStreamContext(streamContext);
 
-    if (stream != null) {
-      List tuples = getTuples(stream);
-      log.error("Here are the tuples (" + tuples.size() + "):" + tuples);
-    } else {
-      log.error("UBI Query Stream is null, can't log query information.");
-    }
+    // if (stream != null) {
+    // We could just call getTuple since there is only one, it's one per query unless we
+    // have a component level stream that is opened...
+    getTuple(stream);
+    // List tuples = getTuples(stream);
+    // log.error("Here are the tuples (" + tuples.size() + "):" + tuples);
+    // } else {
+    //  log.error("UBI Query Stream is null, can't log query information.");
+    // }
   }
 
-  private void addUBIClauseToResponse(UBIQuery ubiQuery, ResponseBuilder rb) {
+  private void addUserBehaviorInsightsToResponse(UBIQuery ubiQuery, ResponseBuilder rb) {
     SimpleOrderedMap ubiResponseInfo = new SimpleOrderedMap<>();
 
     ubiResponseInfo.add(QUERY_ID, ubiQuery.getQueryId());
@@ -367,7 +365,6 @@ public static String readExpression(LineNumberReader bufferedReader, String[] ar
       }
 
       // Substitute parameters
-
       if (line.length() > 0) {
         for (int i = 1; i < args.length; i++) {
           String arg = args[i];
@@ -381,9 +378,18 @@ public static String readExpression(LineNumberReader bufferedReader, String[] ar
     return exprBuff.toString();
   }
 
-  public static TupleStream constructStream(
+  private static TupleStream constructStream(
       StreamFactory streamFactory, StreamExpression streamExpression) throws IOException {
-    return streamFactory.constructStream(streamExpression);
+    try {
+      return streamFactory.constructStream(streamExpression);
+    } catch (IOException exception) {
+      // Throw or just log an error?
+      throw new SolrException(
+          SolrException.ErrorCode.SERVER_ERROR,
+          "Error constructing stream for processing UBI data collection: "
+              + UBIComponent.class.getSimpleName(),
+          exception);
+    }
   }
 
   /*
@@ -399,6 +405,6 @@ public static Map validateLetAndGetParams(TupleStream stream, String expr) throw
   */
   @Override
   public String getDescription() {
-    return "A component that tracks original user query and the resulting documents returned.";
+    return "A component that tracks the original user query and the resulting documents returned.";
   }
 }
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
index b6be2db30d9..cda0c9818c0 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
@@ -81,25 +81,6 @@ public void setDocIds(String docIds) {
     this.docIds = docIds;
   }
 
-  /**
-   * Convert the UBIQuery into the format consumed by a streaming expression tuple()
-   *
-   * @return String The tuple specific formatted data similar to "query_id=123,user_query=foo"
-   */
-  public String toTuple() {
-    return UBIComponent.QUERY_ID
-        + "="
-        + this.queryId
-        + ","
-        + UBIComponent.USER_QUERY
-        + "="
-        + this.userQuery
-        + ","
-        + UBIComponent.APPLICATION
-        + "="
-        + this.application;
-  }
-
   @SuppressWarnings({"rawtypes", "unchecked"})
   public Map toMap() {
     @SuppressWarnings({"rawtypes", "unchecked"})
@@ -107,7 +88,7 @@ public Map toMap() {
     map.put(UBIComponent.QUERY_ID, this.queryId);
     map.put(UBIComponent.APPLICATION, this.application);
     map.put(UBIComponent.USER_QUERY, this.userQuery);
-    // map.put(UBIComponent.QUERY_ATTRIBUTES, this.queryAttributes);
+    map.put(UBIComponent.QUERY_ATTRIBUTES, this.queryAttributes);
 
     return map;
   }
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQueryTupleStream.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQueryStream.java
similarity index 89%
rename from solr/core/src/java/org/apache/solr/handler/component/UBIQueryTupleStream.java
rename to solr/core/src/java/org/apache/solr/handler/component/UBIQueryStream.java
index 906ddbed331..c36d128ef4f 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQueryTupleStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQueryStream.java
@@ -33,14 +33,17 @@
 /**
  * Converts a UBIQuery that is stored in the StreamContext under the key 'ubi-query' into a Tuple
  * and returns it.
+ *
+ * 
I suspect that if I had the right magic with a LetStream or a GetStream, I could somehow just
+ * use that to say "pluck the 'ubi-query' object out of the StreamContext and call .toTuple or make
+ * a map of it and that would be my tuple'.
  */
-public class UBIQueryTupleStream extends TupleStream implements Expressible {
+public class UBIQueryStream extends TupleStream implements Expressible {
 
   private StreamContext streamContext;
   private boolean finished;
 
-  public UBIQueryTupleStream(StreamExpression expression, StreamFactory factory)
-      throws IOException {}
+  public UBIQueryStream(StreamExpression expression, StreamFactory factory) throws IOException {}
 
   @Override
   public StreamExpression toExpression(StreamFactory factory) throws IOException {
diff --git a/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java b/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
index aafe807e4bf..7ca440a4868 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
@@ -80,36 +80,37 @@ public static void setupCluster() throws Exception {
   }
 
   @Test
-  public void testLogStreamExpressionToExpression() throws Exception {
+  public void testLoggingStreamExpressionToExpression() throws Exception {
     String expressionString;
 
     // Basic test
     try (LoggingStream stream =
         new LoggingStream(
-            StreamExpressionParser.parse("logging(bob.txt,echo(\"bob\"))"), factory)) {
+            StreamExpressionParser.parse("logging(bob.jsonl,echo(\"bob\"))"), factory)) {
       expressionString = stream.toExpression(factory).toString();
-      assertTrue(expressionString.contains("logging(bob.txt,"));
+      assertTrue(expressionString.contains("logging(bob.jsonl,"));
       assertTrue(expressionString.contains("echo(\"bob"));
     }
 
     // Unwrap double quotes around file name test
     try (LoggingStream stream =
         new LoggingStream(
-            StreamExpressionParser.parse("logging(\"outputs/bob.txt\",echo(\"bob\"))"), factory)) {
+            StreamExpressionParser.parse("logging(\"outputs/bob.jsonl\",echo(\"bob\"))"),
+            factory)) {
       expressionString = stream.toExpression(factory).toString();
-      assertTrue(expressionString.contains("logging(outputs/bob.txt,"));
+      assertTrue(expressionString.contains("logging(outputs/bob.jsonl,"));
       assertTrue(expressionString.contains("echo(\"bob"));
     }
   }
 
   @Test
-  public void testLogStreamExpressionToExplanation() throws Exception {
+  public void testLoggingStreamExpressionToExplanation() throws Exception {
 
     try (LoggingStream stream =
         new LoggingStream(
-            StreamExpressionParser.parse("logging(bob.txt,echo(\"bob\"))"), factory)) {
+            StreamExpressionParser.parse("logging(bob.jsonl,echo(\"bob\"))"), factory)) {
       Explanation explanation = stream.toExplanation(factory);
-      assertEquals("logging (bob.txt)", explanation.getFunctionName());
+      assertEquals("logging (bob.jsonl)", explanation.getFunctionName());
       assertEquals(LoggingStream.class.getName(), explanation.getImplementingClass());
     }
   }
@@ -119,7 +120,7 @@ public void testFileOutputDirectoryPermissions() throws Exception {
 
     LoggingStream stream =
         new LoggingStream(
-            StreamExpressionParser.parse("logging(/tmp/bob.txt,echo(\"bob\"))"), factory);
+            StreamExpressionParser.parse("logging(/tmp/bob.jsonl,echo(\"bob\"))"), factory);
     stream.setStreamContext(context);
 
     LoggingStream finalStream1 = stream;
@@ -132,7 +133,7 @@ public void testFileOutputDirectoryPermissions() throws Exception {
 
     stream =
         new LoggingStream(
-            StreamExpressionParser.parse("logging(../bob.txt,echo(\"bob\"))"), factory);
+            StreamExpressionParser.parse("logging(../bob.jsonl,echo(\"bob\"))"), factory);
     stream.setStreamContext(context);
 
     LoggingStream finalStream2 = stream;
@@ -145,7 +146,7 @@ public void testFileOutputDirectoryPermissions() throws Exception {
   }
 
   @Test
-  public void testLoggingStreamCombinedWithCatAndJsonStream() throws Exception {
+  public void testLoggingStreamCombinedSourcedFromCSV() throws Exception {
     String expr =
         "logging(parsed_csv_output.jsonl,"
             + "parseCSV(list(tuple(file=\"file1\", line=\"a,b,c\"), "
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index 85a5041ca65..12ba495295e 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -85,13 +85,12 @@ public void testLocalLoggingStream() throws Exception {
 
     StreamFactory streamFactory = new StreamFactory();
 
-    // LocalCatStream extends CatStream and disables the Solr cluster specific
-    // logic about where to read data from.
+    // LoggingStream lets us write out tuples in jsonl format to a file in userfiles space.
     streamFactory.withFunctionName("logging", LoggingStream.class);
 
     Lang.register(streamFactory);
 
-    String clause = "logging(bob.txt,echo(\"bob\"))";
+    String clause = "logging(bob.jsonl,echo(\"bob\"))";
     stream = streamFactory.constructStream(clause);
     stream.setStreamContext(streamContext);
     tuples = getTuples(stream);
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index 1696c5c1f17..50bee76c7f8 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -50,6 +50,8 @@
 /**
  * Tests the ability for {@link UBIComponent} to stream the gathered query data to another Solr
  * index using Streaming Expressions.
+ *
+ * 
This guy needs simplification!!!!!!!!! Needs to look more like some of the tests Joel wrote.
  */
 public class UBIComponentStreamingQueriesTest extends SolrCloudTestCase {
   public static final String COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION =
@@ -242,10 +244,24 @@ private static String getClause(UBIQuery ubiQuery) {
         + ",update("
         + UBI_QUERIES_COLLECTION
         + ",tuple(id=4.0,"
-        + ubiQuery.toTuple()
+        + toTuple(ubiQuery)
         + ")))";
   }
 
+  public static String toTuple(UBIQuery ubiQuery) {
+    return UBIComponent.QUERY_ID
+        + "="
+        + ubiQuery.getQueryId()
+        + ","
+        + UBIComponent.USER_QUERY
+        + "="
+        + ubiQuery.getUserQuery()
+        + ","
+        + UBIComponent.APPLICATION
+        + "="
+        + ubiQuery.getApplication();
+  }
+
   @SuppressWarnings({"rawtypes", "unchecked"})
   public void testRandomDocs() throws Exception {
 
diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats
index 7ebdb08cb1c..5078a762f1f 100644
--- a/solr/packaging/test/test_ubi.bats
+++ b/solr/packaging/test/test_ubi.bats
@@ -75,7 +75,7 @@ teardown() {
    
   # Check UBI query record was written out to default location
   assert_file_exist ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl
-  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '{"query_id":"5678","application":null,"user_query":"give me all"}'
+  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"query_id":"5678"'
   
   # Rich UBI user query tracking enabled query
   run curl -X POST -H 'Content-type:application/json' -d '{
@@ -98,7 +98,7 @@ teardown() {
   }' "http://localhost:${SOLR_PORT}/solr/techproducts/query" 
   assert_output --partial '"query_id":"xyz890"'
   
-  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '{"query_id":"xyz890","application":"primary_search","user_query":"RAM memory"}'
+  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"query_id":"xyz890"'
   
   
 }

From fa209fe1b35a911f54313381525807b3c3b785de Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Thu, 14 Nov 2024 11:51:20 -0500
Subject: [PATCH 057/103] Be clearer

---
 .../solr/handler/component/UBIComponent.java  | 16 ++++++--------
 solr/packaging/test/test_ubi.bats             | 21 +++++--------------
 2 files changed, 11 insertions(+), 26 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 0388b84c94d..2da21fbc3c7 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -160,15 +160,15 @@ public void inform(SolrCore core) {
               "No 'ubiQueryStreamProcessingExpression' file provided to describe processing of UBI query information.");
           log.info(
               "Writing out UBI query information to local $SOLR_HOME/userfiles/ubi_queries.jsonl file instead.");
-          // Most simplisitic version
-          // expr = "logging(ubi_queries.jsonl," + "tuple(id=49,a_i=1,b_i=5)" + ")";
+          // Most simplistic version
+          // expr = "logging(ubi_queries.jsonl, tuple(query_id=49,user_query=\"RAM memory\"))";
 
           // The real version
           expr = "logging(ubi_queries.jsonl," + "ubiQueryTuple()" + ")";
 
-          // feels like stream or something should let me create a tuple out of something in the
-          // context.
-          // expr = "logging(ubi_queries.jsonl," + "get(ubi-core)" + ")";
+          // feels like 'stream' or 'get' or something should let me create a tuple out of something in the
+          // streamContext.   That would turn the "ubi-query" object in the context into a nice tuple and return it.
+          // expr = "logging(ubi_queries.jsonl," + "get(ubi-query)" + ")";
         } else {
           LineNumberReader bufferedReader;
 
@@ -184,8 +184,6 @@ public void inform(SolrCore core) {
 
             bufferedReader.close();
 
-            // not sure if I need this?  Except maybe, we assume let?
-            // Map params = validateLetAndGetParams(stream, expr);
 
           } catch (IOException ioe) {
             throw new SolrException(
@@ -206,15 +204,13 @@ public void inform(SolrCore core) {
                   + streamExpression
                   + " must include the 'ubiQueryTuple()' to record UBI queries.");
         }
-        System.out.println("streamExpression is " + streamExpression);
+
         streamFactory = new DefaultStreamFactory();
         streamFactory.withFunctionName("logging", LoggingStream.class);
         streamFactory.withFunctionName("ubiQueryTuple", UBIQueryStream.class);
 
         streamFactory.withDefaultZkHost(defaultZkHost);
 
-        // Lang.register(streamFactory);
-
       } else {
         log.info("UBI query data collection is only available in SolrCloud mode.");
       }
diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats
index 5078a762f1f..97884a095cd 100644
--- a/solr/packaging/test/test_ubi.bats
+++ b/solr/packaging/test/test_ubi.bats
@@ -29,15 +29,12 @@ teardown() {
   SOLR_STOP_WAIT=1 solr stop --all >/dev/null 2>&1
 }
 
-@test "Run set up process" {
+@test "Track query using UBI with log file." {
   solr start -e techproducts
   
   run solr healthcheck -c techproducts
   refute_output --partial 'error'
 
-  # No luck with this
-  # assert [ -e ${SOLR_LOGS_DIR}/solr_ubi_queries.log ]
-
   run curl -X POST -H 'Content-type:application/json' -d '{
     "add-searchcomponent": {
       "name": "ubi",
@@ -56,19 +53,9 @@ teardown() {
     }
   }' "http://localhost:${SOLR_PORT}/api/collections/techproducts/config"
   
-  assert_output --partial '"status":0'
-  
-  run curl -X POST -H 'Content-type:application/json' -d '{
-    "update-requesthandler": {
-      "name": "/query",
-      "class": "solr.SearchHandler",
-      "last-components":["ubi"]
-    }
-  }' "http://localhost:${SOLR_PORT}/api/collections/techproducts/config"  
-  
   assert_output --partial '"status":0'
 
-  # Simple ubi enabled query
+  # Simple UBI enabled query
   run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=3&ubi=true&user_query=give%20me%20all&query_id=5678"
   assert_output --partial '"status":0'
   assert_output --partial '"query_id":"5678'
@@ -77,7 +64,7 @@ teardown() {
   assert_file_exist ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl
   assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"query_id":"5678"'
   
-  # Rich UBI user query tracking enabled query
+  # Rich UBI user query tracking enabled query with JSON Query
   run curl -X POST -H 'Content-type:application/json' -d '{
     "query" : "ram OR memory",
     "filter": [
@@ -98,7 +85,9 @@ teardown() {
   }' "http://localhost:${SOLR_PORT}/solr/techproducts/query" 
   assert_output --partial '"query_id":"xyz890"'
   
+  # Check UBI query record was written out to default location
   assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"query_id":"xyz890"'
+  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"experiment": "supersecret"'
   
   
 }

From a758d009fffe722d168531adc703eed6682abd65 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 16 Nov 2024 08:07:02 -0500
Subject: [PATCH 058/103] remove comment

---
 .../java/org/apache/solr/handler/component/LoggingStream.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
index b0b0f014863..f63283f5c13 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
@@ -77,7 +77,6 @@ public class LoggingStream extends TupleStream implements Expressible {
 
   private int batchNumber;
   private long totalDocsIndex;
-  // private PushBackStream tupleSource;
   private TupleStream tupleSource;
   private List documentBatch = new ArrayList<>();
 

From ad6cb4edd838e183f9587cda23a282ee555d3b25 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 16 Nov 2024 08:47:17 -0500
Subject: [PATCH 059/103] Lint

---
 .../org/apache/solr/handler/component/UBIComponent.java    | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 2da21fbc3c7..3aa11e9a678 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -166,8 +166,10 @@ public void inform(SolrCore core) {
           // The real version
           expr = "logging(ubi_queries.jsonl," + "ubiQueryTuple()" + ")";
 
-          // feels like 'stream' or 'get' or something should let me create a tuple out of something in the
-          // streamContext.   That would turn the "ubi-query" object in the context into a nice tuple and return it.
+          // feels like 'stream' or 'get' or something should let me create a tuple out of something
+          // in the
+          // streamContext.   That would turn the "ubi-query" object in the context into a nice
+          // tuple and return it.
           // expr = "logging(ubi_queries.jsonl," + "get(ubi-query)" + ")";
         } else {
           LineNumberReader bufferedReader;
@@ -184,7 +186,6 @@ public void inform(SolrCore core) {
 
             bufferedReader.close();
 
-
           } catch (IOException ioe) {
             throw new SolrException(
                 SolrException.ErrorCode.SERVER_ERROR,

From f89854a9c814a26be0c12df1544ed979e903259d Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 16 Nov 2024 10:47:27 -0500
Subject: [PATCH 060/103] Enable in the UI

---
 solr/webapp/web/partials/query.html | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/solr/webapp/web/partials/query.html b/solr/webapp/web/partials/query.html
index 48e618643a7..d212d85bdd9 100644
--- a/solr/webapp/web/partials/query.html
+++ b/solr/webapp/web/partials/query.html
@@ -548,6 +548,15 @@
         
        
       
+      

+        
+          
+        
+        

+      

       

From d2edf13907819037d900f16a57711bdb0dca6769 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 16 Nov 2024 10:59:32 -0500
Subject: [PATCH 061/103] Remove extra layers of nesting that are not required.

UBI can work in non cloud mode, probably just writing locally.
---
 .../solr/handler/component/UBIComponent.java  | 126 +++++++++---------
 1 file changed, 60 insertions(+), 66 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 3aa11e9a678..5f977e5f429 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -116,7 +116,7 @@
  *
  * Notice that we are enabling UBI query tracking, we are providing an explicit query_id and passing
  * in the user's specific choices for querying. The user_query parameters are not specific to Solr
- * syntax, they are defined by the front end user interface.
+ * syntax, they are defined by the creator of the search request.
  */
 public class UBIComponent extends SearchComponent implements SolrCoreAware {
 
@@ -143,78 +143,72 @@ public void init(NamedList args) {
 
   @Override
   public void inform(SolrCore core) {
+    log.info("Initializing UBIComponent");
+
     CoreContainer coreContainer = core.getCoreContainer();
     SolrClientCache solrClientCache = coreContainer.getSolrClientCache();
 
-    if (initArgs != null) {
-      log.info("Initializing UBIComponent");
-      if (coreContainer.isZooKeeperAware()) {
-        String defaultZkHost = core.getCoreContainer().getZkController().getZkServerAddress();
-        String ubiQueryStreamProcessingExpression =
-            initArgs.get("ubiQueryStreamProcessingExpression");
-
-        String expr;
-
-        if (ubiQueryStreamProcessingExpression == null) {
-          log.info(
-              "No 'ubiQueryStreamProcessingExpression' file provided to describe processing of UBI query information.");
-          log.info(
-              "Writing out UBI query information to local $SOLR_HOME/userfiles/ubi_queries.jsonl file instead.");
-          // Most simplistic version
-          // expr = "logging(ubi_queries.jsonl, tuple(query_id=49,user_query=\"RAM memory\"))";
-
-          // The real version
-          expr = "logging(ubi_queries.jsonl," + "ubiQueryTuple()" + ")";
-
-          // feels like 'stream' or 'get' or something should let me create a tuple out of something
-          // in the
-          // streamContext.   That would turn the "ubi-query" object in the context into a nice
-          // tuple and return it.
-          // expr = "logging(ubi_queries.jsonl," + "get(ubi-query)" + ")";
-        } else {
-          LineNumberReader bufferedReader;
-
-          try {
-            bufferedReader =
-                new LineNumberReader(
-                    new InputStreamReader(
-                        core.getResourceLoader().openResource(ubiQueryStreamProcessingExpression),
-                        StandardCharsets.UTF_8));
-
-            String[] args = {}; // maybe we have variables?
-            expr = readExpression(bufferedReader, args);
-
-            bufferedReader.close();
-
-          } catch (IOException ioe) {
-            throw new SolrException(
-                SolrException.ErrorCode.SERVER_ERROR,
-                "Error reading file " + ubiQueryStreamProcessingExpression,
-                ioe);
-          }
-        }
+    String expr;
+    String ubiQueryStreamProcessingExpression = initArgs.get("ubiQueryStreamProcessingExpression");
+
+    if (ubiQueryStreamProcessingExpression == null) {
+      log.info(
+          "No 'ubiQueryStreamProcessingExpression' file provided to describe processing of UBI query information.");
+      log.info(
+          "Writing out UBI query information to local $SOLR_HOME/userfiles/ubi_queries.jsonl file instead.");
+      // Most simplistic version
+      // expr = "logging(ubi_queries.jsonl, tuple(query_id=49,user_query=\"RAM memory\"))";
+
+      // The real version
+      expr = "logging(ubi_queries.jsonl," + "ubiQueryTuple()" + ")";
+
+      // feels like 'stream' or 'get' or something should let me create a tuple out of something
+      // in the
+      // streamContext.   That would turn the "ubi-query" object in the context into a nice
+      // tuple and return it.
+      // expr = "logging(ubi_queries.jsonl," + "get(ubi-query)" + ")";
+    } else {
+      LineNumberReader bufferedReader;
+
+      try {
+        bufferedReader =
+            new LineNumberReader(
+                new InputStreamReader(
+                    core.getResourceLoader().openResource(ubiQueryStreamProcessingExpression),
+                    StandardCharsets.UTF_8));
+
+        String[] args = {}; // maybe we have variables?
+        expr = readExpression(bufferedReader, args);
+
+        bufferedReader.close();
+
+      } catch (IOException ioe) {
+        throw new SolrException(
+            SolrException.ErrorCode.SERVER_ERROR,
+            "Error reading file " + ubiQueryStreamProcessingExpression,
+            ioe);
+      }
+    }
 
-        streamContext = new StreamContext();
-        streamContext.put("solr-core", core);
-        streamContext.setSolrClientCache(solrClientCache);
+    streamContext = new StreamContext();
+    streamContext.put("solr-core", core);
+    streamContext.setSolrClientCache(solrClientCache);
 
-        streamExpression = StreamExpressionParser.parse(expr);
-        if (!streamExpression.toString().contains("ubiQueryTuple")) {
-          log.error(
-              "The streaming expression "
-                  + streamExpression
-                  + " must include the 'ubiQueryTuple()' to record UBI queries.");
-        }
-
-        streamFactory = new DefaultStreamFactory();
-        streamFactory.withFunctionName("logging", LoggingStream.class);
-        streamFactory.withFunctionName("ubiQueryTuple", UBIQueryStream.class);
+    streamExpression = StreamExpressionParser.parse(expr);
+    if (!streamExpression.toString().contains("ubiQueryTuple")) {
+      log.error(
+          "The streaming expression "
+              + streamExpression
+              + " must include the 'ubiQueryTuple()' to record UBI queries.");
+    }
 
-        streamFactory.withDefaultZkHost(defaultZkHost);
+    streamFactory = new DefaultStreamFactory();
+    streamFactory.withFunctionName("logging", LoggingStream.class);
+    streamFactory.withFunctionName("ubiQueryTuple", UBIQueryStream.class);
 
-      } else {
-        log.info("UBI query data collection is only available in SolrCloud mode.");
-      }
+    if (coreContainer.isZooKeeperAware()) {
+      String defaultZkHost = core.getCoreContainer().getZkController().getZkServerAddress();
+      streamFactory.withDefaultZkHost(defaultZkHost);
     }
   }
 

From 5bf1141f313573d7cc4c19989999916c43a9e9a9 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 16 Nov 2024 10:59:54 -0500
Subject: [PATCH 062/103] Fix up test

We don't need both query and select for bats testing.
---
 solr/packaging/test/test_ubi.bats | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats
index 97884a095cd..4e501dcabb5 100644
--- a/solr/packaging/test/test_ubi.bats
+++ b/solr/packaging/test/test_ubi.bats
@@ -82,12 +82,12 @@ teardown() {
         "filter": "productStatus:available"
       }            
     }
-  }' "http://localhost:${SOLR_PORT}/solr/techproducts/query" 
+  }' "http://localhost:${SOLR_PORT}/solr/techproducts/select" 
   assert_output --partial '"query_id":"xyz890"'
   
-  # Check UBI query record was written out to default location
+  # Check UBI query record was written out to default location with additional metadata
   assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"query_id":"xyz890"'
-  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"experiment": "supersecret"'
+  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"experiment":"supersecret"'
   
   
 }

From d4f21cd09824d927a9d9a1018d28561842e5f81a Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 16 Nov 2024 12:58:00 -0500
Subject: [PATCH 063/103] Bit o clean up

---
 .../apache/solr/handler/component/UBIComponent.java | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 5f977e5f429..b17c82748a0 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -228,7 +228,8 @@ public void process(ResponseBuilder rb) throws IOException {
       return;
     }
 
-    UBIQuery ubiQuery = new UBIQuery(params.get(QUERY_ID));
+    String queryId = params.get(QUERY_ID);
+    UBIQuery ubiQuery = new UBIQuery(queryId);
 
     ubiQuery.setUserQuery(params.get(USER_QUERY));
     ubiQuery.setApplication(params.get(APPLICATION));
@@ -262,15 +263,9 @@ public void process(ResponseBuilder rb) throws IOException {
 
   private void recordQuery(UBIQuery ubiQuery) throws IOException {
     TupleStream stream;
-    // try {
+
     stream = constructStream(streamFactory, streamExpression);
-    // } catch (IOException exception) {
-    //   throw new SolrException(
-    //       SolrException.ErrorCode.SERVER_ERROR,
-    //       "Error constructing stream for processing UBI data collection: "
-    //           + UBIComponent.class.getSimpleName(),
-    //      exception);
-    // }
+
     streamContext.put("ubi-query", ubiQuery);
     stream.setStreamContext(streamContext);
 

From 1d949138db9236cf310232263dc20b3c15170c5c Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sat, 16 Nov 2024 12:58:55 -0500
Subject: [PATCH 064/103] Generate actual query_id as UUID

---
 .../solr/handler/component/UBIQuery.java      |  4 +-
 .../handler/component/UBIComponentTest.java   | 44 ++++++++++---------
 2 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
index cda0c9818c0..fc6a3b9e1f0 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
@@ -17,7 +17,9 @@
 package org.apache.solr.handler.component;
 
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
+import java.util.UUID;
 
 /**
  * Handles all the data required for tracking a query using User Behavior Insights.
@@ -36,7 +38,7 @@ public class UBIQuery {
   public UBIQuery(String queryId) {
 
     if (queryId == null) {
-      queryId = "1234";
+      queryId = UUID.randomUUID().toString().toLowerCase(Locale.ROOT);
     }
     this.queryId = queryId;
   }
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
index 33f3ec40ddd..7a20bf3d6ab 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
@@ -36,7 +36,7 @@ public void testGeneratingAQueryId() {
     assertQ(
         "Make sure we generate a query id",
         req("q", "aa", "rows", "2", "ubi", "true"),
-        "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
+        "count(//lst[@name='ubi']/str[@name='query_id'])=1");
   }
 
   @Test
@@ -45,7 +45,7 @@ public void testZeroResultsGeneratesQueryId() {
         "Make sure we generate a query id even when no results are returned",
         req("q", "abcdefgxyz", "rows", "0", "ubi", "true"),
         "//*[@numFound='0']",
-        "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
+        "count(//lst[@name='ubi']/str[@name='query_id'])=1");
   }
 
   @Test
@@ -57,32 +57,34 @@ public void testPassedInQueryIdIsUsed() {
   }
 
   @Test
-  public void testGenerateQueryId() {
+  public void testGenerateQueryIdZeroRowsRequested() {
     assertQ(
         "Make sure we generate a query id if one is not passed in",
         req("q", "aa", "rows", "0", "ubi", "true"),
-        "//lst[@name='ubi']/str[@name='query_id'][.='1234']");
+        "count(//lst[@name='ubi']/str[@name='query_id'])=1");
   }
 
   @Test
-  public void testJSONQuerySyntax() throws Exception {
-    assertJQ(
-        req(
-            "json",
-            "{\n"
-                + "    'query': 'aa',\n"
-                + "    'fields': '*',\n"
-                + "    'offset': 0,\n"
-                + "    'limit': 2,\n"
-                + "    'params': {\n"
-                + "    'df': 'subject',\n"
-                + "    'qt': '/with_ubi',\n"
-                + "    'ubi': 'true'\n"
-                + "   }\n"
-                + "}"),
-        "response/numFound==3",
-        "ubi/query_id=='1234'");
+  public void testJSONQuerySyntaWithJustUBI() throws Exception {
+    String response =
+        JQ(
+            req(
+                "json",
+                "{\n"
+                    + "    'query': 'aa',\n"
+                    + "    'fields': '*',\n"
+                    + "    'offset': 0,\n"
+                    + "    'limit': 2,\n"
+                    + "    'params': {\n"
+                    + "        'df': 'subject',\n"
+                    + "        'ubi': 'true'\n"
+                    + "   }\n"
+                    + "}"));
+    assertTrue(response.indexOf("query_id") != -1);
+  }
 
+  @Test
+  public void testJSONQuerySyntaxWithNestedUBI() throws Exception {
     assertJQ(
         req(
             "json",

From 1c839bac81958e4785c16bbae7bf5ece5244d028 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sun, 17 Nov 2024 09:21:06 -0500
Subject: [PATCH 065/103] Relocate generically useful Streaming expressions

Other streaming expressions live in the .handler package
---
 .../org/apache/solr/handler/{component => }/LoggingStream.java | 3 +--
 .../java/org/apache/solr/handler/SolrDefaultStreamFactory.java | 1 +
 .../java/org/apache/solr/handler/component/UBIComponent.java   | 1 +
 .../apache/solr/handler/{component => }/LoggingStreamTest.java | 3 ++-
 .../solr/handler/component/UBIComponentLocalLoggingTest.java   | 1 +
 5 files changed, 6 insertions(+), 3 deletions(-)
 rename solr/core/src/java/org/apache/solr/handler/{component => }/LoggingStream.java (99%)
 rename solr/core/src/test/org/apache/solr/handler/{component => }/LoggingStreamTest.java (99%)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java b/solr/core/src/java/org/apache/solr/handler/LoggingStream.java
similarity index 99%
rename from solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
rename to solr/core/src/java/org/apache/solr/handler/LoggingStream.java
index f63283f5c13..daa40ff95f7 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/LoggingStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/LoggingStream.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.handler.component;
+package org.apache.solr.handler;
 
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -57,7 +57,6 @@
  *
  * 
WriterStream? LoggingStream? FileoutputStream? JsonOutputStream? LoggingStream??
  *
- * @since 9.8.0
  */
 public class LoggingStream extends TupleStream implements Expressible {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java b/solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java
index d46962189cc..01a8e0bfd63 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java
@@ -36,6 +36,7 @@ public SolrDefaultStreamFactory() {
     super();
     this.withFunctionName("analyze", AnalyzeEvaluator.class);
     this.withFunctionName("cat", CatStream.class);
+    this.withFunctionName("logging", LoggingStream.class);
     this.withFunctionName("classify", ClassifyStream.class);
     this.withFunctionName("haversineMeters", HaversineMetersEvaluator.class);
     this.withFunctionName("wsum", WeightedSumMetric.class);
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index b17c82748a0..93bc224a3fb 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -41,6 +41,7 @@
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.LoggingStream;
 import org.apache.solr.response.ResultContext;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.search.DocIterator;
diff --git a/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java b/solr/core/src/test/org/apache/solr/handler/LoggingStreamTest.java
similarity index 99%
rename from solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
rename to solr/core/src/test/org/apache/solr/handler/LoggingStreamTest.java
index 7ca440a4868..083e2aa5781 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/LoggingStreamTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/LoggingStreamTest.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.handler.component;
+package org.apache.solr.handler;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -39,6 +39,7 @@
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.handler.LoggingStream;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
index 12ba495295e..d2bae3e5363 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
@@ -33,6 +33,7 @@
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.handler.LoggingStream;
 import org.junit.BeforeClass;
 import org.junit.Test;
 

From 5bfec8cd60c6a5e0f59c4996b00f41c1e211cfc2 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sun, 17 Nov 2024 09:21:13 -0500
Subject: [PATCH 066/103] Javadoc

---
 .../test/org/apache/solr/handler/component/UBIComponentTest.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
index 7a20bf3d6ab..0fbdb35033e 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
@@ -20,6 +20,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+/** Tests that the UBI Component augments the response properly **/
 public class UBIComponentTest extends SolrTestCaseJ4 {
 
   @BeforeClass

From 7ab922ceb900d8063dcd97c1eca7b624dedaaabf Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sun, 17 Nov 2024 09:26:43 -0500
Subject: [PATCH 067/103] Lint

---
 solr/core/src/java/org/apache/solr/handler/LoggingStream.java   | 1 -
 .../src/java/org/apache/solr/handler/component/UBIQuery.java    | 2 +-
 .../src/test/org/apache/solr/handler/LoggingStreamTest.java     | 1 -
 .../org/apache/solr/handler/component/UBIComponentTest.java     | 2 +-
 4 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/LoggingStream.java b/solr/core/src/java/org/apache/solr/handler/LoggingStream.java
index daa40ff95f7..e101ab68d9f 100644
--- a/solr/core/src/java/org/apache/solr/handler/LoggingStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/LoggingStream.java
@@ -56,7 +56,6 @@
  * 
Is this generically useful to be added to the streaming jar and Lang?
  *
  * 
WriterStream? LoggingStream? FileoutputStream? JsonOutputStream? LoggingStream??
- *
  */
 public class LoggingStream extends TupleStream implements Expressible {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
index fc6a3b9e1f0..2e2f038a630 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
@@ -25,7 +25,7 @@
  * Handles all the data required for tracking a query using User Behavior Insights.
  *
  * 
Compatible with the
- * https://github.com/o19s/ubi/blob/main/schema/1.0.0/query.request.schema.json.
+ * https://github.com/o19s/ubi/blob/main/schema/1.2.0/query.request.schema.json.
  */
 public class UBIQuery {
 
diff --git a/solr/core/src/test/org/apache/solr/handler/LoggingStreamTest.java b/solr/core/src/test/org/apache/solr/handler/LoggingStreamTest.java
index 083e2aa5781..fc3e2d1e5c2 100644
--- a/solr/core/src/test/org/apache/solr/handler/LoggingStreamTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/LoggingStreamTest.java
@@ -39,7 +39,6 @@
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.handler.LoggingStream;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
index 0fbdb35033e..ee02ad35753 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
@@ -20,7 +20,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-/** Tests that the UBI Component augments the response properly **/
+/** Tests that the UBI Component augments the response properly */
 public class UBIComponentTest extends SolrTestCaseJ4 {
 
   @BeforeClass

From 65f4a8387eee6b0892a1f689143acfb2f07873b1 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sun, 17 Nov 2024 09:41:33 -0500
Subject: [PATCH 068/103] Add some docs

---
 .../solr/handler/component/UBIComponent.java  | 25 ++++++++-----------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 93bc224a3fb..7e1affda886 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -55,16 +55,21 @@
  * User Behavior Insights (UBI) is an open standard for gathering query and event data from users
  * and storing it in a structured format. UBI can be used for in session personalization, implicit
  * judgements, powering recommendation systems among others. Learn more about the UBI standard at https://github.com/o19s/ubi.
+ * href="https://ubisearch.dev">https://ubisearch.dev.
  *
  * 
The response from Solr is augmented by this component, and optionally the query details can be
  * tracked and logged to various systems including log files or other backend systems.
  *
- * 
Data tracked is the collection name, the end user query, as a JSON blob, and the resulting
- * document id's.
+ * 
Data tracked is a unique query_id for the search request, the end user's query, metadata about
+ * the query as a JSON map, and the resulting document id's.
  *
  * 
You provide a streaming expression that is parsed and loaded by the component to stream query
- * data to a target of your choice.
+ * data to a target of your choice. If you do not, then the default expression of
+ * 'logging(ubi_queries.jsonl,ubiQueryTuple())"' is used which logs data to
+ * $SOLR_HOME/userfiles/ubi_queries.jsonl file.
+ *
+ * 
You must source your streaming events using the 'ubiQueryTuple()' streaming expression to
+ * retrieve the {@link UBIQuery} object that contains the data for recording.
  *
  * 
Event data is tracked by letting the user write events directly to the event repository of
  * your choice, it could be a Solr collection, it could be a file or S3 bucket, and that is NOT
@@ -160,8 +165,8 @@ public void inform(SolrCore core) {
       // Most simplistic version
       // expr = "logging(ubi_queries.jsonl, tuple(query_id=49,user_query=\"RAM memory\"))";
 
-      // The real version
-      expr = "logging(ubi_queries.jsonl," + "ubiQueryTuple()" + ")";
+      // The default version
+      expr = "logging(ubi_queries.jsonl,ubiQueryTuple())";
 
       // feels like 'stream' or 'get' or something should let me create a tuple out of something
       // in the
@@ -270,15 +275,7 @@ private void recordQuery(UBIQuery ubiQuery) throws IOException {
     streamContext.put("ubi-query", ubiQuery);
     stream.setStreamContext(streamContext);
 
-    // if (stream != null) {
-    // We could just call getTuple since there is only one, it's one per query unless we
-    // have a component level stream that is opened...
     getTuple(stream);
-    // List tuples = getTuples(stream);
-    // log.error("Here are the tuples (" + tuples.size() + "):" + tuples);
-    // } else {
-    //  log.error("UBI Query Stream is null, can't log query information.");
-    // }
   }
 
   private void addUserBehaviorInsightsToResponse(UBIQuery ubiQuery, ResponseBuilder rb) {

From 249b45b54259b98f91c30699f65c9e9dfea60c2c Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sun, 17 Nov 2024 13:22:11 -0500
Subject: [PATCH 069/103] Able to pump query events into a Solr collection

The ubi-query-pipeline.expr demonstrates writing events to a Solr collection.
---
 .../solr/handler/component/UBIComponent.java  |  18 +--
 .../solr/handler/component/UBIQuery.java      |  12 +-
 .../ubi-enabled/conf/solrconfig.xml           |  12 --
 .../ubi-enabled/conf/ubi-query-pipeline.expr  |   7 +-
 .../UBIComponentLocalLoggingTest.java         | 129 ------------------
 .../component/UBIComponentRecordingTest.java  |  95 +++++++++++++
 solr/packaging/test/test_ubi.bats             |   3 +-
 7 files changed, 119 insertions(+), 157 deletions(-)
 delete mode 100644 solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
 create mode 100644 solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 7e1affda886..3b28db919d0 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -174,20 +174,14 @@ public void inform(SolrCore core) {
       // tuple and return it.
       // expr = "logging(ubi_queries.jsonl," + "get(ubi-query)" + ")";
     } else {
-      LineNumberReader bufferedReader;
 
-      try {
-        bufferedReader =
-            new LineNumberReader(
-                new InputStreamReader(
-                    core.getResourceLoader().openResource(ubiQueryStreamProcessingExpression),
-                    StandardCharsets.UTF_8));
-
-        String[] args = {}; // maybe we have variables?
+      String[] args = {}; // maybe we have variables?
+      try (LineNumberReader bufferedReader =
+          new LineNumberReader(
+              new InputStreamReader(
+                  core.getResourceLoader().openResource(ubiQueryStreamProcessingExpression),
+                  StandardCharsets.UTF_8))) {
         expr = readExpression(bufferedReader, args);
-
-        bufferedReader.close();
-
       } catch (IOException ioe) {
         throw new SolrException(
             SolrException.ErrorCode.SERVER_ERROR,
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
index 2e2f038a630..ddc7a92213c 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
@@ -16,6 +16,8 @@
  */
 package org.apache.solr.handler.component;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -90,7 +92,15 @@ public Map toMap() {
     map.put(UBIComponent.QUERY_ID, this.queryId);
     map.put(UBIComponent.APPLICATION, this.application);
     map.put(UBIComponent.USER_QUERY, this.userQuery);
-    map.put(UBIComponent.QUERY_ATTRIBUTES, this.queryAttributes);
+    if (this.queryAttributes != null) {
+      ObjectMapper objectMapper = new ObjectMapper();
+      try {
+        map.put(
+            UBIComponent.QUERY_ATTRIBUTES, objectMapper.writeValueAsString(this.queryAttributes));
+      } catch (JsonProcessingException e) {
+        e.printStackTrace();
+      }
+    }
 
     return map;
   }
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml
index 2b493e0259b..498b1b8a234 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml
@@ -33,7 +33,6 @@
     
       ${solr.commitwithin.softcommit:true}
     
-    
   
   
   
@@ -51,16 +50,5 @@
     
   
 
-  
-    
-      ${solr.test.maxFields:1234}
-    
-    
-    
-    
-  
 
-  
-    
-  
 
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
index 208287abc69..d55010e7ef9 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
@@ -15,7 +15,12 @@
 
 commit(ubi_queries,
   update(ubi_queries,
-    tuple(id=49,a_i=1,b_i=5)
+    select(
+      ubiQueryTuple(),
+      query_id as id,
+      application,
+      user_query
+    )
   )
 )
 
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
deleted file mode 100644
index d2bae3e5363..00000000000
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentLocalLoggingTest.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.component;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.solr.client.solrj.io.Lang;
-import org.apache.solr.client.solrj.io.SolrClientCache;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.stream.StreamContext;
-import org.apache.solr.client.solrj.io.stream.TupleStream;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.handler.LoggingStream;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/** This test demonstrates, well, not totally sure. It may be we could delete this? */
-public class UBIComponentLocalLoggingTest extends SolrCloudTestCase {
-
-  private static final String COLLECTION = "collection1";
-
-  @BeforeClass
-  public static void setupCluster() throws Exception {
-    configureCluster(1)
-        .addConfig(
-            "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
-        .configure();
-  }
-
-  @Test
-  public void testLocalLoggingStream() throws Exception {
-
-    CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 1, 1, 0)
-        .process(cluster.getSolrClient());
-    cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
-
-    TupleStream stream;
-    List tuples;
-    StreamContext streamContext = new StreamContext();
-
-    SolrCore solrCoreToLoad = null;
-    for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
-      for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
-        if (solrCore != null) {
-          solrCoreToLoad = solrCore;
-          break;
-        }
-      }
-    }
-
-    final Path dataDir = findUserFilesDataDir();
-    Files.createDirectories(dataDir);
-    // populateFileStreamData(dataDir);
-
-    CoreContainer cc = cluster.getJettySolrRunner(0).getCoreContainer();
-
-    var l = cc.getAllCoreNames();
-    SolrCore core = cc.getCore(l.get(0));
-    streamContext.put("solr-core", core);
-    SolrClientCache solrClientCache = new SolrClientCache();
-
-    streamContext.setSolrClientCache(solrClientCache);
-
-    StreamFactory streamFactory = new StreamFactory();
-
-    // LoggingStream lets us write out tuples in jsonl format to a file in userfiles space.
-    streamFactory.withFunctionName("logging", LoggingStream.class);
-
-    Lang.register(streamFactory);
-
-    String clause = "logging(bob.jsonl,echo(\"bob\"))";
-    stream = streamFactory.constructStream(clause);
-    stream.setStreamContext(streamContext);
-    tuples = getTuples(stream);
-    stream.close();
-    solrClientCache.close();
-  }
-
-  private static Path findUserFilesDataDir() {
-    for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
-      for (CoreDescriptor coreDescriptor : jetty.getCoreContainer().getCoreDescriptors()) {
-        if (coreDescriptor.getCollectionName().equals(COLLECTION)) {
-          return jetty.getCoreContainer().getUserFilesPath();
-        }
-      }
-    }
-
-    throw new IllegalStateException("Unable to determine data-dir for: " + COLLECTION);
-  }
-
-  private List getTuples(TupleStream tupleStream) throws IOException {
-    tupleStream.open();
-    List tuples = new ArrayList<>();
-    for (; ; ) {
-      Tuple t = tupleStream.read();
-      // log.info(" ... {}", t.fields);
-      if (t.EOF) {
-        break;
-      } else {
-        tuples.add(t);
-      }
-    }
-    tupleStream.close();
-    return tuples;
-  }
-}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
new file mode 100644
index 00000000000..122e4d75744
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.json.JsonQueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * This test demonstrates that the backend recording of events via Streaming Expressions functions.
+ */
+public class UBIComponentRecordingTest extends SolrCloudTestCase {
+
+  private static final String COLLECTION = "collection1"; // The source of ubi=true enabled queries
+  private static final String UBI_QUERIES_COLLECTION = "ubi_queries"; // where we store query data
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    configureCluster(1)
+        .addConfig(
+            "ubi-enabled-config",
+            TEST_PATH().resolve("configsets").resolve("ubi-enabled").resolve("conf"))
+        .addConfig(
+            "minimal-config",
+            TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+        .configure();
+  }
+
+  @AfterClass
+  public static void afterTest() throws Exception {
+    CollectionAdminRequest.deleteCollection(COLLECTION).process(cluster.getSolrClient());
+    CollectionAdminRequest.deleteCollection(UBI_QUERIES_COLLECTION)
+        .process(cluster.getSolrClient());
+  }
+
+  @Test
+  public void testRecordingUBIQueries() throws Exception {
+
+    assertEquals(
+        "failed to create collection " + COLLECTION,
+        0,
+        CollectionAdminRequest.createCollection(COLLECTION, "ubi-enabled-config", 2, 1, 1, 0)
+            .process(cluster.getSolrClient())
+            .getStatus());
+
+    assertEquals(
+        "failed to create collection " + UBI_QUERIES_COLLECTION,
+        0,
+        CollectionAdminRequest.createCollection(
+                UBI_QUERIES_COLLECTION, "minimal-config", 2, 1, 1, 0)
+            .process(cluster.getSolrClient())
+            .getStatus());
+
+    cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
+    cluster.waitForActiveCollection(UBI_QUERIES_COLLECTION, 2, 2 * (1 + 1));
+
+    // TODO why doens't this work?
+    //    assertQ(
+    //            "Make sure we generate a query id",
+    //            req("q", "aa", "rows", "2", "ubi", "true"),
+    //            "count(//lst[@name='ubi']/str[@name='query_id'])=1");
+
+    // query our collection and confirm no duplicates on the signature field (using faceting)
+    // Check every (node) for consistency...
+    final ModifiableSolrParams overrideParams = new ModifiableSolrParams();
+    overrideParams.set("ubi", true);
+    final JsonQueryRequest req =
+        new JsonQueryRequest(overrideParams)
+            .setQuery("*:*")
+            // .setUBI(true)
+            .setLimit(0);
+    QueryResponse queryResponse = req.process(cluster.getSolrClient(), COLLECTION);
+    // assertResponseFoundNumDocs(queryResponse, expectedResults);
+    System.out.println(queryResponse);
+  }
+}
diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats
index 4e501dcabb5..a7b2df1f73e 100644
--- a/solr/packaging/test/test_ubi.bats
+++ b/solr/packaging/test/test_ubi.bats
@@ -39,7 +39,6 @@ teardown() {
     "add-searchcomponent": {
       "name": "ubi",
       "class": "solr.UBIComponent",
-      "defaults":{ }
     }
   }' "http://localhost:${SOLR_PORT}/api/collections/techproducts/config"
 
@@ -87,7 +86,7 @@ teardown() {
   
   # Check UBI query record was written out to default location with additional metadata
   assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"query_id":"xyz890"'
-  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"experiment":"supersecret"'
+  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl 'supersecret'
   
   
 }

From c86419d1fad255074a605126c223bb654ca8a1a0 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sun, 17 Nov 2024 13:45:01 -0500
Subject: [PATCH 070/103] Rearrange some methods

---
 .../solr/handler/component/UBIComponent.java  | 33 ++++++-----------
 .../component/UBIComponentRecordingTest.java  | 36 ++++++++++++-------
 2 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 3b28db919d0..5a5c78f3f9e 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -223,10 +223,6 @@ public void process(ResponseBuilder rb) throws IOException {
     }
 
     SolrIndexSearcher searcher = rb.req.getSearcher();
-    IndexSchema schema = searcher.getSchema();
-    if (schema.getUniqueKeyField() == null) {
-      return;
-    }
 
     String queryId = params.get(QUERY_ID);
     UBIQuery ubiQuery = new UBIQuery(queryId);
@@ -254,7 +250,7 @@ public void process(ResponseBuilder rb) throws IOException {
     ResultContext rc = (ResultContext) rb.rsp.getResponse();
     DocList docs = rc.getDocList();
 
-    String docIds = extractDocIds(docs, schema, searcher);
+    String docIds = extractDocIds(docs, searcher);
     ubiQuery.setDocIds(docIds);
 
     addUserBehaviorInsightsToResponse(ubiQuery, rb);
@@ -279,8 +275,13 @@ private void addUserBehaviorInsightsToResponse(UBIQuery ubiQuery, ResponseBuilde
     rb.rsp.add("ubi", ubiResponseInfo);
   }
 
-  protected String extractDocIds(DocList dl, IndexSchema schema, SolrIndexSearcher searcher)
-      throws IOException {
+  protected String extractDocIds(DocList dl, SolrIndexSearcher searcher) throws IOException {
+    IndexSchema schema = searcher.getSchema();
+
+    if (schema.getUniqueKeyField() == null) {
+      log.error("Can't track documents for query without unique field.");
+      return "";
+    }
     StringBuilder sb = new StringBuilder();
 
     Set fields = Collections.singleton(schema.getUniqueKeyField().getName());
@@ -357,30 +358,18 @@ public static String readExpression(LineNumberReader bufferedReader, String[] ar
   }
 
   private static TupleStream constructStream(
-      StreamFactory streamFactory, StreamExpression streamExpression) throws IOException {
+      StreamFactory streamFactory, StreamExpression streamExpression) {
     try {
       return streamFactory.constructStream(streamExpression);
     } catch (IOException exception) {
-      // Throw or just log an error?
       throw new SolrException(
           SolrException.ErrorCode.SERVER_ERROR,
-          "Error constructing stream for processing UBI data collection: "
-              + UBIComponent.class.getSimpleName(),
+          "Error constructing stream for processing UBI data collection using expression "
+              + streamExpression,
           exception);
     }
   }
 
-  /*
-    @SuppressWarnings({"rawtypes"})
-    public static Map validateLetAndGetParams(TupleStream stream, String expr) throws IOException {
-      if (stream instanceof LetStream) {
-        LetStream mainStream = (LetStream) stream;
-        return mainStream.getLetParams();
-      } else {
-        throw new IOException("No enclosing let function found in expression:" + expr);
-      }
-    }
-  */
   @Override
   public String getDescription() {
     return "A component that tracks the original user query and the resulting documents returned.";
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
index 122e4d75744..0a7c9dd7a82 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
@@ -16,11 +16,13 @@
  */
 package org.apache.solr.handler.component;
 
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.json.JsonQueryRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.MapSolrParams;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -55,6 +57,7 @@ public static void afterTest() throws Exception {
   @Test
   public void testRecordingUBIQueries() throws Exception {
 
+    // Create the collection that will generate UBI events
     assertEquals(
         "failed to create collection " + COLLECTION,
         0,
@@ -62,6 +65,8 @@ public void testRecordingUBIQueries() throws Exception {
             .process(cluster.getSolrClient())
             .getStatus());
 
+    // Create the collection that will recieve UBI events through the ubi-query-pipeline.expr
+    // streaming expression
     assertEquals(
         "failed to create collection " + UBI_QUERIES_COLLECTION,
         0,
@@ -79,16 +84,23 @@ public void testRecordingUBIQueries() throws Exception {
     //            req("q", "aa", "rows", "2", "ubi", "true"),
     //            "count(//lst[@name='ubi']/str[@name='query_id'])=1");
 
-    // query our collection and confirm no duplicates on the signature field (using faceting)
-    // Check every (node) for consistency...
-    final ModifiableSolrParams overrideParams = new ModifiableSolrParams();
-    overrideParams.set("ubi", true);
-    final JsonQueryRequest req =
-        new JsonQueryRequest(overrideParams)
-            .setQuery("*:*")
-            // .setUBI(true)
-            .setLimit(0);
-    QueryResponse queryResponse = req.process(cluster.getSolrClient(), COLLECTION);
+    final Map queryParamMap = new HashMap<>();
+    queryParamMap.put("q", "*:*");
+    queryParamMap.put("ubi", "true");
+    MapSolrParams queryParams = new MapSolrParams(queryParamMap);
+    final QueryResponse queryResponse = cluster.getSolrClient().query(COLLECTION, queryParams);
+    final SolrDocumentList documents = queryResponse.getResults();
+
+    //
+    //
+    //    final ModifiableSolrParams overrideParams = new ModifiableSolrParams();
+    //    overrideParams.set("ubi", true);
+    //    final JsonQueryRequest req =
+    //        new JsonQueryRequest(overrideParams)
+    //            .setQuery("*:*")
+    //            // .setUBI(true)
+    //            .setLimit(0);
+    //    QueryResponse queryResponse = req.process(cluster.getSolrClient(), COLLECTION);
     // assertResponseFoundNumDocs(queryResponse, expectedResults);
     System.out.println(queryResponse);
   }

From 1d652ef6066bf468d993ba10154dc6e830850bdc Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sun, 17 Nov 2024 14:12:10 -0500
Subject: [PATCH 071/103] Use shorter name "ubiQuery", we know they return
 tuples!

---
 .../solr/handler/component/UBIComponent.java       | 14 +++++++-------
 .../apache/solr/handler/component/UBIQuery.java    |  2 +-
 .../ubi-enabled/conf/ubi-query-pipeline.expr       |  2 +-
 .../solr/handler/component/UBIComponentTest.java   |  4 ++--
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 5a5c78f3f9e..ca20ddef63a 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -65,11 +65,11 @@
  *
  * 
You provide a streaming expression that is parsed and loaded by the component to stream query
  * data to a target of your choice. If you do not, then the default expression of
- * 'logging(ubi_queries.jsonl,ubiQueryTuple())"' is used which logs data to
+ * 'logging(ubi_queries.jsonl,ubiQuery())"' is used which logs data to
  * $SOLR_HOME/userfiles/ubi_queries.jsonl file.
  *
- * 
You must source your streaming events using the 'ubiQueryTuple()' streaming expression to
- * retrieve the {@link UBIQuery} object that contains the data for recording.
+ * 
You must source your streaming events using the 'ubiQuery()' streaming expression to retrieve
+ * the {@link UBIQuery} object that contains the data for recording.
  *
  * 
Event data is tracked by letting the user write events directly to the event repository of
  * your choice, it could be a Solr collection, it could be a file or S3 bucket, and that is NOT
@@ -166,7 +166,7 @@ public void inform(SolrCore core) {
       // expr = "logging(ubi_queries.jsonl, tuple(query_id=49,user_query=\"RAM memory\"))";
 
       // The default version
-      expr = "logging(ubi_queries.jsonl,ubiQueryTuple())";
+      expr = "logging(ubi_queries.jsonl,ubiQuery())";
 
       // feels like 'stream' or 'get' or something should let me create a tuple out of something
       // in the
@@ -195,16 +195,16 @@ public void inform(SolrCore core) {
     streamContext.setSolrClientCache(solrClientCache);
 
     streamExpression = StreamExpressionParser.parse(expr);
-    if (!streamExpression.toString().contains("ubiQueryTuple")) {
+    if (!streamExpression.toString().contains("ubiQuery")) {
       log.error(
           "The streaming expression "
               + streamExpression
-              + " must include the 'ubiQueryTuple()' to record UBI queries.");
+              + " must include the 'ubiQuery()' to record UBI queries.");
     }
 
     streamFactory = new DefaultStreamFactory();
     streamFactory.withFunctionName("logging", LoggingStream.class);
-    streamFactory.withFunctionName("ubiQueryTuple", UBIQueryStream.class);
+    streamFactory.withFunctionName("ubiQuery", UBIQueryStream.class);
 
     if (coreContainer.isZooKeeperAware()) {
       String defaultZkHost = core.getCoreContainer().getZkController().getZkServerAddress();
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
index ddc7a92213c..4642f32bd97 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
@@ -98,7 +98,7 @@ public Map toMap() {
         map.put(
             UBIComponent.QUERY_ATTRIBUTES, objectMapper.writeValueAsString(this.queryAttributes));
       } catch (JsonProcessingException e) {
-        e.printStackTrace();
+        // eat it.
       }
     }
 
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
index d55010e7ef9..ca9b2d9797c 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
@@ -16,7 +16,7 @@
 commit(ubi_queries,
   update(ubi_queries,
     select(
-      ubiQueryTuple(),
+      ubiQuery(),
       query_id as id,
       application,
       user_query
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
index ee02ad35753..96be7433c98 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentTest.java
@@ -66,7 +66,7 @@ public void testGenerateQueryIdZeroRowsRequested() {
   }
 
   @Test
-  public void testJSONQuerySyntaWithJustUBI() throws Exception {
+  public void testJSONQuerySyntaxWithJustUBI() throws Exception {
     String response =
         JQ(
             req(
@@ -81,7 +81,7 @@ public void testJSONQuerySyntaWithJustUBI() throws Exception {
                     + "        'ubi': 'true'\n"
                     + "   }\n"
                     + "}"));
-    assertTrue(response.indexOf("query_id") != -1);
+    assertTrue(response.contains("query_id"));
   }
 
   @Test

From e72a7ecd27778e5cbb7696ffc1e26edc899625c0 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 18 Nov 2024 05:53:21 -0500
Subject: [PATCH 072/103] Back out change, there is an actual test for what I
 think is a bug or a deprecated way of phrasing something

---
 .../org/apache/solr/client/solrj/io/stream/UpdateStream.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java
index 4e1b5e4799b..7d10407ee68 100644
--- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java
+++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java
@@ -211,7 +211,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException {
     StreamExplanation explanation = new StreamExplanation(getStreamNodeId() + "-datastore");
 
     explanation.setFunctionName(String.format(Locale.ROOT, "solr (%s)", collection));
-    explanation.setImplementingClass(this.getClass().getName());
+    explanation.setImplementingClass("Solr/Lucene");
     explanation.setExpressionType(ExpressionType.DATASTORE);
     explanation.setExpression("Update into " + collection);
 

From f1085853211699668e8752368c48f070a906a09b Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 18 Nov 2024 09:22:51 -0500
Subject: [PATCH 073/103] Improved testing of ubiQuery() streams to figure out
 how to deal with queryAttributes map

---
 .../apache/solr/handler/LoggingStream.java    |  10 +-
 .../solr/handler/component/UBIComponent.java  |   7 +-
 .../solr/handler/component/UBIQuery.java      |  20 +-
 .../component/UBIComponentRecordingTest.java  |   4 +-
 .../UBIComponentStreamingQueriesTest.java     | 601 +++++++++---------
 5 files changed, 318 insertions(+), 324 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/LoggingStream.java b/solr/core/src/java/org/apache/solr/handler/LoggingStream.java
index e101ab68d9f..8839686c1e7 100644
--- a/solr/core/src/java/org/apache/solr/handler/LoggingStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/LoggingStream.java
@@ -71,6 +71,8 @@ public class LoggingStream extends TupleStream implements Expressible {
    */
   private String filepath;
 
+  private Path filePath;
+
   private int updateBatchSize;
 
   private int batchNumber;
@@ -122,14 +124,14 @@ private void init(String filepath, TupleStream tupleSource) {
     this.tupleSource = tupleSource;
   }
 
-  /** The name of the file being updated */
-  protected String getFilePath() {
-    return filepath;
+  /** The path of the file being logged to */
+  public Path getFilePath() {
+    return filePath;
   }
 
   @Override
   public void open() throws IOException {
-    Path filePath = chroot.resolve(filepath).normalize();
+    filePath = chroot.resolve(filepath).normalize();
     if (!filePath.startsWith(chroot)) {
       throw new SolrException(
           SolrException.ErrorCode.BAD_REQUEST, "file to log to must be under " + chroot);
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index ca20ddef63a..ef644fecad3 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -230,7 +230,7 @@ public void process(ResponseBuilder rb) throws IOException {
     ubiQuery.setUserQuery(params.get(USER_QUERY));
     ubiQuery.setApplication(params.get(APPLICATION));
 
-    Object queryAttributes = params.get(QUERY_ATTRIBUTES);
+    String queryAttributes = params.get(QUERY_ATTRIBUTES);
 
     if (queryAttributes != null && queryAttributes.toString().startsWith("{")) {
       // Look up the original nested JSON format, typically passed in
@@ -241,8 +241,9 @@ public void process(ResponseBuilder rb) throws IOException {
         @SuppressWarnings("rawtypes")
         Map paramsProperties = (Map) jsonProperties.get("params");
         if (paramsProperties.containsKey(QUERY_ATTRIBUTES)) {
-          queryAttributes = paramsProperties.get(QUERY_ATTRIBUTES);
-          ubiQuery.setQueryAttributes(queryAttributes);
+          @SuppressWarnings("rawtypes")
+          Map queryAttributesAsMap = (Map) paramsProperties.get(QUERY_ATTRIBUTES);
+          ubiQuery.setQueryAttributes(queryAttributesAsMap);
         }
       }
     }
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
index 4642f32bd97..8605142029c 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
@@ -34,7 +34,10 @@ public class UBIQuery {
   private String application;
   private String queryId;
   private String userQuery;
-  private Object queryAttributes;
+
+  @SuppressWarnings("rawtypes")
+  private Map queryAttributes;
+
   private String docIds;
 
   public UBIQuery(String queryId) {
@@ -69,11 +72,13 @@ public void setUserQuery(String userQuery) {
     this.userQuery = userQuery;
   }
 
-  public Object getQueryAttributes() {
+  @SuppressWarnings("rawtypes")
+  public Map getQueryAttributes() {
     return queryAttributes;
   }
 
-  public void setQueryAttributes(Object queryAttributes) {
+  @SuppressWarnings("rawtypes")
+  public void setQueryAttributes(Map queryAttributes) {
     this.queryAttributes = queryAttributes;
   }
 
@@ -90,9 +95,14 @@ public Map toMap() {
     @SuppressWarnings({"rawtypes", "unchecked"})
     Map map = new HashMap();
     map.put(UBIComponent.QUERY_ID, this.queryId);
-    map.put(UBIComponent.APPLICATION, this.application);
-    map.put(UBIComponent.USER_QUERY, this.userQuery);
+    if (this.application != null) {
+      map.put(UBIComponent.APPLICATION, this.application);
+    }
+    if (this.userQuery != null) {
+      map.put(UBIComponent.USER_QUERY, this.userQuery);
+    }
     if (this.queryAttributes != null) {
+
       ObjectMapper objectMapper = new ObjectMapper();
       try {
         map.put(
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
index 0a7c9dd7a82..d32483fbc20 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
@@ -91,6 +91,8 @@ public void testRecordingUBIQueries() throws Exception {
     final QueryResponse queryResponse = cluster.getSolrClient().query(COLLECTION, queryParams);
     final SolrDocumentList documents = queryResponse.getResults();
 
+
+
     //
     //
     //    final ModifiableSolrParams overrideParams = new ModifiableSolrParams();
@@ -102,6 +104,6 @@ public void testRecordingUBIQueries() throws Exception {
     //            .setLimit(0);
     //    QueryResponse queryResponse = req.process(cluster.getSolrClient(), COLLECTION);
     // assertResponseFoundNumDocs(queryResponse, expectedResults);
-    System.out.println(queryResponse);
+    //System.out.println(queryResponse);
   }
 }
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index 50bee76c7f8..5ffac3bcd22 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -1,389 +1,368 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
 package org.apache.solr.handler.component;
 
-import java.io.File;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import org.apache.commons.io.input.ReversedLinesFileReader;
-import org.apache.lucene.util.IOUtils;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.io.Lang;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.CloudSolrStream;
+import org.apache.solr.client.solrj.io.stream.SelectStream;
 import org.apache.solr.client.solrj.io.stream.StreamContext;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
 import org.apache.solr.client.solrj.io.stream.UpdateStream;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.request.json.JsonQueryRequest;
-import org.apache.solr.client.solrj.request.json.TermsFacetMap;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.client.solrj.response.json.BucketBasedJsonFacet;
-import org.apache.solr.client.solrj.response.json.BucketJsonFacet;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
 import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.core.SolrCore;
 import org.apache.solr.embedded.JettySolrRunner;
-import org.junit.After;
-import org.junit.AfterClass;
+import org.apache.solr.handler.LoggingStream;
+import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Test;
 
-/**
- * Tests the ability for {@link UBIComponent} to stream the gathered query data to another Solr
- * index using Streaming Expressions.
- *
- * 
This guy needs simplification!!!!!!!!! Needs to look more like some of the tests Joel wrote.
- */
+@LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene45"})
 public class UBIComponentStreamingQueriesTest extends SolrCloudTestCase {
-  public static final String COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION =
-      "collection_stream_ubi_queries_to_ubi_collection";
-  public static final String COLLECTION_STREAM_UBI_QUERIES_TO_LOG =
-      "collection_stream_ubi_queries_to_log";
-  public static final String UBI_QUERIES_COLLECTION = "ubi_queries";
-
-  /** One client per node */
-  private static final List NODE_CLIENTS = new ArrayList<>(7);
 
-  /**
-   * clients (including cloud client) for easy randomization and looping of collection level
-   * requests
-   */
-  private static final List CLIENTS = new ArrayList<>(7);
+  private static final String COLLECTIONORALIAS = "collection1";
+  private static final int TIMEOUT = DEFAULT_TIMEOUT;
+  private static final String id = "id";
 
-  private static String zkHost;
+  private static boolean useAlias;
 
   @BeforeClass
   public static void setupCluster() throws Exception {
-
-    final int numShards = usually() ? 2 : 1;
-    final int numReplicas = usually() ? 2 : 1;
-    final int numNodes = 1 + (numShards * numReplicas); // at least one node w/o any replicas
-
-    // The configset ubi_enabled has the UBIComponent configured and set to log to a collection
-    // called "ubi".
-    // The ubi collection itself just depends on the typical _default configset.
-    configureCluster(numNodes)
-        .addConfig("ubi-enabled", configset("ubi-enabled"))
+    configureCluster(4)
         .addConfig(
-            "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+            "conf", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
         .configure();
 
-    zkHost = cluster.getZkServer().getZkAddress();
-
-    CLIENTS.add(cluster.getSolrClient());
-    for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
-      final SolrClient c = getHttpSolrClient(jetty.getBaseUrl().toString());
-      NODE_CLIENTS.add(c);
-      CLIENTS.add(c);
+    String collection;
+    useAlias = random().nextBoolean();
+    if (useAlias) {
+      collection = COLLECTIONORALIAS + "_collection";
+    } else {
+      collection = COLLECTIONORALIAS;
     }
 
-    assertEquals(
-        "failed to create collection " + COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION,
-        0,
-        CollectionAdminRequest.createCollection(
-                COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION,
-                "ubi-enabled",
-                numShards,
-                numReplicas)
-            .process(cluster.getSolrClient())
-            .getStatus());
-
-    cluster.waitForActiveCollection(
-        COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, numShards, numShards * numReplicas);
-
-    assertEquals(
-        "failed to create collection " + COLLECTION_STREAM_UBI_QUERIES_TO_LOG,
-        0,
-        CollectionAdminRequest.createCollection(
-                COLLECTION_STREAM_UBI_QUERIES_TO_LOG, "config", numShards, numReplicas)
-            .process(cluster.getSolrClient())
-            .getStatus());
-
-    cluster.waitForActiveCollection(
-        COLLECTION_STREAM_UBI_QUERIES_TO_LOG, numShards, numShards * numReplicas);
-
-    assertEquals(
-        "failed to create UBI queries collection",
-        0,
-        CollectionAdminRequest.createCollection(
-                UBI_QUERIES_COLLECTION, "_default", numShards, numReplicas)
-            .process(cluster.getSolrClient())
-            .getStatus());
-
-    cluster.waitForActiveCollection(UBI_QUERIES_COLLECTION, numShards, numShards * numReplicas);
-  }
+    CollectionAdminRequest.createCollection(collection, "conf", 2, 1)
+        .process(cluster.getSolrClient());
+
+    cluster.waitForActiveCollection(collection, 2, 2);
 
-  @AfterClass
-  public static void closeClients() throws Exception {
-    try {
-      IOUtils.close(NODE_CLIENTS);
-    } finally {
-      NODE_CLIENTS.clear();
-      CLIENTS.clear();
+    AbstractDistribZkTestBase.waitForRecoveriesToFinish(
+        collection, cluster.getZkStateReader(), false, true, TIMEOUT);
+    if (useAlias) {
+      CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection)
+          .process(cluster.getSolrClient());
     }
   }
 
-  @After
-  public void clearCollection() throws Exception {
-    assertEquals(
-        "DBQ failed",
-        0,
-        cluster
-            .getSolrClient()
-            .deleteByQuery(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION, "*:*")
-            .getStatus());
-    assertEquals(
-        "commit failed",
-        0,
-        cluster
-            .getSolrClient()
-            .commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION)
-            .getStatus());
-    assertEquals(
-        "DBQ failed",
-        0,
-        cluster.getSolrClient().deleteByQuery(UBI_QUERIES_COLLECTION, "*:*").getStatus());
-    assertEquals(
-        "commit failed", 0, cluster.getSolrClient().commit(UBI_QUERIES_COLLECTION).getStatus());
+  @Before
+  public void cleanIndex() throws Exception {
+    new UpdateRequest().deleteByQuery("*:*").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
   }
 
-  public void testCreatingStreamingExpression() {
-    UBIQuery ubiQuery = new UBIQuery("5678");
-    ubiQuery.setUserQuery("Apple Memory");
+  @Test
+  public void testUBIQueryStream() throws Exception {
+
+    UBIQuery ubiQuery;
+    StreamExpression expression;
+    TupleStream stream;
+    List tuples;
+    StreamContext streamContext = new StreamContext();
+    SolrClientCache solrClientCache = new SolrClientCache();
 
-    String clause = getClause(ubiQuery);
-    assertEquals(
-        "Check the decoded version for ease of comparison",
-        "commit(ubi,update(ubi,tuple(id=4.0,query_id=5678,user_query=Apple Memory)))",
-        clause);
+    try (solrClientCache) {
+      streamContext.setSolrClientCache(solrClientCache);
+      StreamFactory factory =
+          new StreamFactory().withFunctionName("ubiQuery", UBIQueryStream.class);
+      // Basic test
+      ubiQuery = new UBIQuery("123");
+
+      expression = StreamExpressionParser.parse("ubiQuery()");
+      streamContext.put("ubi-query", ubiQuery);
+      stream = new UBIQueryStream(expression, factory);
+      stream.setStreamContext(streamContext);
+      tuples = getTuples(stream);
+
+      assertEquals(1, tuples.size());
+      assertFields(tuples, "query_id");
+      assertString(tuples.get(0), "query_id", "123");
+      // assertNotFields(tuples, "user_query", "event_attributes");
+
+      // Include another field to see what is returned
+      ubiQuery = new UBIQuery("234");
+      ubiQuery.setApplication("typeahead");
+
+      streamContext.put("ubi-query", ubiQuery);
+      stream = new UBIQueryStream(expression, factory);
+      stream.setStreamContext(streamContext);
+      tuples = getTuples(stream);
+
+      assertEquals(1, tuples.size());
+      assertFields(tuples, "query_id", "application");
+      assertString(tuples.get(0), "query_id", "234");
+      assertString(tuples.get(0), "application", "typeahead");
+
+      // Introduce event_attributes map of data
+      ubiQuery = new UBIQuery("345");
+
+      @SuppressWarnings({"unchecked", "rawtypes"})
+      Map queryAttributes = new HashMap();
+      queryAttributes.put("attribute1", "one");
+      queryAttributes.put("attribute2", 2);
+      ubiQuery.setQueryAttributes(queryAttributes);
+
+      streamContext.put("ubi-query", ubiQuery);
+      stream = new UBIQueryStream(expression, factory);
+      stream.setStreamContext(streamContext);
+      tuples = getTuples(stream);
+
+      assertEquals(1, tuples.size());
+      assertFields(tuples, "query_id", "query_attributes");
+      assertString(tuples.get(0), "query_id", "345");
+      assertString(tuples.get(0), "query_attributes", "{\"attribute1\":\"one\",\"attribute2\":2}");
+    }
   }
 
-  public void testUsingStreamingExpressionDirectly() throws Exception {
+  @Test
+  public void testWritingToLogUbiQueryStream() throws Exception {
+    // Test that we can write out UBIQuery data cleanly to the jsonl file
+    UBIQuery ubiQuery = new UBIQuery("345");
+    ubiQuery.setUserQuery("Memory RAM");
+    ubiQuery.setApplication("typeahead");
+
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    Map queryAttributes = new HashMap();
+    queryAttributes.put("parsed_query", "memory OR ram");
+    queryAttributes.put("experiment", "secret");
+    queryAttributes.put("marginBoost", 2.1);
+    ubiQuery.setQueryAttributes(queryAttributes);
+
+    StreamExpression expression;
+    List tuples;
+    StreamContext streamContext = new StreamContext();
+    SolrClientCache solrClientCache = new SolrClientCache();
+
+    try (solrClientCache) {
+      streamContext.setSolrClientCache(solrClientCache);
+      StreamFactory factory =
+          new StreamFactory()
+              .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+              .withFunctionName("search", CloudSolrStream.class)
+              .withFunctionName("ubiQuery", UBIQueryStream.class)
+              .withFunctionName("logging", LoggingStream.class);
+
+      expression = StreamExpressionParser.parse("logging(test.jsonl,ubiQuery())");
+      streamContext.put("ubi-query", ubiQuery);
+      streamContext.put("solr-core", findSolrCore());
+      LoggingStream stream = new LoggingStream(expression, factory);
+      stream.setStreamContext(streamContext);
+      tuples = getTuples(stream);
+
+      assertEquals(1, tuples.size());
+      assertFields(tuples, "totalIndexed");
+      assertLong(tuples.get(0), "totalIndexed", 1);
+
+      // Someday when we have parseJSON() streaming expression we can replace this.
+      Path filePath = stream.getFilePath();
+      try (ReversedLinesFileReader reader =
+          new ReversedLinesFileReader.Builder()
+              .setCharset(StandardCharsets.UTF_8)
+              .setPath(filePath)
+              .get()) {
+        String jsonLine = reader.readLine(); // Read the last line
+        assertNotNull(jsonLine);
+        ObjectMapper objectMapper = new ObjectMapper();
+        @SuppressWarnings({"unchecked", "rawtypes"})
+        Map myObject = objectMapper.readValue(jsonLine, Map.class);
+        assertEquals(ubiQuery.getQueryId(), myObject.get("query_id"));
+        assertEquals(ubiQuery.getApplication(), myObject.get("application"));
+        // assertEquals(ubiQuery.getQueryAttributes(), myObject.get("query_attributes"));
+        assertEquals(
+            "{\"experiment\":\"secret\",\"marginBoost\":2.1,\"parsed_query\":\"memory OR ram\"}",
+            myObject.get("query_attributes"));
+      }
+    }
+  }
 
-    UBIQuery ubiQuery = new UBIQuery("5678");
-    ubiQuery.setUserQuery("Apple Memory");
+  @Test
+  public void testWritingToSolrUbiQueryStream() throws Exception {
+    // Test that we can write out UBIQuery, especially the queryAttributes map, to Solr collection
 
+    UBIQuery ubiQuery = new UBIQuery("345");
+    ubiQuery.setUserQuery("Memory RAM");
+    ubiQuery.setApplication("typeahead");
+
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    Map queryAttributes = new HashMap();
+    queryAttributes.put("parsed_query", "memory OR ram");
+    queryAttributes.put("experiment", "secret");
+    queryAttributes.put("marginBoost", 2.1);
+    ubiQuery.setQueryAttributes(queryAttributes);
+
+    StreamExpression expression;
     TupleStream stream;
     List tuples;
     StreamContext streamContext = new StreamContext();
     SolrClientCache solrClientCache = new SolrClientCache();
 
-    streamContext.setSolrClientCache(solrClientCache);
-
-    StreamFactory streamFactory =
-        new StreamFactory().withCollectionZkHost(UBI_QUERIES_COLLECTION, zkHost);
-
-    Lang.register(streamFactory);
-
-    String clause = getClause(ubiQuery);
-    stream = streamFactory.constructStream(clause);
-    stream.setStreamContext(streamContext);
-    tuples = getTuples(stream);
-    stream.close();
-    solrClientCache.close();
-
-    assertEquals("Total tuples returned", 1, tuples.size());
-    Tuple tuple = tuples.get(0);
-    assertEquals("1", tuple.getString(UpdateStream.BATCH_INDEXED_FIELD_NAME));
-    assertEquals("1", tuple.getString("totalIndexed"));
-
-    // Check the UBI collection
-    final JsonQueryRequest requestFromUBICollection =
-        new JsonQueryRequest().setQuery("id:4.0").setLimit(1);
-
-    // Randomly grab a client, it shouldn't matter which is used to check UBI event.
-    SolrClient client = getRandClient();
-    final QueryResponse responseUBI =
-        requestFromUBICollection.process(client, UBI_QUERIES_COLLECTION);
-    try {
-      assertEquals(0, responseUBI.getStatus());
-      assertEquals(1, responseUBI.getResults().getNumFound());
-    } catch (AssertionError e) {
-      throw new AssertionError(responseUBI + " + " + client + " => " + e.getMessage(), e);
+    // String zkHost = cluster.getZkServer().getZkAddress();
+
+    try (solrClientCache) {
+      streamContext.setSolrClientCache(solrClientCache);
+      StreamFactory factory =
+          new StreamFactory()
+              .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+              .withFunctionName("search", CloudSolrStream.class)
+              .withFunctionName("update", UpdateStream.class)
+              .withFunctionName("select", SelectStream.class)
+              .withFunctionName("ubiQuery", UBIQueryStream.class);
+
+      expression =
+          StreamExpressionParser.parse(
+              "update("
+                  + COLLECTIONORALIAS
+                  + ", batchSize=5, select(\n"
+                  + "      ubiQuery(),\n"
+                  + "      query_id as id,\n"
+                  + "      application,\n"
+                  + "      user_query,\n"
+                  + "      query_attributes\n"
+                  + "    ))");
+      streamContext.put("ubi-query", ubiQuery);
+      stream = new UpdateStream(expression, factory);
+      stream.setStreamContext(streamContext);
+      tuples = getTuples(stream);
+
+      cluster.getSolrClient().commit(COLLECTIONORALIAS);
+
+      assertEquals(1, tuples.size());
+      Tuple t = tuples.get(0);
+      assertFalse(t.EOF);
+      assertEquals(1, t.get("batchIndexed"));
+      assertEquals(1L, t.get("totalIndexed"));
+
+      // Ensure that destinationCollection actually has the new ubi query docs.
+      expression =
+          StreamExpressionParser.parse(
+              "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,*\", sort=\"id asc\")");
+      stream = new CloudSolrStream(expression, factory);
+      stream.setStreamContext(streamContext);
+      tuples = getTuples(stream);
+      assertEquals(1, tuples.size());
+
+      Tuple tuple = tuples.get(0);
+      assertEquals(ubiQuery.getQueryId(), tuple.get("id"));
+      assertEquals(ubiQuery.getApplication(), tuple.get("application"));
+      assertEquals(
+          "{\"experiment\":\"secret\",\"marginBoost\":2.1,\"parsed_query\":\"memory OR ram\"}",
+          tuple.get("query_attributes"));
     }
   }
 
-  private List getTuples(TupleStream tupleStream) throws IOException {
-    tupleStream.open();
+  protected List getTuples(TupleStream tupleStream) throws IOException {
     List tuples = new ArrayList<>();
-    for (; ; ) {
-      Tuple t = tupleStream.read();
-      // log.info(" ... {}", t.fields);
-      if (t.EOF) {
-        break;
-      } else {
+
+    try (tupleStream) {
+      tupleStream.open();
+      for (Tuple t = tupleStream.read(); !t.EOF; t = tupleStream.read()) {
         tuples.add(t);
       }
     }
-    tupleStream.close();
     return tuples;
   }
 
-  private static String getClause(UBIQuery ubiQuery) {
-    return "commit("
-        + UBI_QUERIES_COLLECTION
-        + ",update("
-        + UBI_QUERIES_COLLECTION
-        + ",tuple(id=4.0,"
-        + toTuple(ubiQuery)
-        + ")))";
+  protected void assertOrder(List tuples, int... ids) throws Exception {
+    assertOrderOf(tuples, "id", ids);
   }
 
-  public static String toTuple(UBIQuery ubiQuery) {
-    return UBIComponent.QUERY_ID
-        + "="
-        + ubiQuery.getQueryId()
-        + ","
-        + UBIComponent.USER_QUERY
-        + "="
-        + ubiQuery.getUserQuery()
-        + ","
-        + UBIComponent.APPLICATION
-        + "="
-        + ubiQuery.getApplication();
+  protected void assertOrderOf(List tuples, String fieldName, int... ids) throws Exception {
+    int i = 0;
+    for (int val : ids) {
+      Tuple t = tuples.get(i);
+      String tip = t.getString(fieldName);
+      if (!tip.equals(Integer.toString(val))) {
+        throw new Exception("Found value:" + tip + " expecting:" + val);
+      }
+      ++i;
+    }
   }
 
-  @SuppressWarnings({"rawtypes", "unchecked"})
-  public void testRandomDocs() throws Exception {
-
-    final UpdateRequest ureq = new UpdateRequest();
-
-    ureq.add(sdoc("id", 1, "data_s", "data_1"));
-    assertEquals(
-        "add failed",
-        0,
-        ureq.process(getRandClient(), COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
-    assertEquals(
-        "commit failed",
-        0,
-        getRandClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
-
-    // query our collection to generate a UBI event and then confirm it was recorded.
-
-    String userQuery = "hot air";
-    Map queryAttributes = new HashMap();
-    queryAttributes.put("results_wanted", 1);
-
-    final JsonQueryRequest req =
-        new JsonQueryRequest()
-            .setQuery("*:*")
-            .setLimit(1)
-            .withParam("ubi", "true")
-            .withParam("query_id", "123")
-            .withParam("user_query", userQuery)
-            .withParam("query_attributes", queryAttributes);
-
-    // Randomly grab a client, it shouldn't matter which is used to generate the query event.
-    SolrClient client = getRandClient();
-    final QueryResponse rsp = req.process(client, COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION);
-    try {
-      assertEquals(0, rsp.getStatus());
-      assertEquals(1, rsp.getResults().getNumFound());
-    } catch (AssertionError e) {
-      throw new AssertionError(rsp + " + " + client + " => " + e.getMessage(), e);
+  public boolean assertString(Tuple tuple, String fieldName, String expected) throws Exception {
+    String actual = (String) tuple.get(fieldName);
+
+    if (!Objects.equals(expected, actual)) {
+      throw new Exception("Longs not equal:" + expected + " : " + actual);
     }
 
-    // Check the UBI collection
-    final JsonQueryRequest requestUBI = new JsonQueryRequest().setQuery("id:49").setLimit(1);
-
-    // Randomly grab a client, it shouldn't matter which is used, to check UBI event was actually
-    // tracked.
-    client = getRandClient();
-    final QueryResponse responseUBI = requestUBI.process(client, UBI_QUERIES_COLLECTION);
-    // try {
-    assertEquals(0, responseUBI.getStatus());
-    assertEquals(1, responseUBI.getResults().getNumFound());
-    // } catch (AssertionError e) {
-    //  throw new AssertionError(responseUBI + " + " + client + " => " + e.getMessage(), e);
-    // }
+    return true;
   }
 
-  public void randomDocs() throws Exception {
-
-    // index some random documents, using a mix-match of batches, to various SolrClients
-
-    final int uniqueMod = atLeast(43); // the number of unique sig values expected
-    final int numBatches = atLeast(uniqueMod); // we'll add at least one doc per batch
-    int docCounter = 0;
-    for (int batchId = 0; batchId < numBatches; batchId++) {
-      final UpdateRequest ureq = new UpdateRequest();
-      final int batchSize = atLeast(2);
-      for (int i = 0; i < batchSize; i++) {
-        docCounter++;
-        ureq.add(
-            sdoc( // NOTE: No 'id' field, SignatureUpdateProcessor fills it in for us
-                "data_s", (docCounter % uniqueMod)));
+  public boolean assertLong(Tuple tuple, String fieldName, long l) throws Exception {
+    long lv = (long) tuple.get(fieldName);
+    if (lv != l) {
+      throw new Exception("Longs not equal:" + l + " : " + lv);
+    }
+
+    return true;
+  }
+
+  protected void assertFields(List tuples, String... fields) throws Exception {
+    for (Tuple tuple : tuples) {
+      for (String field : fields) {
+        if (!tuple.getFields().containsKey(field)) {
+          throw new Exception(String.format(Locale.ROOT, "Expected field '%s' not found", field));
+        }
       }
-      assertEquals(
-          "add failed",
-          0,
-          ureq.process(getRandClient(), COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION)
-              .getStatus());
     }
-    assertEquals(
-        "commit failed",
-        0,
-        getRandClient().commit(COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION).getStatus());
-
-    assertTrue(docCounter > uniqueMod);
-
-    // query our collection and confirm no duplicates on the signature field (using faceting)
-    // Check every (node) for consistency...
-    final JsonQueryRequest req =
-        new JsonQueryRequest()
-            .setQuery("*:*")
-            .setLimit(0)
-            .withFacet("data_facet", new TermsFacetMap("data_s").setLimit(uniqueMod + 1));
-    for (SolrClient client : CLIENTS) {
-      final QueryResponse rsp =
-          req.process(client, COLLECTION_STREAM_UBI_QUERIES_TO_UBI_COLLECTION);
-      try {
-        assertEquals(0, rsp.getStatus());
-        assertEquals(uniqueMod, rsp.getResults().getNumFound());
-
-        final BucketBasedJsonFacet facet =
-            rsp.getJsonFacetingResponse().getBucketBasedFacets("data_facet");
-        assertEquals(uniqueMod, facet.getBuckets().size());
-        for (BucketJsonFacet bucket : facet.getBuckets()) {
-          assertEquals("Bucket " + bucket.getVal(), 1, bucket.getCount());
+  }
+
+  protected void assertNotFields(List tuples, String... fields) throws Exception {
+    for (Tuple tuple : tuples) {
+      for (String field : fields) {
+        if (tuple.getFields().containsKey(field)) {
+          throw new Exception(String.format(Locale.ROOT, "Unexpected field '%s' found", field));
         }
-      } catch (AssertionError e) {
-        throw new AssertionError(rsp + " + " + client + " => " + e.getMessage(), e);
       }
     }
   }
 
-  /**
-   * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed at a node
-   * in our cluster.
-   */
-  private static SolrClient getRandClient() {
-    return CLIENTS.get(random().nextInt(CLIENTS.size()));
+  protected boolean assertGroupOrder(Tuple tuple, int... ids) throws Exception {
+    List group = (List) tuple.get("tuples");
+    int i = 0;
+    for (int val : ids) {
+      Map t = (Map) group.get(i);
+      Long tip = (Long) t.get("id");
+      if (tip.intValue() != val) {
+        throw new Exception("Found value:" + tip.intValue() + " expecting:" + val);
+      }
+      ++i;
+    }
+    return true;
   }
 
-  private static String readLastLineOfFile(File file) throws IOException {
-    try (ReversedLinesFileReader reader =
-        ReversedLinesFileReader.builder().setFile(file).setCharset(StandardCharsets.UTF_8).get()) {
-      return reader.readLine();
+  private static SolrCore findSolrCore() {
+    for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
+      for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
+        if (solrCore != null) {
+          return solrCore;
+        }
+      }
     }
+    throw new RuntimeException("Didn't find any valid cores.");
   }
 }

From 231249ee9813bd7dd151b74b663a499bfa19406b Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 18 Nov 2024 09:29:14 -0500
Subject: [PATCH 074/103] Add License

---
 .../UBIComponentStreamingQueriesTest.java        | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index 5ffac3bcd22..a82e9bbf94d 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.solr.handler.component;
 
 import com.fasterxml.jackson.databind.ObjectMapper;

From 471b5b68e1b3b3a37296f64bf6b321ea8473e89e Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 18 Nov 2024 13:27:42 -0500
Subject: [PATCH 075/103] Lint

---
 .../solr/handler/component/UBIComponentRecordingTest.java     | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
index d32483fbc20..f4e6221dc67 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
@@ -91,8 +91,6 @@ public void testRecordingUBIQueries() throws Exception {
     final QueryResponse queryResponse = cluster.getSolrClient().query(COLLECTION, queryParams);
     final SolrDocumentList documents = queryResponse.getResults();
 
-
-
     //
     //
     //    final ModifiableSolrParams overrideParams = new ModifiableSolrParams();
@@ -104,6 +102,6 @@ public void testRecordingUBIQueries() throws Exception {
     //            .setLimit(0);
     //    QueryResponse queryResponse = req.process(cluster.getSolrClient(), COLLECTION);
     // assertResponseFoundNumDocs(queryResponse, expectedResults);
-    //System.out.println(queryResponse);
+    // System.out.println(queryResponse);
   }
 }

From bf15e5bfcfdc91f83a05c333ee1ed4408cf6b195 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 18 Nov 2024 13:30:09 -0500
Subject: [PATCH 076/103] Comment out this test to see if we can get a clean
 run.

---
 solr/packaging/test/{test_stream.bats => test_stream.bat.off} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename solr/packaging/test/{test_stream.bats => test_stream.bat.off} (100%)

diff --git a/solr/packaging/test/test_stream.bats b/solr/packaging/test/test_stream.bat.off
similarity index 100%
rename from solr/packaging/test/test_stream.bats
rename to solr/packaging/test/test_stream.bat.off

From 99d6b7a7eb7b28a92f4cb36d4a525f8b901ba93c Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 18 Nov 2024 14:04:45 -0500
Subject: [PATCH 077/103] Introduce timestamp to tracking

---
 .../solr/handler/component/UBIComponent.java  | 13 ++++++-----
 .../solr/handler/component/UBIQuery.java      | 12 ++++++++++
 .../ubi-enabled/conf/ubi-query-pipeline.expr  | 11 +++------
 .../UBIComponentStreamingQueriesTest.java     | 23 ++++++++++++-------
 4 files changed, 37 insertions(+), 22 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index ef644fecad3..5790cf480a5 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -110,9 +110,9 @@
  *      ]
  *     params: {
  *       "ubi": "true"
- *       "query_id": "xyz890",
- *       "user_query": {
- *         "query": "Apple iPod",
+ *       "user_query": "Apple iPod",
+ *       "query_attributes": {
+ *         "experiment_name": "super_secret",
  *         "page": 2,
  *         "in_stock": "true"
  *       }
@@ -120,9 +120,10 @@
  *   }
  * 
  *
- * Notice that we are enabling UBI query tracking, we are providing an explicit query_id and passing
- * in the user's specific choices for querying. The user_query parameters are not specific to Solr
- * syntax, they are defined by the creator of the search request.
+ * 
Refer to 
+ * https://solr.apache.org/guide/solr/latest/query-guide/user-behavior-insights.html for more
+ * details
  */
 public class UBIComponent extends SearchComponent implements SolrCoreAware {
 
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
index 8605142029c..fceeb550169 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
@@ -18,6 +18,9 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -34,6 +37,7 @@ public class UBIQuery {
   private String application;
   private String queryId;
   private String userQuery;
+  private Date timestamp;
 
   @SuppressWarnings("rawtypes")
   private Map queryAttributes;
@@ -46,6 +50,11 @@ public UBIQuery(String queryId) {
       queryId = UUID.randomUUID().toString().toLowerCase(Locale.ROOT);
     }
     this.queryId = queryId;
+    this.timestamp = new Date();
+  }
+
+  public Date getTimestamp() {
+    return timestamp;
   }
 
   public void setApplication(String application) {
@@ -95,6 +104,9 @@ public Map toMap() {
     @SuppressWarnings({"rawtypes", "unchecked"})
     Map map = new HashMap();
     map.put(UBIComponent.QUERY_ID, this.queryId);
+    map.put(
+        "timestamp",
+        DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(this.timestamp.getTime())));
     if (this.application != null) {
       map.put(UBIComponent.APPLICATION, this.application);
     }
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
index ca9b2d9797c..6e09615240a 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/ubi-query-pipeline.expr
@@ -18,15 +18,10 @@ commit(ubi_queries,
     select(
       ubiQuery(),
       query_id as id,
+      timestamp,
       application,
-      user_query
+      user_query,
+      query_attributes
     )
   )
 )
-
-#let(cli-zkhost="localhost:9983",    
-#   commit(ubi,
-#    update(ubi,
-#           tuple(id=49,a_i=1,b_i=5))
-#           )
-#)
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index a82e9bbf94d..adf390b1ab0 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -115,8 +116,11 @@ public void testUBIQueryStream() throws Exception {
       tuples = getTuples(stream);
 
       assertEquals(1, tuples.size());
-      assertFields(tuples, "query_id");
+      assertFields(tuples, "query_id", "timestamp");
       assertString(tuples.get(0), "query_id", "123");
+
+      assertNotNull(Instant.parse(tuples.get(0).getString("timestamp")));
+
       // assertNotFields(tuples, "user_query", "event_attributes");
 
       // Include another field to see what is returned
@@ -129,7 +133,7 @@ public void testUBIQueryStream() throws Exception {
       tuples = getTuples(stream);
 
       assertEquals(1, tuples.size());
-      assertFields(tuples, "query_id", "application");
+      assertFields(tuples, "query_id", "timestamp", "application");
       assertString(tuples.get(0), "query_id", "234");
       assertString(tuples.get(0), "application", "typeahead");
 
@@ -148,7 +152,7 @@ public void testUBIQueryStream() throws Exception {
       tuples = getTuples(stream);
 
       assertEquals(1, tuples.size());
-      assertFields(tuples, "query_id", "query_attributes");
+      assertFields(tuples, "query_id", "timestamp", "query_attributes");
       assertString(tuples.get(0), "query_id", "345");
       assertString(tuples.get(0), "query_attributes", "{\"attribute1\":\"one\",\"attribute2\":2}");
     }
@@ -204,13 +208,13 @@ public void testWritingToLogUbiQueryStream() throws Exception {
         assertNotNull(jsonLine);
         ObjectMapper objectMapper = new ObjectMapper();
         @SuppressWarnings({"unchecked", "rawtypes"})
-        Map myObject = objectMapper.readValue(jsonLine, Map.class);
-        assertEquals(ubiQuery.getQueryId(), myObject.get("query_id"));
-        assertEquals(ubiQuery.getApplication(), myObject.get("application"));
-        // assertEquals(ubiQuery.getQueryAttributes(), myObject.get("query_attributes"));
+        Map ubiQueryAsMap = objectMapper.readValue(jsonLine, Map.class);
+        assertEquals(ubiQuery.getQueryId(), ubiQueryAsMap.get("query_id"));
+        assertEquals(ubiQuery.getApplication(), ubiQueryAsMap.get("application"));
+        assertNotNull(ubiQueryAsMap.get("timestamp"));
         assertEquals(
             "{\"experiment\":\"secret\",\"marginBoost\":2.1,\"parsed_query\":\"memory OR ram\"}",
-            myObject.get("query_attributes"));
+            ubiQueryAsMap.get("query_attributes"));
       }
     }
   }
@@ -255,6 +259,7 @@ public void testWritingToSolrUbiQueryStream() throws Exception {
                   + ", batchSize=5, select(\n"
                   + "      ubiQuery(),\n"
                   + "      query_id as id,\n"
+                  + "      timestamp,\n"
                   + "      application,\n"
                   + "      user_query,\n"
                   + "      query_attributes\n"
@@ -284,6 +289,8 @@ public void testWritingToSolrUbiQueryStream() throws Exception {
       Tuple tuple = tuples.get(0);
       assertEquals(ubiQuery.getQueryId(), tuple.get("id"));
       assertEquals(ubiQuery.getApplication(), tuple.get("application"));
+      assertEquals(ubiQuery.getUserQuery(), tuple.get("user_query"));
+      assertEquals(ubiQuery.getTimestamp(), tuple.getDate("timestamp"));
       assertEquals(
           "{\"experiment\":\"secret\",\"marginBoost\":2.1,\"parsed_query\":\"memory OR ram\"}",
           tuple.get("query_attributes"));

From 7d6697934d69510b1e83fb94598934b1078a975b Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 18 Nov 2024 14:24:06 -0500
Subject: [PATCH 078/103] Lint

---
 .../handler/component/UBIComponentStreamingQueriesTest.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index adf390b1ab0..24efdd4d1da 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -290,7 +290,7 @@ public void testWritingToSolrUbiQueryStream() throws Exception {
       assertEquals(ubiQuery.getQueryId(), tuple.get("id"));
       assertEquals(ubiQuery.getApplication(), tuple.get("application"));
       assertEquals(ubiQuery.getUserQuery(), tuple.get("user_query"));
-      assertEquals(ubiQuery.getTimestamp(), tuple.getDate("timestamp"));
+      assertEquals(ubiQuery.getTimestamp().toInstant(), tuple.getDate("timestamp").toInstant());
       assertEquals(
           "{\"experiment\":\"secret\",\"marginBoost\":2.1,\"parsed_query\":\"memory OR ram\"}",
           tuple.get("query_attributes"));

From 79c9d750ad9f1172eed22429e48aba6a8585ee9c Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 18 Nov 2024 15:01:46 -0500
Subject: [PATCH 079/103] Make test pass

---
 gradle/testing/randomization/policies/solr-tests.policy | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy
index 61df0871a35..65161dd16c9 100644
--- a/gradle/testing/randomization/policies/solr-tests.policy
+++ b/gradle/testing/randomization/policies/solr-tests.policy
@@ -37,6 +37,9 @@ grant {
   permission java.io.FilePermission "/path/to/myinst/conf/solrcore.properties", "read";
   // TestConfigSets messes with these (wtf?)
   permission java.io.FilePermission "/path/to/solr/home/lib", "read";
+  
+  // Needed by org.apache.solr.handler.component.UBIComponentTest
+  permission java.io.FilePermission "${common-solr.dir}/core/build/resources/test/solr/userfiles/ubi_queries.jsonl", "write";
 
   permission java.nio.file.LinkPermission "hard";
 

From a5a1f193f002f7a9efa4bae7a15a2cdfac265d0f Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Tue, 19 Nov 2024 05:15:38 -0500
Subject: [PATCH 080/103] Another write permission...

---
 gradle/testing/randomization/policies/solr-tests.policy | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy
index 65161dd16c9..afec809a7f7 100644
--- a/gradle/testing/randomization/policies/solr-tests.policy
+++ b/gradle/testing/randomization/policies/solr-tests.policy
@@ -40,6 +40,7 @@ grant {
   
   // Needed by org.apache.solr.handler.component.UBIComponentTest
   permission java.io.FilePermission "${common-solr.dir}/core/build/resources/test/solr/userfiles/ubi_queries.jsonl", "write";
+  permission java.io.FilePermission "/tmp/src/solr/solr/core/build/resources/test/solr/userfile", "write";
 
   permission java.nio.file.LinkPermission "hard";
 

From d8904ff38d846ee9a5d478a6c89102f1a88c9170 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Tue, 19 Nov 2024 10:24:41 -0500
Subject: [PATCH 081/103] userfile(s) you nimrod!

---
 gradle/testing/randomization/policies/solr-tests.policy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy
index afec809a7f7..eada6893445 100644
--- a/gradle/testing/randomization/policies/solr-tests.policy
+++ b/gradle/testing/randomization/policies/solr-tests.policy
@@ -40,7 +40,7 @@ grant {
   
   // Needed by org.apache.solr.handler.component.UBIComponentTest
   permission java.io.FilePermission "${common-solr.dir}/core/build/resources/test/solr/userfiles/ubi_queries.jsonl", "write";
-  permission java.io.FilePermission "/tmp/src/solr/solr/core/build/resources/test/solr/userfile", "write";
+  permission java.io.FilePermission "/tmp/src/solr/solr/core/build/resources/test/solr/userfiles${/}-", "write";
 
   permission java.nio.file.LinkPermission "hard";
 

From f25228909c39e5489bf484bed0f8ad07110ebe1c Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 20 Nov 2024 08:29:56 -0500
Subject: [PATCH 082/103] Back out disabling, we have a fix!

---
 solr/packaging/test/{test_stream.bat.off => test_stream.bat} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename solr/packaging/test/{test_stream.bat.off => test_stream.bat} (100%)

diff --git a/solr/packaging/test/test_stream.bat.off b/solr/packaging/test/test_stream.bat
similarity index 100%
rename from solr/packaging/test/test_stream.bat.off
rename to solr/packaging/test/test_stream.bat

From 00f8c494dc73dccaafcdab38eb51c8084b5d5b95 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Sun, 24 Nov 2024 07:41:30 -0500
Subject: [PATCH 083/103] We no longer drop the examples into specific sub dir

---
 solr/packaging/test/test_ubi.bats | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/solr/packaging/test/test_ubi.bats b/solr/packaging/test/test_ubi.bats
index a7b2df1f73e..fcc6bdfcac1 100644
--- a/solr/packaging/test/test_ubi.bats
+++ b/solr/packaging/test/test_ubi.bats
@@ -60,8 +60,8 @@ teardown() {
   assert_output --partial '"query_id":"5678'
    
   # Check UBI query record was written out to default location
-  assert_file_exist ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl
-  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"query_id":"5678"'
+  assert_file_exist ${SOLR_TIP}/server/solr/userfiles/ubi_queries.jsonl
+  assert_file_contains ${SOLR_TIP}/server/solr/userfiles/ubi_queries.jsonl '"query_id":"5678"'
   
   # Rich UBI user query tracking enabled query with JSON Query
   run curl -X POST -H 'Content-type:application/json' -d '{
@@ -85,8 +85,8 @@ teardown() {
   assert_output --partial '"query_id":"xyz890"'
   
   # Check UBI query record was written out to default location with additional metadata
-  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl '"query_id":"xyz890"'
-  assert_file_contains ${SOLR_TIP}/example/techproducts/solr/userfiles/ubi_queries.jsonl 'supersecret'
+  assert_file_contains ${SOLR_TIP}/server/solr/userfiles/ubi_queries.jsonl '"query_id":"xyz890"'
+  assert_file_contains ${SOLR_TIP}/server/solr/userfiles/ubi_queries.jsonl 'supersecret'
   
   
 }

From 7c32dac2692298826171c692c2564e0f71474eca Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Tue, 26 Nov 2024 19:53:36 -0500
Subject: [PATCH 084/103] Map.of () limited to nine elements, so use
 Map.ofEntries.

refer to the standard components using more normal pattern.
---
 .../java/org/apache/solr/core/SolrCore.java   |  2 +-
 .../handler/component/SearchComponent.java    | 27 ++++++++++---------
 .../solr/handler/component/SearchHandler.java | 14 +---------
 3 files changed, 16 insertions(+), 27 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 749436a0e8c..13d166cf449 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -1983,7 +1983,7 @@ public SolrRequestHandler registerRequestHandler(String handlerName, SolrRequest
 
   /** Register the default search components */
   private void loadSearchComponents() {
-    Map instances = createInstances(SearchComponent.standard_components);
+    Map instances = createInstances(SearchComponent.STANDARD_COMPONENTS);
     for (Map.Entry e : instances.entrySet())
       e.getValue().setName(e.getKey());
     searchComponents.init(instances, this);
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
index 913078c27ee..e0beeb83ede 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
@@ -105,20 +105,21 @@ public SolrMetricsContext getSolrMetricsContext() {
 
   @Override
   public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
-    // By default don't register any metrics - but prepare a child context
+    // By default, don't register any metrics - but prepare a child context
     this.solrMetricsContext = parentContext.getChildContext(this);
   }
 
-  public static final Map> standard_components =
-      Map.of(
-          HighlightComponent.COMPONENT_NAME, HighlightComponent.class,
-          QueryComponent.COMPONENT_NAME, QueryComponent.class,
-          FacetComponent.COMPONENT_NAME, FacetComponent.class,
-          FacetModule.COMPONENT_NAME, FacetModule.class,
-          MoreLikeThisComponent.COMPONENT_NAME, MoreLikeThisComponent.class,
-          StatsComponent.COMPONENT_NAME, StatsComponent.class,
-          DebugComponent.COMPONENT_NAME, DebugComponent.class,
-          RealTimeGetComponent.COMPONENT_NAME, RealTimeGetComponent.class,
-          ExpandComponent.COMPONENT_NAME, ExpandComponent.class,
-          TermsComponent.COMPONENT_NAME, TermsComponent.class);
+  public static final Map> STANDARD_COMPONENTS =
+      Map.ofEntries(
+          Map.entry(HighlightComponent.COMPONENT_NAME, HighlightComponent.class),
+          Map.entry(QueryComponent.COMPONENT_NAME, QueryComponent.class),
+          Map.entry(FacetComponent.COMPONENT_NAME, FacetComponent.class),
+          Map.entry(FacetModule.COMPONENT_NAME, FacetModule.class),
+          Map.entry(MoreLikeThisComponent.COMPONENT_NAME, MoreLikeThisComponent.class),
+          Map.entry(StatsComponent.COMPONENT_NAME, StatsComponent.class),
+          Map.entry(DebugComponent.COMPONENT_NAME, DebugComponent.class),
+          Map.entry(RealTimeGetComponent.COMPONENT_NAME, RealTimeGetComponent.class),
+          Map.entry(ExpandComponent.COMPONENT_NAME, ExpandComponent.class),
+          Map.entry(TermsComponent.COMPONENT_NAME, TermsComponent.class),
+          Map.entry(UBIComponent.COMPONENT_NAME, UBIComponent.class));
 }
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index 2d6b36ac3cb..661b509bf93 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -71,7 +71,6 @@
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.search.CursorMark;
 import org.apache.solr.search.SortSpec;
-import org.apache.solr.search.facet.FacetModule;
 import org.apache.solr.security.AuthorizationContext;
 import org.apache.solr.security.PermissionNameProvider;
 import org.apache.solr.util.RTimerTree;
@@ -127,18 +126,7 @@ public class SearchHandler extends RequestHandlerBase
   private SolrCore core;
 
   protected List getDefaultComponents() {
-    ArrayList names = new ArrayList<>(9);
-    names.add(QueryComponent.COMPONENT_NAME);
-    names.add(FacetComponent.COMPONENT_NAME);
-    names.add(FacetModule.COMPONENT_NAME);
-    names.add(MoreLikeThisComponent.COMPONENT_NAME);
-    names.add(HighlightComponent.COMPONENT_NAME);
-    names.add(StatsComponent.COMPONENT_NAME);
-    names.add(DebugComponent.COMPONENT_NAME);
-    names.add(ExpandComponent.COMPONENT_NAME);
-    names.add(TermsComponent.COMPONENT_NAME);
-
-    return names;
+    return SearchComponent.STANDARD_COMPONENTS.keySet().stream().toList();
   }
 
   @Override

From 8c77769e4f36ec7b3fb74bf5aa973e3e1e9e8ed8 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 27 Nov 2024 08:36:11 -0500
Subject: [PATCH 085/103] take some weight out of the variable name!

We are already in the UBI component!
---
 .../solr/handler/component/UBIComponent.java  | 20 ++++++++++---------
 .../ubi-enabled/conf/solrconfig.xml           |  2 +-
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 5790cf480a5..0c8b4d761e5 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -156,11 +156,14 @@ public void inform(SolrCore core) {
     SolrClientCache solrClientCache = coreContainer.getSolrClientCache();
 
     String expr;
-    String ubiQueryStreamProcessingExpression = initArgs.get("ubiQueryStreamProcessingExpression");
+    String queryProcessingExpression = null;
+    if (initArgs != null) {
+      queryProcessingExpression = initArgs.get("queryProcessingExpression");
+    }
 
-    if (ubiQueryStreamProcessingExpression == null) {
+    if (queryProcessingExpression == null) {
       log.info(
-          "No 'ubiQueryStreamProcessingExpression' file provided to describe processing of UBI query information.");
+          "No 'queryProcessingExpression' file provided to describe processing of UBI query information.");
       log.info(
           "Writing out UBI query information to local $SOLR_HOME/userfiles/ubi_queries.jsonl file instead.");
       // Most simplistic version
@@ -170,9 +173,8 @@ public void inform(SolrCore core) {
       expr = "logging(ubi_queries.jsonl,ubiQuery())";
 
       // feels like 'stream' or 'get' or something should let me create a tuple out of something
-      // in the
-      // streamContext.   That would turn the "ubi-query" object in the context into a nice
-      // tuple and return it.
+      // in the streamContext.   That would turn the "ubi-query" object in the stream context into a nice
+      // tuple and return it.  streamContext(ubi-query)??
       // expr = "logging(ubi_queries.jsonl," + "get(ubi-query)" + ")";
     } else {
 
@@ -180,13 +182,13 @@ public void inform(SolrCore core) {
       try (LineNumberReader bufferedReader =
           new LineNumberReader(
               new InputStreamReader(
-                  core.getResourceLoader().openResource(ubiQueryStreamProcessingExpression),
+                  core.getResourceLoader().openResource(queryProcessingExpression),
                   StandardCharsets.UTF_8))) {
         expr = readExpression(bufferedReader, args);
       } catch (IOException ioe) {
         throw new SolrException(
             SolrException.ErrorCode.SERVER_ERROR,
-            "Error reading file " + ubiQueryStreamProcessingExpression,
+            "Error reading file " + queryProcessingExpression,
             ioe);
       }
     }
@@ -374,6 +376,6 @@ private static TupleStream constructStream(
 
   @Override
   public String getDescription() {
-    return "A component that tracks the original user query and the resulting documents returned.";
+    return "A component that tracks the original user query and the resulting documents returned to understand the user.";
   }
 }
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml
index 498b1b8a234..ff613fc0da5 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/solrconfig.xml
@@ -36,7 +36,7 @@
   
   
   
-    ubi-query-pipeline.expr
+    ubi-query-pipeline.expr
   
   
   

From 83842e973753ed0e50bb775b3f206d747b4e98ac Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 27 Nov 2024 08:56:22 -0500
Subject: [PATCH 086/103] Track the collection name if an application isn't
 explicitly provided as we interleave data otherwise.

---
 .../org/apache/solr/handler/component/UBIComponent.java     | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 0c8b4d761e5..f13306076d4 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -173,7 +173,8 @@ public void inform(SolrCore core) {
       expr = "logging(ubi_queries.jsonl,ubiQuery())";
 
       // feels like 'stream' or 'get' or something should let me create a tuple out of something
-      // in the streamContext.   That would turn the "ubi-query" object in the stream context into a nice
+      // in the streamContext.   That would turn the "ubi-query" object in the stream context into a
+      // nice
       // tuple and return it.  streamContext(ubi-query)??
       // expr = "logging(ubi_queries.jsonl," + "get(ubi-query)" + ")";
     } else {
@@ -232,6 +233,9 @@ public void process(ResponseBuilder rb) throws IOException {
 
     ubiQuery.setUserQuery(params.get(USER_QUERY));
     ubiQuery.setApplication(params.get(APPLICATION));
+    if (ubiQuery.getApplication() == null){
+      ubiQuery.setApplication(rb.req.getCloudDescriptor().getCollectionName());
+    }
 
     String queryAttributes = params.get(QUERY_ATTRIBUTES);
 

From c128c4aa2d39924d8c015caaeef84ce5cea6820f Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 27 Nov 2024 12:54:25 -0500
Subject: [PATCH 087/103] Need to ensure that Query is always the first
 component.

The fact that we are calling .keySet may be a problem...  Because that means other components might be in a random order?  Maybe we shouldn't even use a map of string/class, it should just be a list of classes?
---
 .../solr/handler/component/SearchHandler.java  | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index 661b509bf93..467286452e0 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -125,8 +125,24 @@ public class SearchHandler extends RequestHandlerBase
   private PluginInfo shfInfo;
   private SolrCore core;
 
+  /**
+   * The default set of components that every handler gets. You can change this by defining the
+   * specific components for a handler. It puts the {@link QueryComponent} first as subsequent
+   * components assume that the QueryComponent ran and populated the document list.
+   *
+   * @return A list of component names.
+   */
   protected List getDefaultComponents() {
-    return SearchComponent.STANDARD_COMPONENTS.keySet().stream().toList();
+    List l = new ArrayList(SearchComponent.STANDARD_COMPONENTS.keySet());
+    moveToFirst(l, QueryComponent.COMPONENT_NAME);
+    return l;
+  }
+
+  private static void moveToFirst(List list, String target) {
+    int index = list.indexOf(target);
+    assert index != -1;
+    list.remove(index);
+    list.add(0, target);
   }
 
   @Override

From 87cbfbee24b12a6fe03261e7cb8487b146a7e219 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Wed, 27 Nov 2024 12:54:36 -0500
Subject: [PATCH 088/103] typo

---
 .../org/apache/solr/handler/component/HighlightComponent.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
index d05c44ef539..1caa82033b5 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java
@@ -197,7 +197,7 @@ public SolrHighlighter getHighlighter(SolrParams params) {
   public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
     if (!rb.doHighlights) return;
 
-    // Turn on highlighting only only when retrieving fields
+    // Turn on highlighting only when retrieving fields
     if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
       sreq.purpose |= ShardRequest.PURPOSE_GET_HIGHLIGHTS;
       // should already be true...

From e63d125a94a9af0cfa8bd6f8b1b2e20457f1a714 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Thu, 28 Nov 2024 07:21:37 -0500
Subject: [PATCH 089/103] Test is not testing anything.

---
 .../component/UBIComponentRecordingTest.java  | 107 ------------------
 1 file changed, 107 deletions(-)
 delete mode 100644 solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java

diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
deleted file mode 100644
index f4e6221dc67..00000000000
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentRecordingTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.component;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.params.MapSolrParams;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * This test demonstrates that the backend recording of events via Streaming Expressions functions.
- */
-public class UBIComponentRecordingTest extends SolrCloudTestCase {
-
-  private static final String COLLECTION = "collection1"; // The source of ubi=true enabled queries
-  private static final String UBI_QUERIES_COLLECTION = "ubi_queries"; // where we store query data
-
-  @BeforeClass
-  public static void setupCluster() throws Exception {
-    configureCluster(1)
-        .addConfig(
-            "ubi-enabled-config",
-            TEST_PATH().resolve("configsets").resolve("ubi-enabled").resolve("conf"))
-        .addConfig(
-            "minimal-config",
-            TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
-        .configure();
-  }
-
-  @AfterClass
-  public static void afterTest() throws Exception {
-    CollectionAdminRequest.deleteCollection(COLLECTION).process(cluster.getSolrClient());
-    CollectionAdminRequest.deleteCollection(UBI_QUERIES_COLLECTION)
-        .process(cluster.getSolrClient());
-  }
-
-  @Test
-  public void testRecordingUBIQueries() throws Exception {
-
-    // Create the collection that will generate UBI events
-    assertEquals(
-        "failed to create collection " + COLLECTION,
-        0,
-        CollectionAdminRequest.createCollection(COLLECTION, "ubi-enabled-config", 2, 1, 1, 0)
-            .process(cluster.getSolrClient())
-            .getStatus());
-
-    // Create the collection that will recieve UBI events through the ubi-query-pipeline.expr
-    // streaming expression
-    assertEquals(
-        "failed to create collection " + UBI_QUERIES_COLLECTION,
-        0,
-        CollectionAdminRequest.createCollection(
-                UBI_QUERIES_COLLECTION, "minimal-config", 2, 1, 1, 0)
-            .process(cluster.getSolrClient())
-            .getStatus());
-
-    cluster.waitForActiveCollection(COLLECTION, 2, 2 * (1 + 1));
-    cluster.waitForActiveCollection(UBI_QUERIES_COLLECTION, 2, 2 * (1 + 1));
-
-    // TODO why doens't this work?
-    //    assertQ(
-    //            "Make sure we generate a query id",
-    //            req("q", "aa", "rows", "2", "ubi", "true"),
-    //            "count(//lst[@name='ubi']/str[@name='query_id'])=1");
-
-    final Map queryParamMap = new HashMap<>();
-    queryParamMap.put("q", "*:*");
-    queryParamMap.put("ubi", "true");
-    MapSolrParams queryParams = new MapSolrParams(queryParamMap);
-    final QueryResponse queryResponse = cluster.getSolrClient().query(COLLECTION, queryParams);
-    final SolrDocumentList documents = queryResponse.getResults();
-
-    //
-    //
-    //    final ModifiableSolrParams overrideParams = new ModifiableSolrParams();
-    //    overrideParams.set("ubi", true);
-    //    final JsonQueryRequest req =
-    //        new JsonQueryRequest(overrideParams)
-    //            .setQuery("*:*")
-    //            // .setUBI(true)
-    //            .setLimit(0);
-    //    QueryResponse queryResponse = req.process(cluster.getSolrClient(), COLLECTION);
-    // assertResponseFoundNumDocs(queryResponse, expectedResults);
-    // System.out.println(queryResponse);
-  }
-}

From 3393d6e40bae0ccda93c504eff9991ce9ff6637c Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Thu, 28 Nov 2024 07:54:29 -0500
Subject: [PATCH 090/103] QueryComponent get's forced to first, so list it
 first.

---
 .../java/org/apache/solr/handler/component/SearchComponent.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
index e0beeb83ede..dc845e1465b 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
@@ -111,8 +111,8 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
 
   public static final Map> STANDARD_COMPONENTS =
       Map.ofEntries(
-          Map.entry(HighlightComponent.COMPONENT_NAME, HighlightComponent.class),
           Map.entry(QueryComponent.COMPONENT_NAME, QueryComponent.class),
+          Map.entry(HighlightComponent.COMPONENT_NAME, HighlightComponent.class),
           Map.entry(FacetComponent.COMPONENT_NAME, FacetComponent.class),
           Map.entry(FacetModule.COMPONENT_NAME, FacetModule.class),
           Map.entry(MoreLikeThisComponent.COMPONENT_NAME, MoreLikeThisComponent.class),

From 92d7f67ba9cb6ccc6bfa454a2f260e80429af924 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Thu, 28 Nov 2024 07:54:43 -0500
Subject: [PATCH 091/103] Added more debugging, but it's not helping.

---
 .../solr/handler/component/UBIComponent.java  | 69 +++++++++++++++++--
 1 file changed, 63 insertions(+), 6 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index f13306076d4..4d591509975 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -42,7 +42,6 @@
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.handler.LoggingStream;
-import org.apache.solr.response.ResultContext;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
@@ -217,15 +216,69 @@ public void inform(SolrCore core) {
   }
 
   @Override
-  public void prepare(ResponseBuilder rb) throws IOException {}
+  public void prepare(ResponseBuilder rb) throws IOException {
+    SolrParams params = rb.req.getParams();
+    if (!params.getBool(COMPONENT_NAME, false)) {
+      return;
+    }
+    rb.setNeedDocList(true);
+  }
 
   @Override
   public void process(ResponseBuilder rb) throws IOException {
+    System.out.println("PROCESSS STAGE: " + rb.stage);
     SolrParams params = rb.req.getParams();
     if (!params.getBool(COMPONENT_NAME, false)) {
       return;
     }
 
+    doStuff(rb);
+  }
+
+  @Override
+  public int distributedProcess(ResponseBuilder rb) throws IOException {
+    System.out.println("STAGE: " + rb.stage);
+    System.out.println("STAGE rb.getResults(): " + (rb.getResults() != null));
+    System.out.println("getResponseDocs.size" + rb.getResponseDocs().size());
+
+    SolrParams params = rb.req.getParams();
+    if (!params.getBool(COMPONENT_NAME, false)) {
+      return ResponseBuilder.STAGE_DONE;
+    }
+
+    if (rb.stage != ResponseBuilder.STAGE_GET_FIELDS) {
+      return ResponseBuilder.STAGE_DONE;
+    }
+
+    doStuff(rb);
+
+    return ResponseBuilder.STAGE_DONE;
+  }
+
+  //  @Override
+  //  public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
+  //    SolrParams params = rb.req.getParams();
+  //    // rb.setNeedDocList(true);
+  //    if (!params.getBool(COMPONENT_NAME, false)) {
+  //      return;
+  //    }
+  //
+  //    // Turn on UBI only when retrieving fields
+  //    if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
+  //      // should already be true...
+  //      sreq.params.set("ubi", "true");
+  //    } else {
+  //      sreq.params.set("ubi", "false");
+  //    }
+  //  }
+
+  public void doStuff(ResponseBuilder rb) throws IOException {
+
+    SolrParams params = rb.req.getParams();
+    // if (!params.getBool(COMPONENT_NAME, false)) {
+    //  return;
+    // }
+
     SolrIndexSearcher searcher = rb.req.getSearcher();
 
     String queryId = params.get(QUERY_ID);
@@ -233,8 +286,11 @@ public void process(ResponseBuilder rb) throws IOException {
 
     ubiQuery.setUserQuery(params.get(USER_QUERY));
     ubiQuery.setApplication(params.get(APPLICATION));
-    if (ubiQuery.getApplication() == null){
-      ubiQuery.setApplication(rb.req.getCloudDescriptor().getCollectionName());
+    if (ubiQuery.getApplication() == null) {
+      ubiQuery.setApplication(
+          rb.isDistrib
+              ? rb.req.getCloudDescriptor().getCollectionName()
+              : searcher.getCore().getName());
     }
 
     String queryAttributes = params.get(QUERY_ATTRIBUTES);
@@ -255,8 +311,9 @@ public void process(ResponseBuilder rb) throws IOException {
       }
     }
 
-    ResultContext rc = (ResultContext) rb.rsp.getResponse();
-    DocList docs = rc.getDocList();
+    // ResultContext rc = (ResultContext) rb.rsp.getResponse();
+    // DocList docs = rc.getDocList();
+    DocList docs = rb.getResults().docList;
 
     String docIds = extractDocIds(docs, searcher);
     ubiQuery.setDocIds(docIds);

From c22f9cbd232075ac4f68b88fb536e577a492e931 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Fri, 29 Nov 2024 15:33:16 -0500
Subject: [PATCH 092/103] back out changes

---
 .../solr/handler/component/UBIComponent.java  | 36 +++++--------------
 .../UBIComponentStreamingQueriesTest.java     | 20 -----------
 2 files changed, 9 insertions(+), 47 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 4d591509975..6c2b4b78594 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -42,6 +42,7 @@
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.handler.LoggingStream;
+import org.apache.solr.response.ResultContext;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
@@ -226,7 +227,6 @@ public void prepare(ResponseBuilder rb) throws IOException {
 
   @Override
   public void process(ResponseBuilder rb) throws IOException {
-    System.out.println("PROCESSS STAGE: " + rb.stage);
     SolrParams params = rb.req.getParams();
     if (!params.getBool(COMPONENT_NAME, false)) {
       return;
@@ -237,9 +237,6 @@ public void process(ResponseBuilder rb) throws IOException {
 
   @Override
   public int distributedProcess(ResponseBuilder rb) throws IOException {
-    System.out.println("STAGE: " + rb.stage);
-    System.out.println("STAGE rb.getResults(): " + (rb.getResults() != null));
-    System.out.println("getResponseDocs.size" + rb.getResponseDocs().size());
 
     SolrParams params = rb.req.getParams();
     if (!params.getBool(COMPONENT_NAME, false)) {
@@ -255,29 +252,14 @@ public int distributedProcess(ResponseBuilder rb) throws IOException {
     return ResponseBuilder.STAGE_DONE;
   }
 
-  //  @Override
-  //  public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
-  //    SolrParams params = rb.req.getParams();
-  //    // rb.setNeedDocList(true);
-  //    if (!params.getBool(COMPONENT_NAME, false)) {
-  //      return;
-  //    }
-  //
-  //    // Turn on UBI only when retrieving fields
-  //    if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
-  //      // should already be true...
-  //      sreq.params.set("ubi", "true");
-  //    } else {
-  //      sreq.params.set("ubi", "false");
-  //    }
-  //  }
-
   public void doStuff(ResponseBuilder rb) throws IOException {
 
+    // not sure why but sometimes we get it twoice...  how can a response have the
+    // the same component run twice?
+    if (rb.rsp.getValues().get("ubi") != null) {
+      return;
+    }
     SolrParams params = rb.req.getParams();
-    // if (!params.getBool(COMPONENT_NAME, false)) {
-    //  return;
-    // }
 
     SolrIndexSearcher searcher = rb.req.getSearcher();
 
@@ -311,9 +293,9 @@ public void doStuff(ResponseBuilder rb) throws IOException {
       }
     }
 
-    // ResultContext rc = (ResultContext) rb.rsp.getResponse();
-    // DocList docs = rc.getDocList();
-    DocList docs = rb.getResults().docList;
+    ResultContext rc = (ResultContext) rb.rsp.getResponse();
+    DocList docs = rc.getDocList();
+    // DocList docs = rb.getResults().docList;
 
     String docIds = extractDocIds(docs, searcher);
     ubiQuery.setDocIds(docIds);
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index 24efdd4d1da..89d1a8ada5c 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -121,8 +121,6 @@ public void testUBIQueryStream() throws Exception {
 
       assertNotNull(Instant.parse(tuples.get(0).getString("timestamp")));
 
-      // assertNotFields(tuples, "user_query", "event_attributes");
-
       // Include another field to see what is returned
       ubiQuery = new UBIQuery("234");
       ubiQuery.setApplication("typeahead");
@@ -309,10 +307,6 @@ protected List getTuples(TupleStream tupleStream) throws IOException {
     return tuples;
   }
 
-  protected void assertOrder(List tuples, int... ids) throws Exception {
-    assertOrderOf(tuples, "id", ids);
-  }
-
   protected void assertOrderOf(List tuples, String fieldName, int... ids) throws Exception {
     int i = 0;
     for (int val : ids) {
@@ -364,20 +358,6 @@ protected void assertNotFields(List tuples, String... fields) throws Exce
     }
   }
 
-  protected boolean assertGroupOrder(Tuple tuple, int... ids) throws Exception {
-    List group = (List) tuple.get("tuples");
-    int i = 0;
-    for (int val : ids) {
-      Map t = (Map) group.get(i);
-      Long tip = (Long) t.get("id");
-      if (tip.intValue() != val) {
-        throw new Exception("Found value:" + tip.intValue() + " expecting:" + val);
-      }
-      ++i;
-    }
-    return true;
-  }
-
   private static SolrCore findSolrCore() {
     for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
       for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {

From 49ddee47e4d076ed85708e45d13e7bfad1018375 Mon Sep 17 00:00:00 2001
From: Mikhail Khludnev 
Date: Sat, 7 Dec 2024 00:17:00 +0300
Subject: [PATCH 093/103] UBI goes distrib

Pardon, I barely understand what's going on there.
---
 .../solr/handler/component/UBIComponent.java  |  50 +++++---
 .../UBIComponentDistrQueriesTest.java         | 110 ++++++++++++++++++
 2 files changed, 147 insertions(+), 13 deletions(-)
 create mode 100644 solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 6c2b4b78594..4279a30e322 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -237,27 +237,44 @@ public void process(ResponseBuilder rb) throws IOException {
 
   @Override
   public int distributedProcess(ResponseBuilder rb) throws IOException {
-
     SolrParams params = rb.req.getParams();
     if (!params.getBool(COMPONENT_NAME, false)) {
       return ResponseBuilder.STAGE_DONE;
     }
 
-    if (rb.stage != ResponseBuilder.STAGE_GET_FIELDS) {
-      return ResponseBuilder.STAGE_DONE;
+    if (rb.stage < ResponseBuilder.STAGE_GET_FIELDS) {
+      return ResponseBuilder.STAGE_GET_FIELDS;
     }
 
-    doStuff(rb);
+    if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+      doDistribStuff(rb);
+      return ResponseBuilder.STAGE_DONE;
+    }
 
     return ResponseBuilder.STAGE_DONE;
   }
 
   public void doStuff(ResponseBuilder rb) throws IOException {
+    UBIQuery ubiQuery = getUbiQuery(rb);
+    if (ubiQuery == null) return;
+
+    ResultContext rc = (ResultContext) rb.rsp.getResponse();
+    DocList docs = rc.getDocList();
+    // DocList docs = rb.getResults().docList;
+
+    String docIds = extractDocIds(docs, rb.req.getSearcher());
+
+    ubiQuery.setDocIds(docIds);
 
-    // not sure why but sometimes we get it twoice...  how can a response have the
+    addUserBehaviorInsightsToResponse(ubiQuery, rb);
+    recordQuery(ubiQuery);
+  }
+
+  private static UBIQuery getUbiQuery(ResponseBuilder rb) {
+    // not sure why but sometimes we get it tw(o)ice...  how can a response have the
     // the same component run twice?
     if (rb.rsp.getValues().get("ubi") != null) {
-      return;
+      return null;
     }
     SolrParams params = rb.req.getParams();
 
@@ -270,9 +287,9 @@ public void doStuff(ResponseBuilder rb) throws IOException {
     ubiQuery.setApplication(params.get(APPLICATION));
     if (ubiQuery.getApplication() == null) {
       ubiQuery.setApplication(
-          rb.isDistrib
-              ? rb.req.getCloudDescriptor().getCollectionName()
-              : searcher.getCore().getName());
+              rb.isDistrib
+                      ? rb.req.getCloudDescriptor().getCollectionName()
+                      : searcher.getCore().getName());
     }
 
     String queryAttributes = params.get(QUERY_ATTRIBUTES);
@@ -292,12 +309,19 @@ public void doStuff(ResponseBuilder rb) throws IOException {
         }
       }
     }
+    return ubiQuery;
+  }
+
+  public void doDistribStuff(ResponseBuilder rb) throws IOException {
+
+    // not sure why but sometimes we get it tw(o)ice...  how can a response have the
+    // the same component run twice?
+    UBIQuery ubiQuery = getUbiQuery(rb);
+    if (ubiQuery == null) return;
 
-    ResultContext rc = (ResultContext) rb.rsp.getResponse();
-    DocList docs = rc.getDocList();
-    // DocList docs = rb.getResults().docList;
 
-    String docIds = extractDocIds(docs, searcher);
+    //String docIds = extractDocIds(docs, searcher);
+    String docIds =String.join(",", rb.resultIds.keySet().stream().map(Object::toString).toList());
     ubiQuery.setDocIds(docIds);
 
     addUserBehaviorInsightsToResponse(ubiQuery, rb);
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
new file mode 100644
index 00000000000..2a222115d2d
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.io.input.ReversedLinesFileReader;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.solr.client.solrj.io.SolrClientCache;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.*;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.handler.LoggingStream;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.*;
+
+@LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene45"})
+public class UBIComponentDistrQueriesTest extends SolrCloudTestCase {
+
+  private static final String COLLECTIONORALIAS = "collection1";
+  private static final int TIMEOUT = DEFAULT_TIMEOUT;
+  private static final String id = "id";
+
+  private static boolean useAlias;
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    configureCluster(4)
+        .addConfig(
+            "conf", TEST_PATH().resolve("configsets").resolve("ubi-enabled").resolve("conf"))
+        .configure();
+
+    String collection;
+    useAlias = random().nextBoolean();
+    if (useAlias) {
+      collection = COLLECTIONORALIAS + "_collection";
+    } else {
+      collection = COLLECTIONORALIAS;
+    }
+
+    CollectionAdminRequest.createCollection(collection, "conf", 2, 1)
+        .process(cluster.getSolrClient());
+
+    cluster.waitForActiveCollection(collection, 2, 2);
+
+    AbstractDistribZkTestBase.waitForRecoveriesToFinish(
+        collection, cluster.getZkStateReader(), false, true, TIMEOUT);
+    if (useAlias) {
+      CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection)
+          .process(cluster.getSolrClient());
+    }
+
+    // -------------------
+
+    CollectionAdminRequest.createCollection("ubi_queries", "_default", 1, 1)
+            .process(cluster.getSolrClient());
+
+    cluster.waitForActiveCollection("ubi_queries", 1, 1);
+
+    AbstractDistribZkTestBase.waitForRecoveriesToFinish(
+            "ubi_queries", cluster.getZkStateReader(), false, true, TIMEOUT);
+  }
+
+  @Before
+  public void cleanIndex() throws Exception {
+    new UpdateRequest().deleteByQuery("*:*").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+  }
+
+  @Test
+  public void testUBIQueryStream() throws Exception {
+    cluster.getSolrClient(COLLECTIONORALIAS).add(List.of(new SolrInputDocument("id", "1", "subject", "aa"),
+            new SolrInputDocument("id", "two", "subject", "aa"),
+            new SolrInputDocument("id", "3", "subject", "aa")));
+    cluster.getSolrClient(COLLECTIONORALIAS).commit(true, true);
+    QueryResponse queryResponse = cluster.getSolrClient(COLLECTIONORALIAS).query(new MapSolrParams(Map.of("q", "aa", "rows", "2", "ubi", "true")));
+    System.out.println(queryResponse);
+  }
+}

From 39dadf250bd6380b7b603403ac8c21461e5f4122 Mon Sep 17 00:00:00 2001
From: Mikhail Khludnev 
Date: Sat, 7 Dec 2024 22:55:06 +0300
Subject: [PATCH 094/103] seems like a right thing

---
 .../org/apache/solr/handler/component/UBIComponent.java  | 7 +++++--
 .../solr/configsets/ubi-enabled/conf/schema.xml          | 6 +++++-
 .../handler/component/UBIComponentDistrQueriesTest.java  | 9 ++++++---
 3 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 4279a30e322..68dbea447f6 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -51,6 +51,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.solr.handler.RequestHandlerBase.isInternalShardRequest;
+
 /**
  * User Behavior Insights (UBI) is an open standard for gathering query and event data from users
  * and storing it in a structured format. UBI can be used for in session personalization, implicit
@@ -231,8 +233,9 @@ public void process(ResponseBuilder rb) throws IOException {
     if (!params.getBool(COMPONENT_NAME, false)) {
       return;
     }
-
-    doStuff(rb);
+    if (!isInternalShardRequest(rb.req)) { // subordinate shard req shouldn't yield logs
+      doStuff(rb);
+    }
   }
 
   @Override
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml
index 661b02a0f96..b080f6b2526 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml
@@ -19,6 +19,10 @@
 
   
   
-  
+  
+  
+  
+  
+  
   id
 
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
index 2a222115d2d..1e395a25a8a 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
@@ -101,10 +101,13 @@ public void cleanIndex() throws Exception {
   @Test
   public void testUBIQueryStream() throws Exception {
     cluster.getSolrClient(COLLECTIONORALIAS).add(List.of(new SolrInputDocument("id", "1", "subject", "aa"),
-            new SolrInputDocument("id", "two", "subject", "aa"),
+            new SolrInputDocument("id", "2" /*"two"*/, "subject", "aa"),
             new SolrInputDocument("id", "3", "subject", "aa")));
     cluster.getSolrClient(COLLECTIONORALIAS).commit(true, true);
-    QueryResponse queryResponse = cluster.getSolrClient(COLLECTIONORALIAS).query(new MapSolrParams(Map.of("q", "aa", "rows", "2", "ubi", "true")));
-    System.out.println(queryResponse);
+    QueryResponse queryResponse = cluster.getSolrClient(COLLECTIONORALIAS).query(new MapSolrParams(
+            Map.of("q", "aa", "df","subject", "rows", "2", "ubi", "true"
+            )));
+    System.out.println(queryResponse.getResponse().get("ubi"));
+    // TODO check that ids were recorded
   }
 }

From 995a0cfa7b7c9adc1f834888cb2e82bf7847e0e6 Mon Sep 17 00:00:00 2001
From: Mikhail Khludnev 
Date: Mon, 9 Dec 2024 00:13:19 +0300
Subject: [PATCH 095/103] now it checks that query were recorded. Not sure
 about doc ids.

---
 .../solr/handler/component/SearchComponent.java  |  3 ++-
 .../solr/handler/component/SearchHandler.java    |  1 +
 .../solr/configsets/ubi-enabled/conf/schema.xml  |  6 +-----
 .../component/UBIComponentDistrQueriesTest.java  | 16 +++++++++++++---
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
index dc845e1465b..df62b7338e1 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
@@ -121,5 +121,6 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
           Map.entry(RealTimeGetComponent.COMPONENT_NAME, RealTimeGetComponent.class),
           Map.entry(ExpandComponent.COMPONENT_NAME, ExpandComponent.class),
           Map.entry(TermsComponent.COMPONENT_NAME, TermsComponent.class),
-          Map.entry(UBIComponent.COMPONENT_NAME, UBIComponent.class));
+          Map.entry(UBIComponent.COMPONENT_NAME, UBIComponent.class)// oh r'lly?? esp giving that it receive some expr via init args
+      );
 }
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index 467286452e0..e4b453c5920 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -135,6 +135,7 @@ public class SearchHandler extends RequestHandlerBase
   protected List getDefaultComponents() {
     List l = new ArrayList(SearchComponent.STANDARD_COMPONENTS.keySet());
     moveToFirst(l, QueryComponent.COMPONENT_NAME);
+    l.remove(RealTimeGetComponent.COMPONENT_NAME); // pardon. it breaks my essential cloud test. there wasn't it there ever!
     return l;
   }
 
diff --git a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml
index b080f6b2526..661b02a0f96 100644
--- a/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml
+++ b/solr/core/src/test-files/solr/configsets/ubi-enabled/conf/schema.xml
@@ -19,10 +19,6 @@
 
   
   
-  
-  
-  
-  
-  
+  
   id
 
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
index 1e395a25a8a..00395be1d76 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
@@ -30,6 +30,7 @@
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.cloud.AbstractDistribZkTestBase;
 import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.cluster.api.SimpleMap;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.MapSolrParams;
 import org.apache.solr.common.params.SolrParams;
@@ -63,7 +64,7 @@ public static void setupCluster() throws Exception {
         .configure();
 
     String collection;
-    useAlias = random().nextBoolean();
+    useAlias = false; //random().nextBoolean();
     if (useAlias) {
       collection = COLLECTIONORALIAS + "_collection";
     } else {
@@ -84,7 +85,8 @@ public static void setupCluster() throws Exception {
 
     // -------------------
 
-    CollectionAdminRequest.createCollection("ubi_queries", "_default", 1, 1)
+    CollectionAdminRequest.createCollection("ubi_queries",// it seems like a hardcoded name why?
+                    "_default", 1, 1)
             .process(cluster.getSolrClient());
 
     cluster.waitForActiveCollection("ubi_queries", 1, 1);
@@ -107,7 +109,15 @@ public void testUBIQueryStream() throws Exception {
     QueryResponse queryResponse = cluster.getSolrClient(COLLECTIONORALIAS).query(new MapSolrParams(
             Map.of("q", "aa", "df","subject", "rows", "2", "ubi", "true"
             )));
-    System.out.println(queryResponse.getResponse().get("ubi"));
+    String qid = (String) ((SimpleMap) queryResponse.getResponse().get("ubi")).get("query_id");
+    assertTrue(qid.length()>10);
+    Thread.sleep(10000); // I know what you think of
     // TODO check that ids were recorded
+    QueryResponse queryCheck = cluster.getSolrClient("ubi_queries").query(new MapSolrParams(
+            Map.of("q", "id:"+qid //doesn't search it why? is it a race?
+            )));
+    // however I can't see doc ids found there. Shouldn't I ?
+    assertEquals(1L, queryCheck.getResults().getNumFound());
+    assertEquals(queryCheck.getResults().get(0).get("id"),qid);
   }
 }

From 08b28a572319eb57da8d66711451791889936a6d Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 9 Dec 2024 13:45:31 -0500
Subject: [PATCH 096/103] Group UBI specific classes into inner classes of
 UBIComponent.

Neither UBIQuery or UBIQueryStream will ever be used outside of the UBICompoent.   Thought about a o.a.s.handler.component.ubi package as well, but this seems more specific...
---
 .../solr/handler/component/UBIComponent.java  | 199 ++++++++++++++++++
 .../solr/handler/component/UBIQuery.java      | 129 ------------
 .../handler/component/UBIQueryStream.java     | 115 ----------
 .../UBIComponentStreamingQueriesTest.java     |  24 +--
 4 files changed, 211 insertions(+), 256 deletions(-)
 delete mode 100644 solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
 delete mode 100644 solr/core/src/java/org/apache/solr/handler/component/UBIQueryStream.java

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 6c2b4b78594..9d8524e78b0 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -21,16 +21,29 @@
 import java.io.LineNumberReader;
 import java.lang.invoke.MethodHandles;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.comp.StreamComparator;
 import org.apache.solr.client.solrj.io.stream.StreamContext;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
 import org.apache.solr.client.solrj.io.stream.expr.DefaultStreamFactory;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
@@ -421,4 +434,190 @@ private static TupleStream constructStream(
   public String getDescription() {
     return "A component that tracks the original user query and the resulting documents returned to understand the user.";
   }
+
+  /**
+   * Handles all the data required for tracking a query using User Behavior Insights.
+   *
+   * 
Compatible with the
+   * https://github.com/o19s/ubi/blob/main/schema/1.2.0/query.request.schema.json.
+   */
+  public static class UBIQuery {
+
+    private String application;
+    private String queryId;
+    private String userQuery;
+    private Date timestamp;
+
+    @SuppressWarnings("rawtypes")
+    private Map queryAttributes;
+
+    private String docIds;
+
+    public UBIQuery(String queryId) {
+
+      if (queryId == null) {
+        queryId = UUID.randomUUID().toString().toLowerCase(Locale.ROOT);
+      }
+      this.queryId = queryId;
+      this.timestamp = new Date();
+    }
+
+    public Date getTimestamp() {
+      return timestamp;
+    }
+
+    public void setApplication(String application) {
+      this.application = application;
+    }
+
+    public String getApplication() {
+      return this.application;
+    }
+
+    public String getQueryId() {
+      return queryId;
+    }
+
+    public void setQueryId(String queryId) {
+      this.queryId = queryId;
+    }
+
+    public String getUserQuery() {
+      return userQuery;
+    }
+
+    public void setUserQuery(String userQuery) {
+      this.userQuery = userQuery;
+    }
+
+    @SuppressWarnings("rawtypes")
+    public Map getQueryAttributes() {
+      return queryAttributes;
+    }
+
+    @SuppressWarnings("rawtypes")
+    public void setQueryAttributes(Map queryAttributes) {
+      this.queryAttributes = queryAttributes;
+    }
+
+    public String getDocIds() {
+      return docIds;
+    }
+
+    public void setDocIds(String docIds) {
+      this.docIds = docIds;
+    }
+
+    @SuppressWarnings({"rawtypes", "unchecked"})
+    public Map toMap() {
+      @SuppressWarnings({"rawtypes", "unchecked"})
+      Map map = new HashMap();
+      map.put(QUERY_ID, this.queryId);
+      map.put(
+          "timestamp",
+          DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(this.timestamp.getTime())));
+      if (this.application != null) {
+        map.put(APPLICATION, this.application);
+      }
+      if (this.userQuery != null) {
+        map.put(USER_QUERY, this.userQuery);
+      }
+      if (this.queryAttributes != null) {
+
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+          map.put(
+                  QUERY_ATTRIBUTES, objectMapper.writeValueAsString(this.queryAttributes));
+        } catch (JsonProcessingException e) {
+          // eat it.
+        }
+      }
+
+      return map;
+    }
+  }
+
+  /**
+   * Converts a UBIQuery that is stored in the StreamContext under the key 'ubi-query' into a Tuple
+   * and returns it.
+   *
+   * 
I suspect that if I had the right magic with a LetStream or a GetStream, I could somehow just
+   * use that to say "pluck the 'ubi-query' object out of the StreamContext and call .toTuple or make
+   * a map of it and that would be my tuple'.
+   */
+  public static class UBIQueryStream extends TupleStream implements Expressible {
+
+    private StreamContext streamContext;
+    private boolean finished;
+
+    public UBIQueryStream(StreamExpression expression, StreamFactory factory) throws IOException {}
+
+    @Override
+    public StreamExpression toExpression(StreamFactory factory) throws IOException {
+      return toExpression(factory, true);
+    }
+
+    private StreamExpression toExpression(StreamFactory factory, boolean includeStreams)
+        throws IOException {
+      // function name
+      StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
+
+      return expression;
+    }
+
+    @Override
+    public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+      StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+      explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+      explanation.setImplementingClass(this.getClass().getName());
+      explanation.setExpressionType(Explanation.ExpressionType.STREAM_SOURCE);
+      explanation.setExpression(toExpression(factory, false).toString());
+
+      return explanation;
+    }
+
+    @Override
+    public void setStreamContext(StreamContext context) {
+      this.streamContext = context;
+    }
+
+    @Override
+    public List children() {
+      List l = new ArrayList<>();
+      return l;
+    }
+
+    @Override
+    public void open() throws IOException {}
+
+    @Override
+    public void close() throws IOException {}
+
+    @SuppressWarnings({"unchecked"})
+    @Override
+    public Tuple read() throws IOException {
+
+      if (finished) {
+        return Tuple.EOF();
+      } else {
+        finished = true;
+
+        UBIQuery ubiQuery = (UBIQuery) streamContext.get("ubi-query");
+
+        return new Tuple(ubiQuery.toMap());
+      }
+    }
+
+    /** Return the stream sort - ie, the order in which records are returned */
+    @Override
+    public StreamComparator getStreamSort() {
+      return null;
+    }
+
+    @Override
+    public int getCost() {
+      return 0;
+    }
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
deleted file mode 100644
index fceeb550169..00000000000
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQuery.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.component;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import java.time.Instant;
-import java.time.format.DateTimeFormatter;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.UUID;
-
-/**
- * Handles all the data required for tracking a query using User Behavior Insights.
- *
- * 
Compatible with the
- * https://github.com/o19s/ubi/blob/main/schema/1.2.0/query.request.schema.json.
- */
-public class UBIQuery {
-
-  private String application;
-  private String queryId;
-  private String userQuery;
-  private Date timestamp;
-
-  @SuppressWarnings("rawtypes")
-  private Map queryAttributes;
-
-  private String docIds;
-
-  public UBIQuery(String queryId) {
-
-    if (queryId == null) {
-      queryId = UUID.randomUUID().toString().toLowerCase(Locale.ROOT);
-    }
-    this.queryId = queryId;
-    this.timestamp = new Date();
-  }
-
-  public Date getTimestamp() {
-    return timestamp;
-  }
-
-  public void setApplication(String application) {
-    this.application = application;
-  }
-
-  public String getApplication() {
-    return this.application;
-  }
-
-  public String getQueryId() {
-    return queryId;
-  }
-
-  public void setQueryId(String queryId) {
-    this.queryId = queryId;
-  }
-
-  public String getUserQuery() {
-    return userQuery;
-  }
-
-  public void setUserQuery(String userQuery) {
-    this.userQuery = userQuery;
-  }
-
-  @SuppressWarnings("rawtypes")
-  public Map getQueryAttributes() {
-    return queryAttributes;
-  }
-
-  @SuppressWarnings("rawtypes")
-  public void setQueryAttributes(Map queryAttributes) {
-    this.queryAttributes = queryAttributes;
-  }
-
-  public String getDocIds() {
-    return docIds;
-  }
-
-  public void setDocIds(String docIds) {
-    this.docIds = docIds;
-  }
-
-  @SuppressWarnings({"rawtypes", "unchecked"})
-  public Map toMap() {
-    @SuppressWarnings({"rawtypes", "unchecked"})
-    Map map = new HashMap();
-    map.put(UBIComponent.QUERY_ID, this.queryId);
-    map.put(
-        "timestamp",
-        DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(this.timestamp.getTime())));
-    if (this.application != null) {
-      map.put(UBIComponent.APPLICATION, this.application);
-    }
-    if (this.userQuery != null) {
-      map.put(UBIComponent.USER_QUERY, this.userQuery);
-    }
-    if (this.queryAttributes != null) {
-
-      ObjectMapper objectMapper = new ObjectMapper();
-      try {
-        map.put(
-            UBIComponent.QUERY_ATTRIBUTES, objectMapper.writeValueAsString(this.queryAttributes));
-      } catch (JsonProcessingException e) {
-        // eat it.
-      }
-    }
-
-    return map;
-  }
-}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIQueryStream.java b/solr/core/src/java/org/apache/solr/handler/component/UBIQueryStream.java
deleted file mode 100644
index c36d128ef4f..00000000000
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIQueryStream.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.component;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.comp.StreamComparator;
-import org.apache.solr.client.solrj.io.stream.StreamContext;
-import org.apache.solr.client.solrj.io.stream.TupleStream;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-/**
- * Converts a UBIQuery that is stored in the StreamContext under the key 'ubi-query' into a Tuple
- * and returns it.
- *
- * 
I suspect that if I had the right magic with a LetStream or a GetStream, I could somehow just
- * use that to say "pluck the 'ubi-query' object out of the StreamContext and call .toTuple or make
- * a map of it and that would be my tuple'.
- */
-public class UBIQueryStream extends TupleStream implements Expressible {
-
-  private StreamContext streamContext;
-  private boolean finished;
-
-  public UBIQueryStream(StreamExpression expression, StreamFactory factory) throws IOException {}
-
-  @Override
-  public StreamExpression toExpression(StreamFactory factory) throws IOException {
-    return toExpression(factory, true);
-  }
-
-  private StreamExpression toExpression(StreamFactory factory, boolean includeStreams)
-      throws IOException {
-    // function name
-    StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
-
-    return expression;
-  }
-
-  @Override
-  public Explanation toExplanation(StreamFactory factory) throws IOException {
-
-    StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
-    explanation.setFunctionName(factory.getFunctionName(this.getClass()));
-    explanation.setImplementingClass(this.getClass().getName());
-    explanation.setExpressionType(ExpressionType.STREAM_SOURCE);
-    explanation.setExpression(toExpression(factory, false).toString());
-
-    return explanation;
-  }
-
-  @Override
-  public void setStreamContext(StreamContext context) {
-    this.streamContext = context;
-  }
-
-  @Override
-  public List children() {
-    List l = new ArrayList<>();
-    return l;
-  }
-
-  @Override
-  public void open() throws IOException {}
-
-  @Override
-  public void close() throws IOException {}
-
-  @SuppressWarnings({"unchecked"})
-  @Override
-  public Tuple read() throws IOException {
-
-    if (finished) {
-      return Tuple.EOF();
-    } else {
-      finished = true;
-
-      UBIQuery ubiQuery = (UBIQuery) streamContext.get("ubi-query");
-
-      return new Tuple(ubiQuery.toMap());
-    }
-  }
-
-  /** Return the stream sort - ie, the order in which records are returned */
-  @Override
-  public StreamComparator getStreamSort() {
-    return null;
-  }
-
-  @Override
-  public int getCost() {
-    return 0;
-  }
-}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index 89d1a8ada5c..b7ff119391b 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -95,7 +95,7 @@ public void cleanIndex() throws Exception {
   @Test
   public void testUBIQueryStream() throws Exception {
 
-    UBIQuery ubiQuery;
+    UBIComponent.UBIQuery ubiQuery;
     StreamExpression expression;
     TupleStream stream;
     List tuples;
@@ -105,13 +105,13 @@ public void testUBIQueryStream() throws Exception {
     try (solrClientCache) {
       streamContext.setSolrClientCache(solrClientCache);
       StreamFactory factory =
-          new StreamFactory().withFunctionName("ubiQuery", UBIQueryStream.class);
+          new StreamFactory().withFunctionName("ubiQuery", UBIComponent.UBIQueryStream.class);
       // Basic test
-      ubiQuery = new UBIQuery("123");
+      ubiQuery = new UBIComponent.UBIQuery("123");
 
       expression = StreamExpressionParser.parse("ubiQuery()");
       streamContext.put("ubi-query", ubiQuery);
-      stream = new UBIQueryStream(expression, factory);
+      stream = new UBIComponent.UBIQueryStream(expression, factory);
       stream.setStreamContext(streamContext);
       tuples = getTuples(stream);
 
@@ -122,11 +122,11 @@ public void testUBIQueryStream() throws Exception {
       assertNotNull(Instant.parse(tuples.get(0).getString("timestamp")));
 
       // Include another field to see what is returned
-      ubiQuery = new UBIQuery("234");
+      ubiQuery = new UBIComponent.UBIQuery("234");
       ubiQuery.setApplication("typeahead");
 
       streamContext.put("ubi-query", ubiQuery);
-      stream = new UBIQueryStream(expression, factory);
+      stream = new UBIComponent.UBIQueryStream(expression, factory);
       stream.setStreamContext(streamContext);
       tuples = getTuples(stream);
 
@@ -136,7 +136,7 @@ public void testUBIQueryStream() throws Exception {
       assertString(tuples.get(0), "application", "typeahead");
 
       // Introduce event_attributes map of data
-      ubiQuery = new UBIQuery("345");
+      ubiQuery = new UBIComponent.UBIQuery("345");
 
       @SuppressWarnings({"unchecked", "rawtypes"})
       Map queryAttributes = new HashMap();
@@ -145,7 +145,7 @@ public void testUBIQueryStream() throws Exception {
       ubiQuery.setQueryAttributes(queryAttributes);
 
       streamContext.put("ubi-query", ubiQuery);
-      stream = new UBIQueryStream(expression, factory);
+      stream = new UBIComponent.UBIQueryStream(expression, factory);
       stream.setStreamContext(streamContext);
       tuples = getTuples(stream);
 
@@ -159,7 +159,7 @@ public void testUBIQueryStream() throws Exception {
   @Test
   public void testWritingToLogUbiQueryStream() throws Exception {
     // Test that we can write out UBIQuery data cleanly to the jsonl file
-    UBIQuery ubiQuery = new UBIQuery("345");
+    UBIComponent.UBIQuery ubiQuery = new UBIComponent.UBIQuery("345");
     ubiQuery.setUserQuery("Memory RAM");
     ubiQuery.setApplication("typeahead");
 
@@ -181,7 +181,7 @@ public void testWritingToLogUbiQueryStream() throws Exception {
           new StreamFactory()
               .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
               .withFunctionName("search", CloudSolrStream.class)
-              .withFunctionName("ubiQuery", UBIQueryStream.class)
+              .withFunctionName("ubiQuery", UBIComponent.UBIQueryStream.class)
               .withFunctionName("logging", LoggingStream.class);
 
       expression = StreamExpressionParser.parse("logging(test.jsonl,ubiQuery())");
@@ -221,7 +221,7 @@ public void testWritingToLogUbiQueryStream() throws Exception {
   public void testWritingToSolrUbiQueryStream() throws Exception {
     // Test that we can write out UBIQuery, especially the queryAttributes map, to Solr collection
 
-    UBIQuery ubiQuery = new UBIQuery("345");
+    UBIComponent.UBIQuery ubiQuery = new UBIComponent.UBIQuery("345");
     ubiQuery.setUserQuery("Memory RAM");
     ubiQuery.setApplication("typeahead");
 
@@ -248,7 +248,7 @@ public void testWritingToSolrUbiQueryStream() throws Exception {
               .withFunctionName("search", CloudSolrStream.class)
               .withFunctionName("update", UpdateStream.class)
               .withFunctionName("select", SelectStream.class)
-              .withFunctionName("ubiQuery", UBIQueryStream.class);
+              .withFunctionName("ubiQuery", UBIComponent.UBIQueryStream.class);
 
       expression =
           StreamExpressionParser.parse(

From d01f7950ed24e48bf78c74128bde3dd17e671406 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 9 Dec 2024 15:05:50 -0500
Subject: [PATCH 097/103] Start tracking in the backend the document id's.

This doesn't cover converting doc_ids into a JSON array of any type...
---
 .../solr/handler/component/UBIComponent.java  | 18 ++++---
 .../UBIComponentStreamingQueriesTest.java     | 54 +++++++++----------
 2 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 9d8524e78b0..8e49de3464d 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -16,6 +16,8 @@
  */
 package org.apache.solr.handler.component;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.LineNumberReader;
@@ -32,9 +34,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
@@ -145,6 +144,7 @@ public class UBIComponent extends SearchComponent implements SolrCoreAware {
   public static final String QUERY_ATTRIBUTES = "query_attributes";
   public static final String USER_QUERY = "user_query";
   public static final String APPLICATION = "application";
+  public static final String DOC_IDS = "doc_ids";
 
   protected PluginInfo info = PluginInfo.EMPTY_INFO;
 
@@ -522,12 +522,14 @@ public Map toMap() {
       if (this.userQuery != null) {
         map.put(USER_QUERY, this.userQuery);
       }
+      if (this.docIds != null) {
+        map.put(DOC_IDS, this.docIds);
+      }
       if (this.queryAttributes != null) {
 
         ObjectMapper objectMapper = new ObjectMapper();
         try {
-          map.put(
-                  QUERY_ATTRIBUTES, objectMapper.writeValueAsString(this.queryAttributes));
+          map.put(QUERY_ATTRIBUTES, objectMapper.writeValueAsString(this.queryAttributes));
         } catch (JsonProcessingException e) {
           // eat it.
         }
@@ -541,9 +543,9 @@ public Map toMap() {
    * Converts a UBIQuery that is stored in the StreamContext under the key 'ubi-query' into a Tuple
    * and returns it.
    *
-   * 
I suspect that if I had the right magic with a LetStream or a GetStream, I could somehow just
-   * use that to say "pluck the 'ubi-query' object out of the StreamContext and call .toTuple or make
-   * a map of it and that would be my tuple'.
+   * 
I suspect that if I had the right magic with a LetStream or a GetStream, I could somehow
+   * just use that to say "pluck the 'ubi-query' object out of the StreamContext and call .toTuple
+   * or make a map of it and that would be my tuple'.
    */
   public static class UBIQueryStream extends TupleStream implements Expressible {
 
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
index b7ff119391b..45f8d4a9a23 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentStreamingQueriesTest.java
@@ -55,9 +55,6 @@ public class UBIComponentStreamingQueriesTest extends SolrCloudTestCase {
 
   private static final String COLLECTIONORALIAS = "collection1";
   private static final int TIMEOUT = DEFAULT_TIMEOUT;
-  private static final String id = "id";
-
-  private static boolean useAlias;
 
   @BeforeClass
   public static void setupCluster() throws Exception {
@@ -67,7 +64,7 @@ public static void setupCluster() throws Exception {
         .configure();
 
     String collection;
-    useAlias = random().nextBoolean();
+    boolean useAlias = random().nextBoolean();
     if (useAlias) {
       collection = COLLECTIONORALIAS + "_collection";
     } else {
@@ -92,6 +89,10 @@ public void cleanIndex() throws Exception {
     new UpdateRequest().deleteByQuery("*:*").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
   }
 
+  /**
+   * Test using the UBIQuery and UBIQueryStream classes independent of the UBIComponent to stream
+   * events.
+   */
   @Test
   public void testUBIQueryStream() throws Exception {
 
@@ -121,6 +122,19 @@ public void testUBIQueryStream() throws Exception {
 
       assertNotNull(Instant.parse(tuples.get(0).getString("timestamp")));
 
+      // Introduce docIds
+      ubiQuery = new UBIComponent.UBIQuery("678");
+      ubiQuery.setDocIds(("1,2,3,doc_52"));
+      streamContext.put("ubi-query", ubiQuery);
+      stream = new UBIComponent.UBIQueryStream(expression, factory);
+      stream.setStreamContext(streamContext);
+      tuples = getTuples(stream);
+
+      assertEquals(1, tuples.size());
+      assertFields(tuples, "query_id", "doc_ids");
+      // assertEquals(new String[]{"1", "2", "3", "doc_52"}, tuples.get(0).getStrings("doc_ids"));
+      assertEquals("1,2,3,doc_52", tuples.get(0).getString("doc_ids"));
+
       // Include another field to see what is returned
       ubiQuery = new UBIComponent.UBIQuery("234");
       ubiQuery.setApplication("typeahead");
@@ -158,7 +172,7 @@ public void testUBIQueryStream() throws Exception {
 
   @Test
   public void testWritingToLogUbiQueryStream() throws Exception {
-    // Test that we can write out UBIQuery data cleanly to the jsonl file
+    // Test that we can write out UBIQuery data cleanly to the JSON w Lines formatted log file.
     UBIComponent.UBIQuery ubiQuery = new UBIComponent.UBIQuery("345");
     ubiQuery.setUserQuery("Memory RAM");
     ubiQuery.setApplication("typeahead");
@@ -195,7 +209,7 @@ public void testWritingToLogUbiQueryStream() throws Exception {
       assertFields(tuples, "totalIndexed");
       assertLong(tuples.get(0), "totalIndexed", 1);
 
-      // Someday when we have parseJSON() streaming expression we can replace this.
+      // Someday when we have a parseJSON() streaming expression we can replace this.
       Path filePath = stream.getFilePath();
       try (ReversedLinesFileReader reader =
           new ReversedLinesFileReader.Builder()
@@ -209,6 +223,7 @@ public void testWritingToLogUbiQueryStream() throws Exception {
         Map ubiQueryAsMap = objectMapper.readValue(jsonLine, Map.class);
         assertEquals(ubiQuery.getQueryId(), ubiQueryAsMap.get("query_id"));
         assertEquals(ubiQuery.getApplication(), ubiQueryAsMap.get("application"));
+        assertEquals(ubiQuery.getDocIds(), ubiQueryAsMap.get("doc_ids"));
         assertNotNull(ubiQueryAsMap.get("timestamp"));
         assertEquals(
             "{\"experiment\":\"secret\",\"marginBoost\":2.1,\"parsed_query\":\"memory OR ram\"}",
@@ -219,7 +234,8 @@ public void testWritingToLogUbiQueryStream() throws Exception {
 
   @Test
   public void testWritingToSolrUbiQueryStream() throws Exception {
-    // Test that we can write out UBIQuery, especially the queryAttributes map, to Solr collection
+    // Test that we can write out UBIQuery, especially the queryAttributes map, to a Solr
+    // collection.
 
     UBIComponent.UBIQuery ubiQuery = new UBIComponent.UBIQuery("345");
     ubiQuery.setUserQuery("Memory RAM");
@@ -257,6 +273,7 @@ public void testWritingToSolrUbiQueryStream() throws Exception {
                   + ", batchSize=5, select(\n"
                   + "      ubiQuery(),\n"
                   + "      query_id as id,\n"
+                  + "      doc_ids,\n"
                   + "      timestamp,\n"
                   + "      application,\n"
                   + "      user_query,\n"
@@ -288,6 +305,7 @@ public void testWritingToSolrUbiQueryStream() throws Exception {
       assertEquals(ubiQuery.getQueryId(), tuple.get("id"));
       assertEquals(ubiQuery.getApplication(), tuple.get("application"));
       assertEquals(ubiQuery.getUserQuery(), tuple.get("user_query"));
+      assertEquals(ubiQuery.getDocIds(), tuple.get("doc_ids"));
       assertEquals(ubiQuery.getTimestamp().toInstant(), tuple.getDate("timestamp").toInstant());
       assertEquals(
           "{\"experiment\":\"secret\",\"marginBoost\":2.1,\"parsed_query\":\"memory OR ram\"}",
@@ -307,18 +325,6 @@ protected List getTuples(TupleStream tupleStream) throws IOException {
     return tuples;
   }
 
-  protected void assertOrderOf(List tuples, String fieldName, int... ids) throws Exception {
-    int i = 0;
-    for (int val : ids) {
-      Tuple t = tuples.get(i);
-      String tip = t.getString(fieldName);
-      if (!tip.equals(Integer.toString(val))) {
-        throw new Exception("Found value:" + tip + " expecting:" + val);
-      }
-      ++i;
-    }
-  }
-
   public boolean assertString(Tuple tuple, String fieldName, String expected) throws Exception {
     String actual = (String) tuple.get(fieldName);
 
@@ -348,16 +354,6 @@ protected void assertFields(List tuples, String... fields) throws Excepti
     }
   }
 
-  protected void assertNotFields(List tuples, String... fields) throws Exception {
-    for (Tuple tuple : tuples) {
-      for (String field : fields) {
-        if (tuple.getFields().containsKey(field)) {
-          throw new Exception(String.format(Locale.ROOT, "Unexpected field '%s' found", field));
-        }
-      }
-    }
-  }
-
   private static SolrCore findSolrCore() {
     for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
       for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {

From 674f83ef85aca3d8df8001bba4deb06018204614 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 9 Dec 2024 16:07:23 -0500
Subject: [PATCH 098/103] Restore previous approach for standard components...

Exxcept that we had to change to support TEN standard components by not using Map.of.  Also, I couldn't stand the lower case + underscore "standard_components" object name.
---
 .../handler/component/SearchComponent.java    | 30 +++++++++++--------
 .../solr/handler/component/SearchHandler.java | 24 ++++++++-------
 2 files changed, 31 insertions(+), 23 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
index dc845e1465b..b7de8aa620a 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
@@ -17,6 +17,7 @@
 package org.apache.solr.handler.component;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import org.apache.solr.core.SolrInfoBean;
 import org.apache.solr.metrics.SolrMetricsContext;
@@ -109,17 +110,20 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
     this.solrMetricsContext = parentContext.getChildContext(this);
   }
 
-  public static final Map> STANDARD_COMPONENTS =
-      Map.ofEntries(
-          Map.entry(QueryComponent.COMPONENT_NAME, QueryComponent.class),
-          Map.entry(HighlightComponent.COMPONENT_NAME, HighlightComponent.class),
-          Map.entry(FacetComponent.COMPONENT_NAME, FacetComponent.class),
-          Map.entry(FacetModule.COMPONENT_NAME, FacetModule.class),
-          Map.entry(MoreLikeThisComponent.COMPONENT_NAME, MoreLikeThisComponent.class),
-          Map.entry(StatsComponent.COMPONENT_NAME, StatsComponent.class),
-          Map.entry(DebugComponent.COMPONENT_NAME, DebugComponent.class),
-          Map.entry(RealTimeGetComponent.COMPONENT_NAME, RealTimeGetComponent.class),
-          Map.entry(ExpandComponent.COMPONENT_NAME, ExpandComponent.class),
-          Map.entry(TermsComponent.COMPONENT_NAME, TermsComponent.class),
-          Map.entry(UBIComponent.COMPONENT_NAME, UBIComponent.class));
+  public static final Map> STANDARD_COMPONENTS;
+
+  static {
+    STANDARD_COMPONENTS = new HashMap<>();
+    STANDARD_COMPONENTS.put(HighlightComponent.COMPONENT_NAME, HighlightComponent.class);
+    STANDARD_COMPONENTS.put(QueryComponent.COMPONENT_NAME, QueryComponent.class);
+    STANDARD_COMPONENTS.put(FacetComponent.COMPONENT_NAME, FacetComponent.class);
+    STANDARD_COMPONENTS.put(FacetModule.COMPONENT_NAME, FacetModule.class);
+    STANDARD_COMPONENTS.put(MoreLikeThisComponent.COMPONENT_NAME, MoreLikeThisComponent.class);
+    STANDARD_COMPONENTS.put(StatsComponent.COMPONENT_NAME, StatsComponent.class);
+    STANDARD_COMPONENTS.put(DebugComponent.COMPONENT_NAME, DebugComponent.class);
+    STANDARD_COMPONENTS.put(RealTimeGetComponent.COMPONENT_NAME, RealTimeGetComponent.class);
+    STANDARD_COMPONENTS.put(ExpandComponent.COMPONENT_NAME, ExpandComponent.class);
+    STANDARD_COMPONENTS.put(TermsComponent.COMPONENT_NAME, TermsComponent.class);
+    STANDARD_COMPONENTS.put(UBIComponent.COMPONENT_NAME, UBIComponent.class);
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index 467286452e0..4399a4ca88b 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -71,6 +71,7 @@
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.search.CursorMark;
 import org.apache.solr.search.SortSpec;
+import org.apache.solr.search.facet.FacetModule;
 import org.apache.solr.security.AuthorizationContext;
 import org.apache.solr.security.PermissionNameProvider;
 import org.apache.solr.util.RTimerTree;
@@ -133,16 +134,19 @@ public class SearchHandler extends RequestHandlerBase
    * @return A list of component names.
    */
   protected List getDefaultComponents() {
-    List l = new ArrayList(SearchComponent.STANDARD_COMPONENTS.keySet());
-    moveToFirst(l, QueryComponent.COMPONENT_NAME);
-    return l;
-  }
-
-  private static void moveToFirst(List list, String target) {
-    int index = list.indexOf(target);
-    assert index != -1;
-    list.remove(index);
-    list.add(0, target);
+    ArrayList names = new ArrayList<>(9);
+    names.add(QueryComponent.COMPONENT_NAME);
+    names.add(FacetComponent.COMPONENT_NAME);
+    names.add(FacetModule.COMPONENT_NAME);
+    names.add(MoreLikeThisComponent.COMPONENT_NAME);
+    names.add(HighlightComponent.COMPONENT_NAME);
+    names.add(StatsComponent.COMPONENT_NAME);
+    names.add(DebugComponent.COMPONENT_NAME);
+    names.add(ExpandComponent.COMPONENT_NAME);
+    names.add(TermsComponent.COMPONENT_NAME);
+    names.add(UBIComponent.COMPONENT_NAME);
+
+    return names;
   }
 
   @Override

From 56ec0d0bbdf1e5887f9e52ac07749ded046a4e01 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 9 Dec 2024 16:34:47 -0500
Subject: [PATCH 099/103] maybe needed?

---
 gradle/testing/randomization/policies/solr-tests.policy | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy
index eada6893445..31b9da9807b 100644
--- a/gradle/testing/randomization/policies/solr-tests.policy
+++ b/gradle/testing/randomization/policies/solr-tests.policy
@@ -41,6 +41,7 @@ grant {
   // Needed by org.apache.solr.handler.component.UBIComponentTest
   permission java.io.FilePermission "${common-solr.dir}/core/build/resources/test/solr/userfiles/ubi_queries.jsonl", "write";
   permission java.io.FilePermission "/tmp/src/solr/solr/core/build/resources/test/solr/userfiles${/}-", "write";
+  permission java.io.FilePermission "/tmp/src/solr/solr/core/build/resources/test/solr/userfiles", "write";
 
   permission java.nio.file.LinkPermission "hard";
 

From dc378fe35ed025acd235b87626011bd970ae5c1c Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 9 Dec 2024 17:13:49 -0500
Subject: [PATCH 100/103] Introduce UBI to the SolrJ client.

I suspect lots of places for fixing.  Like pulling out field names into a UBIParams.java file.   May want to name space ubi query params under "ubi."..  what about in JSON query?
---
 .../apache/solr/client/solrj/SolrQuery.java   | 52 +++++++++++++++++++
 .../solr/client/solrj/SolrQueryTest.java      | 13 +++++
 2 files changed, 65 insertions(+)

diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java
index 76b053a0824..5102f845a9f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java
@@ -818,6 +818,58 @@ public SolrQuery setHighlight(boolean b) {
     return this;
   }
 
+  /**
+   * Enable User Behavior Insights tracking for this query.
+   *
+   * @param b
+   */
+  public SolrQuery setUBITracking(boolean b) {
+    if (b) {
+      this.set("ubi", true);
+    } else {
+      // makes me wonder if this should be all under ubi.* namespace.
+      // so ubi.application, ubi.query_id....
+      this.remove("ubi");
+      this.remove("application");
+      this.remove("query_id");
+      this.remove("client_id");
+      this.remove("user_query");
+      // this.remove("object_id_fields");
+      this.remove("query_attributes");
+    }
+    return this;
+  }
+
+  /** Determine status of User Behavior Insights tracking for this query. */
+  public boolean getUBITracking() {
+    return this.getBool("ubi", false);
+  }
+
+  public SolrQuery setApplication(String application) {
+    this.set("application", application);
+    return this;
+  }
+
+  public SolrQuery setQueryId(String queryId) {
+    this.set("query_id", queryId);
+    return this;
+  }
+
+  public SolrQuery setClientId(String clientId) {
+    this.set("client_id", clientId);
+    return this;
+  }
+
+  public SolrQuery setUserQuery(String userQuery) {
+    this.set("user_query", userQuery);
+    return this;
+  }
+
+  /**
+   * public SolrQuery setQueryAttributes(Map queryAttributes) { this.set("query_attributes",
+   * queryAttributes); return this; }
+   */
+
   /**
    * Add field for MoreLikeThis. Automatically enables MoreLikeThis.
    *
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrQueryTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrQueryTest.java
index 2786fd64385..2e72193b96b 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrQueryTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrQueryTest.java
@@ -471,4 +471,17 @@ public void testMoreLikeThis() {
     assertEquals(15, solrQuery.setMoreLikeThisMaxQueryTerms(15).getMoreLikeThisMaxQueryTerms());
     assertEquals(16, solrQuery.setMoreLikeThisCount(16).getMoreLikeThisCount());
   }
+
+  public void testUserBehaviorInsights() {
+    SolrQuery solrQuery = new SolrQuery();
+    solrQuery.setUBITracking(true);
+    assertTrue(solrQuery.getUBITracking());
+
+    assertNull(solrQuery.get("query_id"));
+    solrQuery.setQueryId("12345");
+    assertEquals("12345", solrQuery.get("query_id"));
+
+    // need to figure out how to test query_attributes
+
+  }
 }

From dc9a0ed1b6e8408852d5957217c0598b56734f69 Mon Sep 17 00:00:00 2001
From: Eric Pugh 
Date: Mon, 9 Dec 2024 17:13:59 -0500
Subject: [PATCH 101/103] lint

---
 .../src/java/org/apache/solr/handler/component/UBIComponent.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 8e49de3464d..08ab3a1059e 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -308,7 +308,6 @@ public void doStuff(ResponseBuilder rb) throws IOException {
 
     ResultContext rc = (ResultContext) rb.rsp.getResponse();
     DocList docs = rc.getDocList();
-    // DocList docs = rb.getResults().docList;
 
     String docIds = extractDocIds(docs, searcher);
     ubiQuery.setDocIds(docIds);

From 0887cae41b7bb6b33699fd64bdacbfd0b2f901c6 Mon Sep 17 00:00:00 2001
From: Mikhail Khludnev 
Date: Tue, 10 Dec 2024 14:26:41 +0300
Subject: [PATCH 102/103] spotless apply

---
 .../handler/component/SearchComponent.java    |  2 +-
 .../solr/handler/component/UBIComponent.java  | 14 ++--
 .../UBIComponentDistrQueriesTest.java         | 69 +++++++++----------
 3 files changed, 41 insertions(+), 44 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
index bdd121a8c97..b7de8aa620a 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
@@ -109,7 +109,7 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
     // By default, don't register any metrics - but prepare a child context
     this.solrMetricsContext = parentContext.getChildContext(this);
   }
-  
+
   public static final Map> STANDARD_COMPONENTS;
 
   static {
diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index 0a944221f67..fc5756f24bb 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -16,6 +16,8 @@
  */
 package org.apache.solr.handler.component;
 
+import static org.apache.solr.handler.RequestHandlerBase.isInternalShardRequest;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
@@ -63,8 +65,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.solr.handler.RequestHandlerBase.isInternalShardRequest;
-
 /**
  * User Behavior Insights (UBI) is an open standard for gathering query and event data from users
  * and storing it in a structured format. UBI can be used for in session personalization, implicit
@@ -303,9 +303,9 @@ private static UBIQuery getUbiQuery(ResponseBuilder rb) {
     ubiQuery.setApplication(params.get(APPLICATION));
     if (ubiQuery.getApplication() == null) {
       ubiQuery.setApplication(
-              rb.isDistrib
-                      ? rb.req.getCloudDescriptor().getCollectionName()
-                      : searcher.getCore().getName());
+          rb.isDistrib
+              ? rb.req.getCloudDescriptor().getCollectionName()
+              : searcher.getCore().getName());
     }
 
     String queryAttributes = params.get(QUERY_ATTRIBUTES);
@@ -334,8 +334,8 @@ public void doDistribStuff(ResponseBuilder rb) throws IOException {
     // the same component run twice?
     UBIQuery ubiQuery = getUbiQuery(rb);
     if (ubiQuery == null) return;
-    //String docIds = extractDocIds(docs, searcher);
-    String docIds =String.join(",", rb.resultIds.keySet().stream().map(Object::toString).toList());
+    // String docIds = extractDocIds(docs, searcher);
+    String docIds = String.join(",", rb.resultIds.keySet().stream().map(Object::toString).toList());
     ubiQuery.setDocIds(docIds);
 
     addUserBehaviorInsightsToResponse(ubiQuery, rb);
diff --git a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
index 00395be1d76..c87b8f5c268 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/UBIComponentDistrQueriesTest.java
@@ -16,15 +16,9 @@
  */
 package org.apache.solr.handler.component;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.input.ReversedLinesFileReader;
+import java.util.List;
+import java.util.Map;
 import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.solr.client.solrj.io.SolrClientCache;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.stream.*;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
@@ -33,20 +27,10 @@
 import org.apache.solr.cluster.api.SimpleMap;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.MapSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.handler.LoggingStream;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.time.Instant;
-import java.util.*;
-
 @LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene45"})
 public class UBIComponentDistrQueriesTest extends SolrCloudTestCase {
 
@@ -59,12 +43,11 @@ public class UBIComponentDistrQueriesTest extends SolrCloudTestCase {
   @BeforeClass
   public static void setupCluster() throws Exception {
     configureCluster(4)
-        .addConfig(
-            "conf", TEST_PATH().resolve("configsets").resolve("ubi-enabled").resolve("conf"))
+        .addConfig("conf", TEST_PATH().resolve("configsets").resolve("ubi-enabled").resolve("conf"))
         .configure();
 
     String collection;
-    useAlias = false; //random().nextBoolean();
+    useAlias = false; // random().nextBoolean();
     if (useAlias) {
       collection = COLLECTIONORALIAS + "_collection";
     } else {
@@ -85,14 +68,17 @@ public static void setupCluster() throws Exception {
 
     // -------------------
 
-    CollectionAdminRequest.createCollection("ubi_queries",// it seems like a hardcoded name why?
-                    "_default", 1, 1)
-            .process(cluster.getSolrClient());
+    CollectionAdminRequest.createCollection(
+            "ubi_queries", // it seems like a hardcoded name why?
+            "_default",
+            1,
+            1)
+        .process(cluster.getSolrClient());
 
     cluster.waitForActiveCollection("ubi_queries", 1, 1);
 
     AbstractDistribZkTestBase.waitForRecoveriesToFinish(
-            "ubi_queries", cluster.getZkStateReader(), false, true, TIMEOUT);
+        "ubi_queries", cluster.getZkStateReader(), false, true, TIMEOUT);
   }
 
   @Before
@@ -102,22 +88,33 @@ public void cleanIndex() throws Exception {
 
   @Test
   public void testUBIQueryStream() throws Exception {
-    cluster.getSolrClient(COLLECTIONORALIAS).add(List.of(new SolrInputDocument("id", "1", "subject", "aa"),
-            new SolrInputDocument("id", "2" /*"two"*/, "subject", "aa"),
-            new SolrInputDocument("id", "3", "subject", "aa")));
+    cluster
+        .getSolrClient(COLLECTIONORALIAS)
+        .add(
+            List.of(
+                new SolrInputDocument("id", "1", "subject", "aa"),
+                new SolrInputDocument("id", "2" /*"two"*/, "subject", "aa"),
+                new SolrInputDocument("id", "3", "subject", "aa")));
     cluster.getSolrClient(COLLECTIONORALIAS).commit(true, true);
-    QueryResponse queryResponse = cluster.getSolrClient(COLLECTIONORALIAS).query(new MapSolrParams(
-            Map.of("q", "aa", "df","subject", "rows", "2", "ubi", "true"
-            )));
+    QueryResponse queryResponse =
+        cluster
+            .getSolrClient(COLLECTIONORALIAS)
+            .query(
+                new MapSolrParams(Map.of("q", "aa", "df", "subject", "rows", "2", "ubi", "true")));
     String qid = (String) ((SimpleMap) queryResponse.getResponse().get("ubi")).get("query_id");
-    assertTrue(qid.length()>10);
+    assertTrue(qid.length() > 10);
     Thread.sleep(10000); // I know what you think of
     // TODO check that ids were recorded
-    QueryResponse queryCheck = cluster.getSolrClient("ubi_queries").query(new MapSolrParams(
-            Map.of("q", "id:"+qid //doesn't search it why? is it a race?
-            )));
+    QueryResponse queryCheck =
+        cluster
+            .getSolrClient("ubi_queries")
+            .query(
+                new MapSolrParams(
+                    Map.of(
+                        "q", "id:" + qid // doesn't search it why? is it a race?
+                        )));
     // however I can't see doc ids found there. Shouldn't I ?
     assertEquals(1L, queryCheck.getResults().getNumFound());
-    assertEquals(queryCheck.getResults().get(0).get("id"),qid);
+    assertEquals(queryCheck.getResults().get(0).get("id"), qid);
   }
 }

From a44abb6fc014095b50ca43345d72db6e0b28112c Mon Sep 17 00:00:00 2001
From: Mikhail Khludnev 
Date: Tue, 10 Dec 2024 16:46:16 +0300
Subject: [PATCH 103/103] extract method

---
 .../solr/handler/component/UBIComponent.java  | 47 ++++++++-----------
 1 file changed, 19 insertions(+), 28 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
index fc5756f24bb..13c892be4d7 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/UBIComponent.java
@@ -36,6 +36,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.function.Consumer;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
@@ -247,7 +248,16 @@ public void process(ResponseBuilder rb) throws IOException {
       return;
     }
     if (!isInternalShardRequest(rb.req)) { // subordinate shard req shouldn't yield logs
-      doStuff(rb);
+      storeUbiDetails(
+          rb,
+          ubiQuery -> {
+            try {
+              DocList docList = ((ResultContext) rb.rsp.getResponse()).getDocList();
+              ubiQuery.setDocIds(extractDocIds(docList, rb.req.getSearcher()));
+            } catch (IOException e) {
+              throw new RuntimeException(e);
+            }
+          });
     }
   }
 
@@ -263,32 +273,18 @@ public int distributedProcess(ResponseBuilder rb) throws IOException {
     }
 
     if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
-      doDistribStuff(rb);
+      storeUbiDetails(
+          rb,
+          ubiQuery ->
+              ubiQuery.setDocIds(
+                  String.join(",", rb.resultIds.keySet().stream().map(Object::toString).toList())));
       return ResponseBuilder.STAGE_DONE;
     }
 
     return ResponseBuilder.STAGE_DONE;
   }
 
-  public void doStuff(ResponseBuilder rb) throws IOException {
-    UBIQuery ubiQuery = getUbiQuery(rb);
-    if (ubiQuery == null) return;
-
-    ResultContext rc = (ResultContext) rb.rsp.getResponse();
-    DocList docs = rc.getDocList();
-    // DocList docs = rb.getResults().docList;
-
-    String docIds = extractDocIds(docs, rb.req.getSearcher());
-
-    ubiQuery.setDocIds(docIds);
-
-    addUserBehaviorInsightsToResponse(ubiQuery, rb);
-    recordQuery(ubiQuery);
-  }
-
   private static UBIQuery getUbiQuery(ResponseBuilder rb) {
-    // not sure why but sometimes we get it tw(o)ice...  how can a response have the
-    // the same component run twice?
     if (rb.rsp.getValues().get("ubi") != null) {
       return null;
     }
@@ -328,16 +324,11 @@ private static UBIQuery getUbiQuery(ResponseBuilder rb) {
     return ubiQuery;
   }
 
-  public void doDistribStuff(ResponseBuilder rb) throws IOException {
-
-    // not sure why but sometimes we get it tw(o)ice...  how can a response have the
-    // the same component run twice?
+  private void storeUbiDetails(ResponseBuilder rb, Consumer docIdsSetter)
+      throws IOException {
     UBIQuery ubiQuery = getUbiQuery(rb);
     if (ubiQuery == null) return;
-    // String docIds = extractDocIds(docs, searcher);
-    String docIds = String.join(",", rb.resultIds.keySet().stream().map(Object::toString).toList());
-    ubiQuery.setDocIds(docIds);
-
+    docIdsSetter.accept(ubiQuery);
     addUserBehaviorInsightsToResponse(ubiQuery, rb);
     recordQuery(ubiQuery);
   }