From 88c2d82c2ef2c9c122b9675eab7f35333fae1bd8 Mon Sep 17 00:00:00 2001 From: Nathan Carlson Date: Tue, 24 Sep 2019 13:54:22 -0700 Subject: [PATCH 1/2] Remove all client commits --- src/java/main/esg/search/publish/impl/solr/SolrClient.java | 4 ++-- src/java/main/esg/search/publish/impl/solr/SolrIndexer.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/java/main/esg/search/publish/impl/solr/SolrClient.java b/src/java/main/esg/search/publish/impl/solr/SolrClient.java index 20647ad4..f04609bf 100644 --- a/src/java/main/esg/search/publish/impl/solr/SolrClient.java +++ b/src/java/main/esg/search/publish/impl/solr/SolrClient.java @@ -83,7 +83,7 @@ public String index(final String xml, final String type, boolean commit) throws String response = httpClient.doPost(postUrl, xml, true); // commit changes, do not optimize for a single record - if (commit) this.commit(); + // if (commit) this.commit(); return response; @@ -133,7 +133,7 @@ public String delete(List ids, Collection cores) throws Exceptio } // commit changes to all cores - commit(); + // commit(); return sb.toString(); diff --git a/src/java/main/esg/search/publish/impl/solr/SolrIndexer.java b/src/java/main/esg/search/publish/impl/solr/SolrIndexer.java index 09de1bde..b56a1fdf 100644 --- a/src/java/main/esg/search/publish/impl/solr/SolrIndexer.java +++ b/src/java/main/esg/search/publish/impl/solr/SolrIndexer.java @@ -93,7 +93,7 @@ public void consume(final Collection records) throws Exception { } // commit all records at once, to all cores - solrClient.commit(); + // solrClient.commit(); } From c2d047c5924272497595a723bef84f08c27bfa0b Mon Sep 17 00:00:00 2001 From: Nathan Carlson Date: Fri, 27 Sep 2019 10:56:22 -0700 Subject: [PATCH 2/2] Comment on removal of commits Solr commits are heavy operations that put a significant amount of load on Solr instances. Solr provides "autoCommit" features that perform commits at regular intervals or after a set number of updates. --- src/java/main/esg/search/publish/impl/solr/SolrClient.java | 7 ++++++- .../main/esg/search/publish/impl/solr/SolrIndexer.java | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/java/main/esg/search/publish/impl/solr/SolrClient.java b/src/java/main/esg/search/publish/impl/solr/SolrClient.java index f04609bf..fcd95530 100644 --- a/src/java/main/esg/search/publish/impl/solr/SolrClient.java +++ b/src/java/main/esg/search/publish/impl/solr/SolrClient.java @@ -67,7 +67,7 @@ public SolrClient(final URL url) { * Method to index a single XML record. * @param xml. * @param type : chosen among the supported record types. - * @param commit : true to commit the transaction after indexing this record, false if other records are coming. + * @param commit : This parameter has no effect * @return * @throws Exception */ @@ -83,6 +83,10 @@ public String index(final String xml, final String type, boolean commit) throws String response = httpClient.doPost(postUrl, xml, true); // commit changes, do not optimize for a single record + // This results in a high frequency of commits + // Which puts non-negligable load on Solr instances + // Future work would be to make this externally configurable + // For now, Solr "autoCommit" features will be used to perform commits // if (commit) this.commit(); return response; @@ -133,6 +137,7 @@ public String delete(List ids, Collection cores) throws Exceptio } // commit changes to all cores + // Solr "autoCommit" features will be used to perform commits // commit(); return sb.toString(); diff --git a/src/java/main/esg/search/publish/impl/solr/SolrIndexer.java b/src/java/main/esg/search/publish/impl/solr/SolrIndexer.java index b56a1fdf..c3c246d0 100644 --- a/src/java/main/esg/search/publish/impl/solr/SolrIndexer.java +++ b/src/java/main/esg/search/publish/impl/solr/SolrIndexer.java @@ -93,6 +93,7 @@ public void consume(final Collection records) throws Exception { } // commit all records at once, to all cores + // Solr "autoCommit" features will be used to perform commits // solrClient.commit(); }