[#8568] support basic auth to solr

Allura/allura/command/show_models.py

@@ -79,6 +79,8 @@ class ReindexCommand(base.Command):
                            'which are needed for some markdown macros to run properly')
     parser.add_option('--solr-hosts', dest='solr_hosts',
                       help='Override the solr host(s) to post to.  Comma-separated list of solr server URLs')
+    parser.add_option('--solr-creds', dest='solr_creds',
+                      help='Creds for the solr host(s).  Comma-separated list of user:pwd strings')
     parser.add_option(
         '--max-chunk', dest='max_chunk', type=int, default=100 * 1000,
         help='Max number of artifacts to index in one Solr update command')
@@ -148,9 +150,13 @@ def command(self):
 
     @property
     def add_artifact_kwargs(self):
+        kwargs = {}
         if self.options.solr_hosts:
-            return {'solr_hosts': self.options.solr_hosts.split(',')}
-        return {}
+            kwargs['solr_hosts'] = self.options.solr_hosts.split(',')
+        if self.options.solr_creds:
+            kwargs['solr_creds'] = [cred.split(':')
+                                    for cred in self.options.solr_creds.split(',')]
+        return kwargs
 
     def _chunked_add_artifacts(self, ref_ids):
         # ref_ids contains solr index ids which can easily be over

Allura/allura/lib/app_globals.py

@@ -203,13 +203,22 @@ def __init__(self):
         self.solr_server = aslist(config.get('solr.server'), ',')
         # skip empty strings in case of extra commas
         self.solr_server = [s for s in self.solr_server if s]
+        push_auths = zip(aslist(config.get('solr.user'), ','),
+                         aslist(config.get('solr.pass'), ','))
         self.solr_query_server = config.get('solr.query_server')
+        query_auth = (config.get('solr.query_user'), config.get('solr.query_pass'))
         if self.solr_server:
             self.solr = make_solr_from_config(
-                self.solr_server, self.solr_query_server)
+                self.solr_server, self.solr_query_server,
+                push_servers_auths=push_auths,
+                query_server_auth=query_auth,
+            )
             self.solr_short_timeout = make_solr_from_config(
                 self.solr_server, self.solr_query_server,
-                timeout=int(config.get('solr.short_timeout', 10)))
+                push_servers_auths=push_auths,
+                query_server_auth=query_auth,
+                timeout=int(config.get('solr.short_timeout', 10)),
+            )
         else:  # pragma no cover
             log.warning('Solr config not set; using in-memory MockSOLR')
             self.solr = self.solr_short_timeout = MockSOLR()

Allura/allura/lib/solr.py

@@ -14,10 +14,12 @@
 #       KIND, either express or implied.  See the License for the
 #       specific language governing permissions and limitations
 #       under the License.
-
+from __future__ import annotations
 
 import json
 import logging
+from itertools import zip_longest
+from collections.abc import Iterable
 
 from tg import config
 from webob.exc import HTTPRequestEntityTooLarge
@@ -61,7 +63,10 @@ def escape_solr_arg(term):
     return term
 
 
-def make_solr_from_config(push_servers, query_server=None, **kwargs):
+def make_solr_from_config(push_servers: Iterable[str], query_server: str|None=None,
+                          push_servers_auths: Iterable[tuple[str, str] | None] = (),
+                          query_server_auth: tuple[str, str] | None = None,
+                          **kwargs):
     """
     Make a :class:`Solr <Solr>` instance from config defaults.  Use
     `**kwargs` to override any value
@@ -72,7 +77,7 @@ def make_solr_from_config(push_servers, query_server=None, **kwargs):
         timeout=int(config.get('solr.long_timeout', 60)),
     )
     solr_kwargs.update(kwargs)
-    return Solr(push_servers, query_server, **solr_kwargs)
+    return Solr(push_servers, query_server, push_servers_auths, query_server_auth, **solr_kwargs)
 
 
 class Solr:
@@ -87,13 +92,18 @@ class Solr:
     unless explicitly overridden.
     """
 
-    def __init__(self, push_servers, query_server=None,
+    def __init__(self, push_servers: Iterable[str], query_server: str|None = None,
+                 push_servers_auths: Iterable[tuple[str, str] | None] = (),
+                 query_server_auth: tuple[str, str] | None = None,
                  commit=True, commitWithin=None, **kw):
-        self.push_pool = [pysolr.Solr(s, **kw) for s in push_servers]
+        self.push_pool = [pysolr.Solr(s, auth=auth, **kw)
+                          for s, auth in zip_longest(push_servers, push_servers_auths)]
         if query_server:
-            self.query_server = pysolr.Solr(query_server, **kw)
+            self.query_server = pysolr.Solr(query_server, auth=query_server_auth, **kw)
         else:
             self.query_server = self.push_pool[0]
+            if query_server_auth:
+                self.query_server.auth = query_server_auth
         self._commit = commit
         self.commitWithin = commitWithin
 

Allura/allura/tasks/index_tasks.py

@@ -18,6 +18,7 @@
 
 import sys
 import logging
+from collections.abc import Iterable
 from contextlib import contextmanager
 import typing
 
@@ -38,12 +39,12 @@
 log = logging.getLogger(__name__)
 
 
-def __get_solr(solr_hosts=None):
-    return make_solr_from_config(solr_hosts) if solr_hosts else g.solr
+def __get_solr(solr_hosts=None, solr_creds=()):
+    return make_solr_from_config(solr_hosts, push_servers_auths=solr_creds) if solr_hosts else g.solr
 
 
-def __add_objects(objects, solr_hosts=None):
-    solr_instance = __get_solr(solr_hosts)
+def __add_objects(objects, solr_hosts=None, solr_creds=()):
+    solr_instance = __get_solr(solr_hosts, solr_creds)
     solr_instance.add([obj.solarize() for obj in objects])
 
 
@@ -94,16 +95,19 @@ def del_users(user_solr_ids):
 
 
 @task
-def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
+def add_artifacts(ref_ids, update_solr=True, update_refs=True,
+                  solr_hosts: Iterable[str] = (),
+                  solr_creds: Iterable[tuple[str, str]] = (),
+                  ):
     '''
     Add the referenced artifacts to SOLR and shortlinks.
-
-    :param solr_hosts: a list of solr hosts to use instead of the defaults
-    :type solr_hosts: [str]
     '''
     from allura import model as M
     from allura.lib.search import find_shortlinks
 
+    # task params end up as instrumented lists, need to make this a list of plain tuples
+    solr_creds = [tuple(cred) for cred in solr_creds]
+
     exceptions = []
     solr_updates = []
     with _indexing_disabled(M.session.artifact_orm_session._get()):
@@ -146,7 +150,7 @@ def _add_artifact(solr: pysolr.Solr, artifacts: list):
                     log.info("Solr.add raised HTTPRequestEntityTooLarge but there is only one artifact. Raising exception.")
                     raise
 
-        _add_artifact(__get_solr(solr_hosts), solr_updates)
+        _add_artifact(__get_solr(solr_hosts, solr_creds), solr_updates)
 
     if len(exceptions) == 1:
         raise exceptions[0][1].with_traceback(exceptions[0][2])

Allura/allura/tests/test_commands.py

@@ -532,7 +532,7 @@ def test_project_regex(self, utils):
     @patch('allura.command.show_models.add_artifacts')
     def test_chunked_add_artifacts(self, add_artifacts):
         cmd = show_models.ReindexCommand('reindex')
-        cmd.options = Mock(tasks=True, max_chunk=10 * 1000, ming_config=None)
+        cmd.options = Mock(tasks=True, max_chunk=10 * 1000, ming_config=None, solr_creds='')
         ref_ids = list(range(10 * 1000 * 2 + 20))
         cmd._chunked_add_artifacts(ref_ids)
         assert len(add_artifacts.post.call_args_list) == 3
@@ -575,7 +575,7 @@ def on_post(chunk, **kw):
             raise pymongo.errors.InvalidDocument("Cannot encode object...")
         add_artifacts.post.side_effect = on_post
         cmd = show_models.ReindexCommand('reindex')
-        cmd.options = Mock(ming_config=None)
+        cmd.options = Mock(ming_config=None, solr_creds='')
         with td.raises(pymongo.errors.InvalidDocument):
             cmd._post_add_artifacts(list(range(5)))
 

Allura/allura/tests/unit/test_solr.py

@@ -39,15 +39,18 @@ def setup_method(self, method):
     @mock.patch('allura.lib.solr.pysolr')
     def test_init(self, pysolr):
         servers = ['server1', 'server2']
-        solr = Solr(servers, commit=False, commitWithin='10000')
-        calls = [mock.call('server1'), mock.call('server2')]
+        auths = [('u', 'pwd'),]
+        solr = Solr(servers, push_servers_auths=auths, commit=False, commitWithin='10000')
+        calls = [mock.call('server1', auth=('u', 'pwd')), mock.call('server2', auth=None)]
         pysolr.Solr.assert_has_calls(calls)
         assert len(solr.push_pool) == 2
 
         pysolr.reset_mock()
         solr = Solr(servers, 'server3', commit=False, commitWithin='10000')
-        calls = [mock.call('server1'), mock.call('server2'),
-                 mock.call('server3')]
+        calls = [mock.call('server1', auth=None),
+                 mock.call('server2', auth=None),
+                 mock.call('server3', auth=None),
+        ]
         pysolr.Solr.assert_has_calls(calls)
         assert len(solr.push_pool) == 2
 

Allura/development.ini

@@ -571,10 +571,15 @@ stats.sample_rate = 1
 ; number of seconds to sleep between checking for new tasks
 monq.poll_interval=2
 
-; SOLR setup
+; SOLR setup.  This can be a comma-separated list, to index into multiple locations
 solr.server = http://localhost:8983/solr/allura
-; Alternate server to use just for querying
+# auth for solr, if needed.  These can be a list too, to match solr.server
+;solr.user =
+;solr.pass =
+; Alternate server to use just for querying.  Single server, not a list.
 ;solr.query_server =
+;solr.query_user =
+;solr.query_pass =
 ; Shorter timeout for search queries (longer timeout for saving to solr)
 solr.short_timeout = 10
 ; commit on every add/delete?