Merge branch '3757/add_node_names_to_db'

READ_BEFORE_UPDATE.md

@@ -1,5 +1,10 @@
 # Update Notes
 
+## Update from 3.9 to 3.10
+
+* The `PI_NODES` configuration option is not used anymore. The nodes will be added
+  to the database with a unique identifier for each installation
+
 ## Update from 3.8 to 3.9
 
 * The response of the API `POST /auth` has changed if the WebUI policy action

doc/conf.py

@@ -76,7 +76,7 @@ def wrapper(decorator):
 
 # General information about the project.
 project = u'privacyIDEA'
-copyright = u'2014-2023, Cornelius Kölbel'
+copyright = u'2014-2024, Cornelius Kölbel'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/installation/system/inifile.rst

@@ -226,13 +226,27 @@ privacyIDEA Nodes
 -----------------
 
 privacyIDEA can run in a redundant setup. For statistics and monitoring purposes you
-can give these different nodes, dedicated names.
+can give these different nodes dedicated names.
 
-``PI_NODE`` is a string with the name of this very node. ``PI_NODES`` is a list of
-all available nodes in the cluster.
+``PI_NODE`` is a string with the name of this very node. At the startup of
+privacyIDEA, an installation specific unique ID will be used to tie the
+node name to an installation. The administrator can set a unique ID for this
+installation as well with the ``PI_UUID`` configuration value (it must conform to
+`RFC 4122 <https://datatracker.ietf.org/doc/html/rfc4122.html>`_).
+
+If no ``PI_UUID`` is configured, privacyIDEA tries to read the ID from a file.
+The administrator can specify the file with ``PI_UUID_FILE``. The default value
+is ``/etc/privacyidea/uuid.txt``. If this file does not provide an ID, the
+``/etc/machine-id``  will be used. And if all fails, a unique ID will be generated
+and made persistent in the ``PI_UUID_FILE``.
+
+Before version 3.10, the available nodes of the setup were defined with the
+``PI_NODES`` configuration value. Since version 3.10, this configuration value
+is not used anymore. The names of all nodes
+in a redundant setup will be made available through the database.
 
 If ``PI_NODE`` is not set, then ``PI_AUDIT_SERVERNAME`` is used as node name.
-If this is also not set, the node name is returned as "localnode".
+If this is not set as well, the node name is returned as "localnode".
 
 .. _trusted_jwt:
 

migrations/versions/e3a64b4ca634_.py

@@ -0,0 +1,46 @@
+"""v3.10: Add table nodename
+
+Revision ID: e3a64b4ca634
+Revises: 5cb310101a1f
+Create Date: 2023-11-23 11:41:30.149153
+
+"""
+from datetime import datetime
+from dateutil.tz import tzutc
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.exc import OperationalError, ProgrammingError
+
+# revision identifiers, used by Alembic.
+revision = 'e3a64b4ca634'
+down_revision = '5cb310101a1f'
+
+
+def upgrade():
+    try:
+        # ### commands auto generated by Alembic - please adjust! ###
+        op.create_table('nodename',
+                        sa.Column('id', sa.Unicode(length=36), nullable=False),
+                        sa.Column('name', sa.Unicode(length=100), nullable=True),
+                        sa.Column('lastseen', sa.DateTime(), nullable=True, default=datetime.now(tz=tzutc())),
+                        sa.PrimaryKeyConstraint('id')
+                        )
+        op.create_index(op.f('ix_nodename_lastseen'), 'nodename', ['lastseen'], unique=False)
+        op.create_index(op.f('ix_nodename_name'), 'nodename', ['name'], unique=False)
+        # ### end Alembic commands ###
+    except (OperationalError, ProgrammingError) as exx:
+        if "already exists" in str(exx.orig).lower():
+            print("Ok, Table 'nodename' already exists.")
+        else:
+            print(exx)
+    except Exception as exx:
+        print("Could not add table 'nodename' to database")
+        print(exx)
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_nodename_name'), table_name='nodename')
+    op.drop_index(op.f('ix_nodename_lastseen'), table_name='nodename')
+    op.drop_table('nodename')
+    # ### end Alembic commands ###

privacyidea/app.py

@@ -19,16 +19,20 @@
 # You should have received a copy of the GNU Affero General Public
 # License along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
+import datetime
 import os
 import os.path
 import logging
 import logging.config
 import sys
+import uuid
+
 import yaml
 from flask import Flask, request, Response
 from flask_babel import Babel
 from flask_migrate import Migrate
 from flaskext.versioned import Versioned
+import sqlalchemy as sa
 
 # we need this import to add the before/after request function to the blueprints
 import privacyidea.api.before_after
@@ -65,12 +69,16 @@
 from privacyidea.lib import queue
 from privacyidea.lib.log import DEFAULT_LOGGING_CONFIG
 from privacyidea.config import config
-from privacyidea.models import db
+from privacyidea.models import db, NodeName
 from privacyidea.lib.crypto import init_hsm
 
 
 ENV_KEY = "PRIVACYIDEA_CONFIGFILE"
 
+DEFAULT_UUID_FILE = "/etc/privacyidea/uuid.txt"
+
+migrate = Migrate()
+
 
 class PiResponseClass(Response):
     """Custom Response class overwriting the flask.Response.
@@ -91,7 +99,7 @@ def json(self):
 
 def create_app(config_name="development",
                config_file='/etc/privacyidea/pi.cfg',
-               silent=False, init_hsm=False):
+               silent=False, initialize_hsm=False):
     """
     First the configuration from the config.py is loaded depending on the
     config type like "production" or "development" or "testing".
@@ -107,8 +115,8 @@ def create_app(config_name="development",
     :param silent: If set to True the additional information are not printed
         to stdout
     :type silent: bool
-    :param init_hsm: Whether the HSM should be initialized on app startup
-    :type init_hsm: bool
+    :param initialize_hsm: Whether the HSM should be initialized on app startup
+    :type initialize_hsm: bool
     :return: The flask application
     :rtype: App object
     """
@@ -124,9 +132,6 @@ def create_app(config_name="development",
     if config_name:
         app.config.from_object(config[config_name])
 
-    # Set up flask-versioned
-    versioned = Versioned(app, format='%(path)s?v=%(version)s')
-
     try:
         # Try to load the given config_file.
         # If it does not exist, just ignore it.
@@ -178,8 +183,19 @@ def create_app(config_name="development",
     app.register_blueprint(monitoring_blueprint, url_prefix='/monitoring')
     app.register_blueprint(tokengroup_blueprint, url_prefix='/tokengroup')
     app.register_blueprint(serviceid_blueprint, url_prefix='/serviceid')
+
+    # Set up Plug-Ins
     db.init_app(app)
-    migrate = Migrate(app, db)
+    migrate.init_app(app, db)
+
+    Versioned(app, format='%(path)s?v=%(version)s')
+
+    babel = Babel()
+    babel.init_app(app)
+
+    @babel.localeselector
+    def get_locale():
+        return get_accepted_language(request)
 
     app.response_class = PiResponseClass
 
@@ -218,18 +234,65 @@ def create_app(config_name="development",
         DEFAULT_LOGGING_CONFIG["loggers"]["privacyidea"]["level"] = level
         logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
 
-    babel = Babel(app)
-
-    @babel.localeselector
-    def get_locale():
-        return get_accepted_language(request)
-
     queue.register_app(app)
 
-    if init_hsm:
+    if initialize_hsm:
         with app.app_context():
             init_hsm()
 
+    # check that we have a correct node_name -> UUID relation
+    with app.app_context():
+        # first check if we have a UUID in the config file which takes precedence
+        try:
+            pi_uuid = uuid.UUID(app.config.get("PI_UUID", ""))
+        except ValueError as e:
+            logging.getLogger(__name__).debug(f"Could not determine UUID from config: {e}")
+            # check if we can get the UUID from an external file
+            pi_uuid_file = app.config.get('PI_UUID_FILE', DEFAULT_UUID_FILE)
+            try:
+                with open(pi_uuid_file, 'r') as f:
+                    pi_uuid = uuid.UUID(f.read().strip())
+            except Exception as e:  # pragma: no cover
+                logging.getLogger(__name__).debug(f"Could not determine UUID "
+                                                  f"from file '{pi_uuid_file}': {e}")
+
+                # we try to get the unique installation id (See <https://0pointer.de/blog/projects/ids.html>)
+                try:
+                    with open("/etc/machine-id", 'r') as f:
+                        pi_uuid = uuid.UUID(f.read().strip())
+                except Exception as e:  # pragma: no cover
+                    logging.getLogger(__name__).debug(f"Could not determine the machine "
+                                                      f"id: {e}")
+                    # we generate a random UUID which will change on every startup
+                    # unless it is persisted to the pi_uuid_file
+                    pi_uuid = uuid.uuid4()
+                    logging.getLogger(__name__).warning(f"Generating a random UUID: {pi_uuid}! "
+                                                        f"If persisting the UUID fails, "
+                                                        f"it will change on every application start")
+                    # only in case of a generated UUID we save it to the uuid file
+                    try:
+                        with open(pi_uuid_file, 'a') as f:  # pragma: no cover
+                            f.write(f"{str(pi_uuid)}\n")
+                            logging.getLogger(__name__).info(f"Successfully wrote current UUID"
+                                                             f" to file '{pi_uuid_file}'")
+                    except IOError as exx:
+                        logging.getLogger(__name__).warning(f"Could not write UUID to "
+                                                            f"file '{pi_uuid_file}': {exx}")
+
+            app.config["PI_UUID"] = str(pi_uuid)
+            logging.getLogger(__name__).debug(f"Current UUID: '{pi_uuid}'")
+
+        pi_node_name = app.config.get("PI_NODE") or app.config.get("PI_AUDIT_SERVERNAME", "localnode")
+
+        insp = sa.inspect(db.get_engine())
+        if insp.has_table(NodeName.__tablename__):
+            db.session.merge(NodeName(id=str(pi_uuid), name=pi_node_name,
+                                      lastseen=datetime.datetime.utcnow()))
+            db.session.commit()
+        else:
+            logging.getLogger(__name__).warning(f"Could not update node names in "
+                                                f"db. Check that table '{NodeName.__tablename__}' exists.")
+
     logging.getLogger(__name__).debug("Reading application from the static "
                                       "folder {0!s} and the template folder "
                                       "{1!s}".format(app.static_folder, app.template_folder))

privacyidea/config.py

@@ -85,8 +85,8 @@ class TestingConfig(Config):
     CACHE_TYPE = "None"
     PI_SCRIPT_HANDLER_DIRECTORY = "tests/testdata/scripts/"
     PI_NOTIFICATION_HANDLER_SPOOLDIRECTORY = "tests/testdata/"
+    PI_UUID = "8e4272a9-9037-40df-8aa3-976e4a04b5a9"
     PI_NODE = "Node1"
-    PI_NODES = ["Node1", "Node2"]
     PI_ENGINE_REGISTRY_CLASS = "null"
     PI_TRUSTED_JWT = [{"public_key": pubtest_key,
                        "algorithm": "HS256",

privacyidea/lib/config.py

@@ -42,7 +42,8 @@
 
 from .log import log_with
 from ..models import (Config, db, Resolver, Realm, PRIVACYIDEA_TIMESTAMP,
-                      save_config_timestamp, Policy, EventHandler, CAConnector)
+                      save_config_timestamp, Policy, EventHandler, CAConnector,
+                      NodeName)
 from privacyidea.lib.framework import get_request_local_store, get_app_config_value, get_app_local_store
 from privacyidea.lib.utils import to_list
 from privacyidea.lib.utils.export import (register_import, register_export)
@@ -988,7 +989,9 @@ def get_privacyidea_node(default='localnode'):
     This returns the node name of the privacyIDEA node as found in the pi.cfg
     file in PI_NODE.
     If it does not exist, the PI_AUDIT_SERVERNAME is used.
+
     :return: the distinct node name
+    :rtype: str
     """
     node_name = get_app_config_value("PI_NODE", get_app_config_value("PI_AUDIT_SERVERNAME", default))
     return node_name
@@ -997,13 +1000,16 @@ def get_privacyidea_node(default='localnode'):
 def get_privacyidea_nodes():
     """
     This returns the list of the nodes, including the own local node name
+
     :return: list of nodes
+    :rtype: list
     """
-    own_node_name = get_privacyidea_node()
-    nodes = get_app_config_value("PI_NODES", [])[:]
-    if own_node_name not in nodes:
-        nodes.append(own_node_name)
-    return nodes
+    node_names = []
+    nodes = db.session.query(NodeName).all()
+    for node in nodes:
+        node_names.append(node.name)
+
+    return node_names
 
 
 @register_export()