[#4396] Moved public functions to prefill.service and added prefill p…

…lugin for ObjectsApi

src/openforms/contrib/haal_centraal/tests/test_integration.py

@@ -8,8 +8,8 @@
 from openforms.authentication.service import AuthAttribute
 from openforms.authentication.utils import store_auth_details, store_registrator_details
 from openforms.config.models import GlobalConfiguration
-from openforms.prefill import prefill_variables
 from openforms.prefill.contrib.haalcentraal_brp.plugin import PLUGIN_IDENTIFIER
+from openforms.prefill.service import prefill_variables
 from openforms.submissions.tests.factories import SubmissionFactory
 from openforms.typing import JSONValue
 from openforms.utils.tests.vcr import OFVCRMixin

src/openforms/formio/service.py

@@ -14,7 +14,6 @@
 import elasticapm
 from rest_framework.request import Request
 
-from openforms.prefill import inject_prefill
 from openforms.submissions.models import Submission
 from openforms.typing import DataMapping
 
@@ -73,6 +72,9 @@ def get_dynamic_configuration(
  The configuration is modified in the context of the provided ``submission``
  parameter.
  """
+ # Avoid circular imports
+ from openforms.prefill.service import inject_prefill
+
  rewrite_formio_components(config_wrapper, submission=submission, data=data)
 
  # Add to each component the custom errors in the current locale

src/openforms/formio/tests/test_component_translations.py

@@ -28,7 +28,10 @@ def disable_prefill_injection():
  """
  Disable prefill to prevent prefill-related queries.
  """
- return patch("openforms.formio.service.inject_prefill", new=MagicMock)
+ return patch(
+ "openforms.prefill.service.inject_prefill",
+ new=MagicMock,
+ )
 
 
 TEST_CONFIGURATION = {

src/openforms/forms/migrations/0101_remove_formvariable_prefill_config_empty_or_complete_and_more.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.16 on 2024-10-02 07:06
+# Generated by Django 4.2.16 on 2024-10-02 10:03
 
 from django.db import migrations, models
 
@@ -34,19 +34,19 @@ class Migration(migrations.Migration):
  models.Q(
  models.Q(("prefill_plugin", ""), _negated=True),
  ("prefill_attribute", ""),
- ("prefill_options", {}),
+ models.Q(("prefill_options", {}), _negated=True),
  ("source", "user_defined"),
  ),
  models.Q(
  models.Q(("prefill_plugin", ""), _negated=True),
- models.Q(("prefill_attribute", ""), _negated=True),
+ ("prefill_attribute", ""),
  ("prefill_options", {}),
+ models.Q(("source", "user_defined"), _negated=True),
  ),
  models.Q(
  models.Q(("prefill_plugin", ""), _negated=True),
- ("prefill_attribute", ""),
- models.Q(("prefill_options", {}), _negated=True),
- ("source", "user_defined"),
+ models.Q(("prefill_attribute", ""), _negated=True),
+ ("prefill_options", {}),
  ),
  _connector="OR",
  )

src/openforms/forms/models/form_variable.py

@@ -216,19 +216,19 @@ class Meta:
  | (
  ~EMPTY_PREFILL_PLUGIN
  & EMPTY_PREFILL_ATTRIBUTE
- & EMPTY_PREFILL_OPTIONS
+ & ~EMPTY_PREFILL_OPTIONS
  & USER_DEFINED
  )
  | (
  ~EMPTY_PREFILL_PLUGIN
- & ~EMPTY_PREFILL_ATTRIBUTE
+ & EMPTY_PREFILL_ATTRIBUTE
  & EMPTY_PREFILL_OPTIONS
+ & ~USER_DEFINED
  )
  | (
  ~EMPTY_PREFILL_PLUGIN
- & EMPTY_PREFILL_ATTRIBUTE
- & ~EMPTY_PREFILL_OPTIONS
- & USER_DEFINED
+ & ~EMPTY_PREFILL_ATTRIBUTE
+ & EMPTY_PREFILL_OPTIONS
  )
  ),
  name="prefill_config_component_or_user_defined",

src/openforms/forms/tests/variables/test_model.py

@@ -32,10 +32,17 @@ def test_prefill_plugin_prefill_attribute_prefill_options_empty(self):
  )
 
  def test_prefill_attribute_prefill_options_empty(self):
- FormVariableFactory.create(
- prefill_plugin="demo",
- prefill_attribute="",
- prefill_options={},
+ FormStepFactory.create(
+ form_definition__configuration={
+ "components": [
+ {
+ "type": "textfield",
+ "key": "test-key",
+ "label": "Test label",
+ "prefill": {"plugin": "demo", "attribute": ""},
+ }
+ ]
+ }
  )
 
  def test_prefill_options_empty(self):

src/openforms/prefill/__init__.py

@@ -1,206 +0,0 @@
-"""
-This package holds the base module structure for the pre-fill plugins used in Open Forms.
-
-Various sources exist that can be consulted to fetch data for an active session,
-where the BSN, CoC number... can be used to retrieve this data. Think of pre-filling
-the address details of a person after logging in with DigiD.
-
-The package integrates with the form builder such that it's possible for every form
-field to select which pre-fill plugin to use and which value to use from the fetched
-result. Plugins can be registered using a similar approach to the registrations
-package. Each plugin is responsible for exposing which attributes/data fragments are
-available, and for performing the actual look-up. Plugins receive the
-:class:`openforms.submissions.models.Submission` instance that represents the current
-form session of an end-user.
-
-Prefill values are embedded as default values for form fields, dynamically for every
-user session using the component rewrite functionality in the serializers.
-
-So, to recap:
-
-1. Plugins are defined and registered
-2. When editing form definitions in the admin, content editors can opt-in to pre-fill
- functionality. They select the desired plugin, and then the desired attribute from
- that plugin.
-3. End-user starts the form and logs in, thereby creating a session/``Submission``
-4. The submission-specific form definition configuration is enhanced with the pre-filled
- form field default values.
-
-.. todo:: Move the public API into ``openforms.prefill.service``.
-
-"""
-
-from __future__ import annotations
-
-import logging
-from collections import defaultdict
-from typing import TYPE_CHECKING, Any
-
-import elasticapm
-from glom import Path, PathAccessError, assign, glom
-from zgw_consumers.concurrent import parallel
-
-from openforms.plugins.exceptions import PluginNotEnabled
-from openforms.variables.constants import FormVariableSources
-
-if TYPE_CHECKING:
- from openforms.formio.service import FormioConfigurationWrapper
- from openforms.submissions.models import Submission
-
- from .registry import Registry
-
-logger = logging.getLogger(__name__)
-
-
-@elasticapm.capture_span(span_type="app.prefill")
-def _fetch_prefill_values(
- grouped_fields: dict[str, dict[str, list[str]]],
- submission: Submission,
- register: Registry,
-) -> dict[str, dict[str, Any]]:
- # local import to prevent AppRegistryNotReady:
- from openforms.logging import logevent
-
- @elasticapm.capture_span(span_type="app.prefill")
- def invoke_plugin(
- item: tuple[str, str, list[str]]
- ) -> tuple[str, str, dict[str, Any]]:
- plugin_id, identifier_role, fields = item
-
- plugin = register[plugin_id]
- if not plugin.is_enabled:
- raise PluginNotEnabled()
-
- try:
- values = plugin.get_prefill_values(submission, fields, identifier_role)
- except Exception as e:
- logger.exception(f"exception in prefill plugin '{plugin_id}'")
- logevent.prefill_retrieve_failure(submission, plugin, e)
- values = {}
- else:
- if values:
- logevent.prefill_retrieve_success(submission, plugin, fields)
- else:
- logevent.prefill_retrieve_empty(submission, plugin, fields)
-
- return plugin_id, identifier_role, values
-
- invoke_plugin_args = []
- for plugin_id, field_groups in grouped_fields.items():
- for identifier_role, fields in field_groups.items():
- invoke_plugin_args.append((plugin_id, identifier_role, fields))
-
- with parallel() as executor:
- results = executor.map(invoke_plugin, invoke_plugin_args)
-
- collected_results = {}
- for plugin_id, identifier_role, values_dict in list(results):
- assign(
- collected_results,
- Path(plugin_id, identifier_role),
- values_dict,
- missing=dict,
- )
-
- return collected_results
-
-
-def inject_prefill(
- configuration_wrapper: FormioConfigurationWrapper, submission: Submission
-) -> None:
- """
- Mutates each component found in configuration according to the prefilled values.
-
- :param configuration_wrapper: The Formiojs JSON schema wrapper describing an entire
- form or an individual component within the form.
- :param submission: The :class:`openforms.submissions.models.Submission` instance
- that holds the values of the prefill data. The prefill data was fetched earlier,
- see :func:`prefill_variables`.
-
- The prefill values are looped over by key: value, and for each value the matching
- component is looked up to normalize it in the context of the component.
- """
- from openforms.formio.service import normalize_value_for_component
-
- prefilled_data = submission.get_prefilled_data()
- for key, prefill_value in prefilled_data.items():
- try:
- component = configuration_wrapper[key]
- except KeyError:
- # The component to prefill is not in this step
- continue
-
- if not (prefill := component.get("prefill")):
- continue
- if not prefill.get("plugin"):
- continue
- if not prefill.get("attribute"):
- continue
-
- default_value = component.get("defaultValue")
- # 1693: we need to normalize values according to the format expected by the
- # component. For example, (some) prefill plugins return postal codes without
- # space between the digits and the letters.
- prefill_value = normalize_value_for_component(component, prefill_value)
-
- if prefill_value != default_value and default_value is not None:
- logger.info(
- "Overwriting non-null default value for component %r",
- component,
- )
- component["defaultValue"] = prefill_value
-
-
-@elasticapm.capture_span(span_type="app.prefill")
-def prefill_variables(submission: Submission, register: Registry | None = None) -> None:
- """Update the submission variables state with the fetched attribute values.
-
- For each submission value variable that need to be prefilled, the according plugin will
- be used to fetch the value. If ``register`` is not specified, the default registry instance
- will be used.
- """
- from openforms.formio.service import normalize_value_for_component
-
- from .registry import register as default_register
-
- register = register or default_register
-
- state = submission.load_submission_value_variables_state()
- variables_to_prefill = state.get_prefill_variables()
-
- # grouped_fields is a dict of the following shape:
- # {"plugin_id": {"identifier_role": ["attr_1", "attr_2"]}}
- # "identifier_role" is either "main" or "authorizee"
- grouped_fields: defaultdict[str, defaultdict[str, list[str]]] = defaultdict(
- lambda: defaultdict(list)
- )
- for variable in variables_to_prefill:
- plugin_id = variable.form_variable.prefill_plugin
- identifier_role = variable.form_variable.prefill_identifier_role
- attribute_name = variable.form_variable.prefill_attribute
-
- grouped_fields[plugin_id][identifier_role].append(attribute_name)
-
- results = _fetch_prefill_values(grouped_fields, submission, register)
-
- total_config_wrapper = submission.total_configuration_wrapper
- prefill_data = {}
- for variable in variables_to_prefill:
- try:
- prefill_value = glom(
- results,
- Path(
- variable.form_variable.prefill_plugin,
- variable.form_variable.prefill_identifier_role,
- variable.form_variable.prefill_attribute,
- ),
- )
- except PathAccessError:
- continue
- else:
- if variable.form_variable.source == FormVariableSources.component:
- component = total_config_wrapper[variable.key]
- prefill_value = normalize_value_for_component(component, prefill_value)
- prefill_data[variable.key] = prefill_value
-
- state.save_prefill_data(prefill_data)

src/openforms/prefill/base.py

@@ -1,6 +1,7 @@
 from typing import Any, Container, Iterable
 
 from openforms.authentication.service import AuthAttribute
+from openforms.forms.models import FormVariable
 from openforms.plugins.plugin import AbstractBasePlugin
 from openforms.submissions.models import Submission
 from openforms.typing import JSONEncodable, JSONObject
@@ -50,6 +51,26 @@ def get_prefill_values(
  """
  raise NotImplementedError("You must implement the 'get_prefill_values' method.")
 
+ @classmethod
+ def get_prefill_values_from_mappings(
+ cls, submission: Submission, form_variable: FormVariable
+ ) -> dict[str, str]:
+ """
+ Given the saved form variable, which contains the prefill_options, look up the appropriate
+ values and return them.
+
+ :param submission: an active :class:`Submission` instance, which can supply
+ the required initial data reference to fetch the correct prefill values.
+ :param form_variable: The form variable for which we want to retrieve the data. Its
+ atribute prefill_options contains all the mappings that are needed for retrieving
+ and returning the values.
+ :return: a key-value dictionary, where the key is the mapped property and
+ the value is the prefill value to use for that property.
+ """
+ raise NotImplementedError(
+ "You must implement the 'get_prefill_values_from_mappings' method."
+ )
+
  @classmethod
  def get_co_sign_values(
  cls, submission: Submission, identifier: str