[ckan#7971] docs, review suggestions

changes/7971.feature

@@ -1 +1,8 @@
-IDataDictionaryForm for extending and validating datastore data dictionary fields
+IDataDictionaryForm for extending and validating new keys in the `fields`
+dicts. Unlike the `info` free-form dict these new keys are possible to
+tightly control with a schema. The schema is built by combining schemas
+from from all plugins implementing this interface so plugins implementing
+different features may all contribute to the same schema.
+
+The underlying storage for data dictionary fields has changed. Use:
+`ckan datastore upgrade` after upgrading to this release.

ckanext/datastore/backend/postgres.py

@@ -1078,16 +1078,17 @@ def create_table(
 
     info_sql = []
     for i, f in enumerate(supplied_fields):
-        ccom = plugin_data.get(i, {})
+        column_comment = plugin_data.get(i, {})
         info = f.get(u'info')
         if isinstance(info, dict):
-            ccom['_info'] = info
-        if ccom:
+            column_comment['_info'] = info
+        if column_comment:
             info_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format(
                 identifier(data_dict['resource_id']),
                 identifier(f['id']),
                 literal_string(' ' + json.dumps(  # ' ' prefix for data version
-                        ccom, ensure_ascii=False, separators=(',', ':')))))
+                    column_comment, ensure_ascii=False, separators=(',', ':')))
+            ))
 
     context['connection'].execute(sa.text(
         sql_string + u';'.join(info_sql).replace(':', r'\:')  # no bind params
@@ -1179,12 +1180,12 @@ def alter_table(
                 raw.update(plugin_data[i])
 
             # ' ' prefix for data version
-            raw_sql = literal_string(' ' + json.dumps(
+            column_comment = literal_string(' ' + json.dumps(
                 raw, ensure_ascii=False, separators=(',', ':')))
             alter_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format(
                 identifier(data_dict['resource_id']),
                 identifier(f['id']),
-                raw_sql))
+                column_comment))
 
     if data_dict['delete_fields']:
         for id_ in current_ids - field_ids - set(f['id'] for f in new_fields):

ckanext/datastore/interfaces.py

@@ -207,13 +207,13 @@ def update_datastore_create_schema(self, schema: Schema) -> Schema:
         data for that plugin.
 
         e.g. a statistics plugin that needs to store per-column information
-        might store this with plugin_data by inserting values like:
+        might store this with plugin_data by inserting values like::
 
-            {0: {'statistics': {'minimum': 34, ...}, ...}, ...}
+          {0: {'statistics': {'minimum': 34, ...}, ...}, ...}
 
-                                ^ the data stored for this field+plugin
-                  ^ the name of the plugin
-             ^ 0 for the first field passed in fields
+          #                   ^ the data stored for this field+plugin
+          #     ^ the name of the plugin
+          #^ 0 for the first field passed in fields
 
         Values not removed from field info by validation will be available in
         the field `info` dict returned from `datastore_search` and

ckanext/datastore/logic/validators.py

@@ -0,0 +1,49 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from ckan.types import (
+    Context, FlattenDataDict, FlattenKey, FlattenErrorDict,
+)
+from ckan.plugins.toolkit import missing
+
+
+def to_datastore_plugin_data(plugin_key: str):
+    """
+    Return a validator that will move values from data to
+    context['plugin_data'][field_index][plugin_key][field_name]
+
+    where field_index is the field number, plugin_key (passed to this
+    function) is typically set to the plugin name and field name is the
+    original field name being validated.
+    """
+
+    def validator(
+            key: FlattenKey,
+            data: FlattenDataDict,
+            errors: FlattenErrorDict,
+            context: Context):
+        value = data.pop(key)
+        field_index = key[-2]
+        field_name = key[-1]
+        context['plugin_data'].setdefault(
+            field_index, {}).setdefault(
+            plugin_key, {})[field_name] = value
+    return validator
+
+
+def datastore_default_current(
+        key: FlattenKey, data: FlattenDataDict,
+        errors: FlattenErrorDict, context: Context):
+    '''default to currently stored value if empty or missing'''
+    value = data[key]
+    if value is not None and value != '' and value is not missing:
+        return
+    field_index = key[-2]
+    field_name = key[-1]
+    # current values for plugin_data are available as
+    # context['plugin_data'][field_index]['_current']
+    current = context['plugin_data'].get(field_index, {}).get(
+        '_current', {}).get('example_idatadictionaryform', {}).get(
+        field_name)
+    if current:
+        data[key] = current

ckanext/datastore/plugin.py

@@ -8,10 +8,7 @@
 import ckan.plugins as p
 from ckan.model.core import State
 
-from ckan.types import (
-    Action, AuthFunction, Context, FlattenDataDict, FlattenKey,
-    FlattenErrorDict,
-)
+from ckan.types import Action, AuthFunction, Context
 from ckan.common import CKANConfig
 
 import ckanext.datastore.helpers as datastore_helpers
@@ -38,6 +35,7 @@ def sql_functions_allowlist_file():
 
 
 @p.toolkit.blanket.config_declarations
+@p.toolkit.blanket.validators
 class DatastorePlugin(p.SingletonPlugin):
     p.implements(p.IConfigurable, inherit=True)
     p.implements(p.IConfigurer)
@@ -49,7 +47,6 @@ class DatastorePlugin(p.SingletonPlugin):
     p.implements(interfaces.IDatastore, inherit=True)
     p.implements(interfaces.IDatastoreBackend, inherit=True)
     p.implements(p.IBlueprint)
-    p.implements(p.IValidators)
 
     resource_show_action = None
 
@@ -286,32 +283,3 @@ def get_blueprint(self):
         u'''Return a Flask Blueprint object to be registered by the app.'''
 
         return view.datastore
-
-    # IValidators
-
-    def get_validators(self):
-        return {'to_datastore_plugin_data': to_datastore_plugin_data}
-
-
-def to_datastore_plugin_data(plugin_key: str):
-    """
-    Return a validator that will move values from data to
-    context['plugin_data'][field_index][plugin_key][field_name]
-
-    where field_index is the field number, plugin_key (passed to this
-    function) is typically set to the plugin name and field name is the
-    original field name being validated.
-    """
-
-    def validator(
-            key: FlattenKey,
-            data: FlattenDataDict,
-            errors: FlattenErrorDict,
-            context: Context):
-        value = data.pop(key)
-        field_index = key[-2]
-        field_name = key[-1]
-        context['plugin_data'].setdefault(
-            field_index, {}).setdefault(
-            plugin_key, {})[field_name] = value
-    return validator

ckanext/example_idatadictionaryform/plugin.py

@@ -34,6 +34,7 @@ def update_datastore_create_schema(self, schema: Schema):
         ignore_empty = get_validator('ignore_empty')
         int_validator = get_validator('int_validator')
         unicode_only = get_validator('unicode_only')
+        datastore_default_current = get_validator('datastore_default_current')
         to_datastore_plugin_data = cast(
             ValidatorFactory, get_validator('to_datastore_plugin_data'))
         to_eg_iddf = to_datastore_plugin_data('example_idatadictionaryform')
@@ -44,7 +45,7 @@ def update_datastore_create_schema(self, schema: Schema):
         f['only_up'] = [
             only_increasing, ignore_empty, int_validator, to_eg_iddf]
         f['sticky'] = [
-            default_current, ignore_empty, unicode_only, to_eg_iddf]
+            datastore_default_current, ignore_empty, unicode_only, to_eg_iddf]
 
         # use different plugin_key so that value isn't removed
         # when above fields are updated & value not exposed in
@@ -100,21 +101,3 @@ def only_increasing(
     else:
         # keep current value when empty/missing
         data[key] = current
-
-
-def default_current(
-        key: FlattenKey, data: FlattenDataDict,
-        errors: FlattenErrorDict, context: Context):
-    '''default to currently stored value if empty or missing'''
-    value = data[key]
-    if value is not None and value != '' and value is not missing:
-        return
-    field_index = key[-2]
-    field_name = key[-1]
-    # current values for plugin_data are available as
-    # context['plugin_data'][field_index]['_current']
-    current = context['plugin_data'].get(field_index, {}).get(
-        '_current', {}).get('example_idatadictionaryform', {}).get(
-        field_name)
-    if current:
-        data[key] = current

doc/extensions/custom-data-dictionary.rst

@@ -0,0 +1,56 @@
+==============================================
+Customizing the DataStore Data Dictionary Form
+==============================================
+
+Extensions can customize the Data Dictionary form, keys available and values
+stored for each column using the
+:py:class:`~ckanext.datastore.interfaces.IDataDictionaryForm` interface.
+
+.. autoclass:: ckanext.datastore.interfaces.IDataDictionaryForm
+   :members:
+
+Let's add five new keys with custom validation rules to the data dictionary
+fields.
+
+With this plugin enabled each field in the Data Dictionary form will have
+an input for:
+
+ - an integer value
+ - a JSON object
+ - a numeric value that can only be increased when edited
+ - a "sticky" value that will not be removed if left blank
+ - a secret value that will be stored but never displayed in the form.
+
+First extend the form template to render the form inputs:
+
+.. literalinclude:: ../../ckanext/example_idatadictionaryform/templates/datastore/snippets/dictionary_form.html
+
+We use the ``form.input`` macro to render the form fields. The name
+of each field starts with ``fields__`` and includes a ``position`` index
+because this block will be rendered once for every field in the data
+dictionary.
+
+The value for each input is set to either the value from ``data`` the text
+data passed when re-rendering a form containing errors, or ``field`` the
+json value (text, number, object etc.) currently stored in the data
+dictionary when rendering a form for the first time.
+
+The error for each field is set from ``errors``.
+
+Next we create a plugin to apply the template and validation rules for each
+data dictionary field key.
+
+.. literalinclude:: ../../ckanext/example_idatadictionaryform/plugin.py
+
+In ``update_datastore_create_schema`` the ``to_datastore_plugin_data`` factory
+generates a validator that will store our new keys as plugin data.
+The string passed is used to group keys for this plugin to allow multiple
+separate ``IDataDictionaryForm`` plugins to store data for Data Dictionary
+fields at the same time. It's possible to use multiple groups from the same
+plugin: here we use a different group for the ``secret`` key because we want
+to treat it differently.
+
+In ``update_datastore_info_field`` we can add keys stored as plugin data
+to the ``fields`` objects returned by ``datastore_info``. Here we add
+everything but the ``secret`` key. These values are also passed to the
+form template above as ``field``.

doc/extensions/index.rst

@@ -41,3 +41,4 @@ features by developing your own CKAN extensions.
    translating-extensions
    flask-migration
    signals
+   custom-data-dictionary

doc/maintaining/datastore.rst

@@ -319,36 +319,20 @@ Fields define the column names and the type of the data in a column. A field is
             "label":  # human-readable label for column
             "notes":  # markdown description of column
             "type_override":  # type for datapusher to use when importing data
-            ...:  # other user-defined fields
+            ...:  # free-form user-defined values
 	}
+        ...:  # values defined and validated with IDataDictionaryForm
     }
 
 Field types not provided will be guessed based on the first row of provided data.
 Set the types to ensure that future inserts will not fail because of an incorrectly
 guessed type. See :ref:`valid-types` for details on which types are valid.
 
-Extra ``"info"`` field values will be stored along with the column. ``"label"``,
-``"notes"`` and ``"type_override"`` can be managed from the default :ref:`data_dictionary`
-form.  Additional fields can be stored by customizing the Data Dictionary form or by
-passing their values to the API directly.
+.. seealso::
 
-Example::
+   For more on custom field values and customizing the Data Dictionary form, see
+   :doc:`/extensions/custom-data-dictionary`.
 
-    [
-        {
-            "id": "code_number",
-            "type": "numeric"
-        },
-        {
-            "id": "description"
-            "type": "text",
-            "info": {
-                "label": "Description",
-                "notes": "A brief usage description for this code",
-                "example": "Used for temporary service interruptions"
-            }
-        }
-    ]
 
 .. _records: