diff --git a/docs/developer/architecture/core/saved-objects-service-use-case-examples.asciidoc b/docs/developer/architecture/core/saved-objects-service-use-case-examples.asciidoc new file mode 100644 index 0000000000000..2b2cbde0b3f1a --- /dev/null +++ b/docs/developer/architecture/core/saved-objects-service-use-case-examples.asciidoc @@ -0,0 +1,761 @@ +[[saved-objects-service-use-case-examples]] +=== Use-case examples + +These are example of the migration scenario currently supported (out of +the box) by the system. + +*note:* _more complex scenarios (e.g field mutation by copy/sync) could +already be implemented, but without the proper tooling exposed from +Core, most of the work related to sync and compatibility would have to +be implemented in the domain layer of the type owners, which is why +we’re not documenting them yet._ + +==== Adding a non-indexed field without default value + +We are currently in model version 1, and our type has 2 indexed fields +defined: `foo` and `bar`. + +The definition of the type at version 1 would look like: + +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + namespaceType: 'single', + switchToModelVersionAt: '8.10.0', + modelVersions: { + // initial (and current) model version + 1: { + changes: [], + schemas: { + // FC schema defining the known fields (indexed or not) for this version + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string() }, + { unknowns: 'ignore' } // note the `unknown: ignore` which is how we're evicting the unknown fields + ), + // schema that will be used to validate input during `create` and `bulkCreate` + create: schema.object( + { foo: schema.string(), bar: schema.string() }, + ) + }, + }, + }, + mappings: { + properties: { + foo: { type: 'text' }, + bar: { type: 'text' }, + }, + }, +}; +---- + +From here, say we want to introduce a new `dolly` field that is not +indexed, and that we don’t need to populate with a default value. + +To achieve that, we need to introduce a new model version, with the only +thing to do will be to define the associated schemas to include this new +field. + +The added model version would look like: + +[source,ts] +---- +// the new model version adding the `dolly` field +let modelVersion2: SavedObjectsModelVersion = { + // not an indexed field, no data backfill, so changes are actually empty + changes: [], + schemas: { + // the only addition in this model version: taking the new field into account for the schemas + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + { unknowns: 'ignore' } // note the `unknown: ignore` which is how we're evicting the unknown fields + ), + create: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + ) + }, +}; +---- + +The full type definition after the addition of the new model version: + +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + namespaceType: 'single', + switchToModelVersionAt: '8.10.0', + modelVersions: { + 1: { + changes: [], + schemas: { + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { foo: schema.string(), bar: schema.string() }, + ) + }, + }, + 2: { + changes: [], + schemas: { + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + ) + }, + }, + }, + mappings: { + properties: { + foo: { type: 'text' }, + bar: { type: 'text' }, + }, + }, +}; +---- + +==== Adding an indexed field without default value + +This scenario is fairly close to the previous one. The difference being +that working with an indexed field means adding a `mappings_addition` +change and to also update the root mappings accordingly. + +To reuse the previous example, let’s say the `dolly` field we want to +add would need to be indexed instead. + +In that case, the new version needs to do the following: - add a +`mappings_addition` type change to define the new mappings - update the +root `mappings` accordingly - add the updated schemas as we did for the +previous example + +The new version definition would look like: + +[source,ts] +---- +let modelVersion2: SavedObjectsModelVersion = { + // add a change defining the mapping for the new field + changes: [ + { + type: 'mappings_addition', + addedMappings: { + dolly: { type: 'text' }, + }, + }, + ], + schemas: { + // adding the new field to the forwardCompatibility schema + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + ) + }, +}; +---- + +As said, we will also need to update the root mappings definition: + +[source,ts] +---- +mappings: { + properties: { + foo: { type: 'text' }, + bar: { type: 'text' }, + dolly: { type: 'text' }, + }, +}, +---- + +the full type definition after the addition of the model version 2 would +be: + +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + namespaceType: 'single', + switchToModelVersionAt: '8.10.0', + modelVersions: { + 1: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + foo: { type: 'text' }, + bar: { type: 'text' }, + }, + }, + ], + schemas: { + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { foo: schema.string(), bar: schema.string() }, + ) + }, + }, + 2: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + dolly: { type: 'text' }, + }, + }, + ], + schemas: { + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + ) + }, + }, + }, + mappings: { + properties: { + foo: { type: 'text' }, + bar: { type: 'text' }, + dolly: { type: 'text' }, + }, + }, +}; +---- + +==== Adding an indexed field with a default value + +Now a slightly different scenario where we’d like to populate the newly +introduced field with a default value. + +In that case, we’d need to add an additional `data_backfill` change to +populate the new field’s value (in addition to the `mappings_addition` +one): + +[source,ts] +---- +let modelVersion2: SavedObjectsModelVersion = { + changes: [ + // setting the `dolly` field's default value. + { + type: 'data_backfill', + transform: (document) => { + return { attributes: { dolly: 'default_value' } }; + }, + }, + // define the mappings for the new field + { + type: 'mappings_addition', + addedMappings: { + dolly: { type: 'text' }, + }, + }, + ], + schemas: { + // define `dolly` as an know field in the schema + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + ) + }, +}; +---- + +The full type definition would look like: + +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + namespaceType: 'single', + switchToModelVersionAt: '8.10.0', + modelVersions: { + 1: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + foo: { type: 'text' }, + bar: { type: 'text' }, + }, + }, + ], + schemas: { + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { foo: schema.string(), bar: schema.string() }, + ) + }, + }, + 2: { + changes: [ + { + type: 'data_backfill', + transform: (document) => { + return { attributes: { dolly: 'default_value' } }; + }, + }, + { + type: 'mappings_addition', + addedMappings: { + dolly: { type: 'text' }, + }, + }, + ], + schemas: { + forwardCompatibility: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { foo: schema.string(), bar: schema.string(), dolly: schema.string() }, + ) + }, + }, + }, + mappings: { + properties: { + foo: { type: 'text' }, + bar: { type: 'text' }, + dolly: { type: 'text' }, + }, + }, +}; +---- + +*Note:* _if the field was non-indexed, we would just not use the +`mappings_addition` change or update the mappings (as done in example +1)_ + +==== Removing an existing field + +We are currently in model version 1, and our type has 2 indexed fields +defined: `kept` and `removed`. + +The definition of the type at version 1 would look like: + +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + namespaceType: 'single', + switchToModelVersionAt: '8.10.0', + modelVersions: { + // initial (and current) model version + 1: { + changes: [], + schemas: { + // FC schema defining the known fields (indexed or not) for this version + forwardCompatibility: schema.object( + { kept: schema.string(), removed: schema.string() }, + { unknowns: 'ignore' } // note the `unknown: ignore` which is how we're evicting the unknown fields + ), + // schema that will be used to validate input during `create` and `bulkCreate` + create: schema.object( + { kept: schema.string(), removed: schema.string() }, + ) + }, + }, + }, + mappings: { + properties: { + kept: { type: 'text' }, + removed: { type: 'text' }, + }, + }, +}; +---- + +From here, say we want to remove the `removed` field, as our application +doesn’t need it anymore since a recent change. + +The first thing to understand here is the impact toward backward +compatibility: Say that Kibana version `X` was still using this field, +and that we stopped utilizing the field in version `X+1`. + +We can’t remove the data in version `X+1`, as we need to be able to +rollback to the prior version at *any time*. If we were to delete the +data of this `removed` field during the upgrade to version `X+1`, and if +then, for any reason, we’d need to rollback to version `X`, it would +cause a data loss, as version `X` was still using this field, but it +would no longer present in our document after the rollback. + +Which is why we need to perform any field removal as a 2-step operation: +- release `X`: Kibana still utilize the field - release `X+1`: Kibana no +longer utilize the field, but the data is still present in the documents +- release `X+2`: The data is effectively deleted from the documents. + +That way, any prior-version rollback (`X+2` to `X+1` *or* `X+1` to `X` +is safe in term of data integrity) + +The main question then, is what’s the best way of having our application +layer simply ignore this `removed` field during version `X+1`, as we +don’t want this field (now non-utilized) to be returned from the +persistence layer, as it could ``pollute'' the higher-layers where the +field is effectively no longer used or even known. + +This can easily be done by introducing a new version and using the +`forwardCompatibility` schema to ``shallow'' the field. + +The `X+1` model version would look like: + +[source,ts] +---- +// the new model version ignoring the `removed` field +let modelVersion2: SavedObjectsModelVersion = { + changes: [], + schemas: { + forwardCompatibility: schema.object( + { kept: schema.string() }, // `removed` is no longer defined here + { unknowns: 'ignore' } + ), + create: schema.object( + { kept: schema.string() }, // `removed` is no longer defined here + ) + }, +}; +---- + +The full type definition after the addition of the new model version: + +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + namespaceType: 'single', + switchToModelVersionAt: '8.10.0', + modelVersions: { + // initial (and current) model version + 1: { + changes: [], + schemas: { + // FC schema defining the known fields (indexed or not) for this version + forwardCompatibility: schema.object( + { kept: schema.string(), removed: schema.string() }, + { unknowns: 'ignore' } // note the `unknown: ignore` which is how we're evicting the unknown fields + ), + // schema that will be used to validate input during `create` and `bulkCreate` + create: schema.object( + { kept: schema.string(), removed: schema.string() }, + ) + }, + }, + 2: { + changes: [], + schemas: { + forwardCompatibility: schema.object( + { kept: schema.string() }, // `removed` is no longer defined here + { unknowns: 'ignore' } + ), + create: schema.object( + { kept: schema.string() }, // `removed` is no longer defined here + ) + }, + } + }, + mappings: { + properties: { + kept: { type: 'text' }, + removed: { type: 'text' }, + }, + }, +}; +---- + +then, in a *later* release, we can then deploy the change that will +effectively remove the data from the documents: + +[source,ts] +---- +// the new model version ignoring the `removed` field +let modelVersion3: SavedObjectsModelVersion = { + changes: [ // define a data_removal change to delete the field + { + type: 'data_removal', + removedAttributePaths: ['removed'] + } + ], + schemas: { + forwardCompatibility: schema.object( + { kept: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { kept: schema.string() }, + ) + }, +}; +---- + +The full type definition after the data removal would look like: + +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + namespaceType: 'single', + switchToModelVersionAt: '8.10.0', + modelVersions: { + // initial (and current) model version + 1: { + changes: [], + schemas: { + // FC schema defining the known fields (indexed or not) for this version + forwardCompatibility: schema.object( + { kept: schema.string(), removed: schema.string() }, + { unknowns: 'ignore' } // note the `unknown: ignore` which is how we're evicting the unknown fields + ), + // schema that will be used to validate input during `create` and `bulkCreate` + create: schema.object( + { kept: schema.string(), removed: schema.string() }, + ) + }, + }, + 2: { + changes: [], + schemas: { + forwardCompatibility: schema.object( + { kept: schema.string() }, // `removed` is no longer defined here + { unknowns: 'ignore' } + ), + create: schema.object( + { kept: schema.string() }, // `removed` is no longer defined here + ) + }, + }, + 3: { + changes: [ // define a data_removal change to delete the field + { + type: 'data_removal', + removedAttributePaths: ['removed'] + } + ], + schemas: { + forwardCompatibility: schema.object( + { kept: schema.string() }, + { unknowns: 'ignore' } + ), + create: schema.object( + { kept: schema.string() }, + ) + }, + } + }, + mappings: { + properties: { + kept: { type: 'text' }, + removed: { type: 'text' }, + }, + }, +}; +---- + +=== Testing model versions + +Model versions definitions are more structured than the legacy migration +functions, which makes them harder to test without the proper tooling. +This is why a set of testing tools and utilities are exposed from the +`@kbn/core-test-helpers-model-versions` package, to help properly test +the logic associated with model version and their associated +transformations. + +==== Tooling for unit tests + +For unit tests, the package exposes utilities to easily test the impact +of transforming documents from a model version to another one, either +upward or backward. + +===== Model version test migrator + +The `createModelVersionTestMigrator` helper allows to create a test +migrator that can be used to test model version changes between +versions, by transforming documents the same way the migration algorithm +would during an upgrade. + +*Example:* + +[source,ts] +---- +import { + createModelVersionTestMigrator, + type ModelVersionTestMigrator +} from '@kbn/core-test-helpers-model-versions'; + +const mySoTypeDefinition = someSoType(); + +describe('mySoTypeDefinition model version transformations', () => { + let migrator: ModelVersionTestMigrator; + + beforeEach(() => { + migrator = createModelVersionTestMigrator({ type: mySoTypeDefinition }); + }); + + describe('Model version 2', () => { + it('properly backfill the expected fields when converting from v1 to v2', () => { + const obj = createSomeSavedObject(); + + const migrated = migrator.migrate({ + document: obj, + fromVersion: 1, + toVersion: 2, + }); + + expect(migrated.properties).toEqual(expectedV2Properties); + }); + + it('properly removes the expected fields when converting from v2 to v1', () => { + const obj = createSomeSavedObject(); + + const migrated = migrator.migrate({ + document: obj, + fromVersion: 2, + toVersion: 1, + }); + + expect(migrated.properties).toEqual(expectedV1Properties); + }); + }); +}); +---- + +==== Tooling for integration tests + +During integration tests, we can boot a real Elasticsearch cluster, +allowing us to manipulate SO documents in a way almost similar to how it +would be done on production runtime. With integration tests, we can even +simulate the cohabitation of two Kibana instances with different model +versions to assert the behavior of their interactions. + +===== Model version test bed + +The package exposes a `createModelVersionTestBed` function that can be +used to fully setup a test bed for model version integration testing. It +can be used to start and stop the ES server, and to initiate the +migration between the two versions we’re testing. + +*Example:* + +[source,ts] +---- +import { + createModelVersionTestBed, + type ModelVersionTestKit +} from '@kbn/core-test-helpers-model-versions'; + +describe('myIntegrationTest', () => { + const testbed = createModelVersionTestBed(); + let testkit: ModelVersionTestKit; + + beforeAll(async () => { + await testbed.startES(); + }); + + afterAll(async () => { + await testbed.stopES(); + }); + + beforeEach(async () => { + // prepare the test, preparing the index and performing the SO migration + testkit = await testbed.prepareTestKit({ + savedObjectDefinitions: [{ + definition: mySoTypeDefinition, + // the model version that will be used for the "before" version + modelVersionBefore: 1, + // the model version that will be used for the "after" version + modelVersionAfter: 2, + }] + }) + }); + + afterEach(async () => { + if(testkit) { + // delete the indices between each tests to perform a migration again + await testkit.tearsDown(); + } + }); + + it('can be used to test model version cohabitation', async () => { + // last registered version is `1` (modelVersionBefore) + const repositoryV1 = testkit.repositoryBefore; + // last registered version is `2` (modelVersionAfter) + const repositoryV2 = testkit.repositoryAfter; + + // do something with the two repositories, e.g + await repositoryV1.create(someAttrs, { id }); + const v2docReadFromV1 = await repositoryV2.get('my-type', id); + expect(v2docReadFromV1.attributes).toEqual(whatIExpect); + }); +}); +---- + +*Limitations:* + +Because the test bed is only creating the parts of Core required to +instantiate the two SO repositories, and because we’re not able to +properly load all plugins (for proper isolation), the integration test +bed currently has some limitations: + +* no extensions are enabled +** no security +** no encryption +** no spaces +* all SO types will be using the same SO index + +=== Limitations and edge cases in serverless environments + +The serverless environment, and the fact that upgrade in such +environments are performed in a way where, at some point, the old and +new version of the application are living in cohabitation, leads to some +particularities regarding the way the SO APIs works, and to some +limitations / edge case that we need to document + +==== Using the `fields` option of the `find` savedObjects API + +By default, the `find` API (as any other SO API returning documents) +will migrate all documents before returning them, to ensure that +documents can be used by both versions during a cohabitation (e.g an old +node searching for documents already migrated, or a new node searching +for documents not yet migrated). + +However, when using the `fields` option of the `find` API, the documents +can’t be migrated, as some model version changes can’t be applied +against a partial set of attributes. For this reason, when the `fields` +option is provided, the documents returned from `find` will *not* be +migrated. + +Which is why, when using this option, the API consumer needs to make +sure that _all_ the fields passed to the `fields` option *were already +present in the prior model version*. Otherwise, it may lead to +inconsistencies during upgrades, where newly introduced or backfilled +fields may not necessarily appear in the documents returned from the +`search` API when the option is used. + +(_note_: both the previous and next version of Kibana must follow this +rule then) + +==== Using `bulkUpdate` for fields with large `json` blobs + +The savedObjects `bulkUpdate` API will update documents client-side and +then reindex the updated documents. These update operations are done +in-memory, and cause memory constraint issues when updating many objects +with large `json` blobs stored in some fields. As such, we recommend +against using `bulkUpdate` for savedObjects that: - use arrays (as these +tend to be large objects) - store large `json` blobs in some fields + diff --git a/docs/developer/architecture/core/saved-objects-service.asciidoc b/docs/developer/architecture/core/saved-objects-service.asciidoc index eb9f1c7fd4516..71ece4ae3d735 100644 --- a/docs/developer/architecture/core/saved-objects-service.asciidoc +++ b/docs/developer/architecture/core/saved-objects-service.asciidoc @@ -47,6 +47,11 @@ export const dashboardVisualization: SavedObjectsType = { name: 'dashboard_visualization', // <1> hidden: true, namespaceType: 'multiple-isolated', // <2> + switchToModelVersionAt: '8.10.0', + modelVersions: { + 1: modelVersion1, + 2: modelVersion2, + }, mappings: { dynamic: false, properties: { @@ -58,10 +63,7 @@ export const dashboardVisualization: SavedObjectsType = { }, }, }, - migrations: { - '1.0.0': migratedashboardVisualizationToV1, - '2.0.0': migratedashboardVisualizationToV2, - }, + // ...other mandatory properties }; ---- <1> Since the name of a Saved Object type may form part of the URL path for the @@ -95,33 +97,32 @@ Each Saved Object type can define it's own {es} field mappings. Because multiple Saved Object types can share the same index, mappings defined by a type will be nested under a top-level field that matches the type name. -For example, the mappings defined by the `dashboard_visualization` Saved +For example, the mappings defined by the `search` Saved Object type: -.src/plugins/my_plugin/server/saved_objects/dashboard_visualization.ts +https://github.com/elastic/kibana/blob/main/src/plugins/saved_search/server/saved_objects/search.ts#L19-L70[.src/plugins/saved_search/server/saved_objects/search.ts] [source,typescript] ---- import { SavedObjectsType } from 'src/core/server'; - -export const dashboardVisualization: SavedObjectsType = { - name: 'dashboard_visualization', - ... +// ... other imports +export function getSavedSearchObjectType: SavedObjectsType = { // <1> + name: 'search', + hidden: false, + namespaceType: 'multiple-isolated', mappings: { + dynamic: false, properties: { - dynamic: false, - description: { - type: 'text', - }, - hits: { - type: 'integer', - }, + title: { type: 'text' }, + description: { type: 'text' }, }, }, - migrations: { ... }, + modelVersions: { ... }, + // ...other optional properties }; ---- +<1> Simplification -Will result in the following mappings being applied to the `.kibana` index: +Will result in the following mappings being applied to the `.kibana_analytics` index: [source,json] ---- { @@ -129,14 +130,14 @@ Will result in the following mappings being applied to the `.kibana` index: "dynamic": "strict", "properties": { ... - "dashboard_vizualization": { + "search": { "dynamic": false, "properties": { - "description": { + "title": { "type": "text", }, - "hits": { - "type": "integer", + "description": { + "type": "text", }, }, } @@ -157,242 +158,336 @@ Saved Object types should never use `dynamic: true` as this can cause an arbitrary amount of fields to be added to the `.kibana` index. [[saved-objects-service-writing-migrations]] -==== Writing Migrations +==== Writing Migrations by defining model versions -Saved Objects support schema changes between Kibana versions, which we call -migrations. Migrations are applied when a Kibana installation is upgraded from -one version to the next, when exports are imported via the Saved Objects -Management UI, or when a new object is created via the HTTP API. +Saved Objects support changes using `modelVersions``. The modelVersion API is a new way to define transformations +(_``migrations''_) for your savedObject types, and will replace the +``legacy'' migration API after Kibana version `8.10.0`. The legacy migration API has been deprecated, meaning it is no longer possible to register migrations using the legacy system. -Each Saved Object type may define migrations for its schema. Migrations are -specified by the Kibana version number, receive an input document, and must -return the fully migrated document to be persisted to Elasticsearch. +Model versions are decoupled from the stack version and satisfy the requirements for zero downtime and backward-compatibility. -Let's say we want to define two migrations: -- In version 1.1.0, we want to drop the `subtitle` field and append it to the - title -- In version 1.4.0, we want to add a new `id` field to every panel with a newly - generated UUID. +Each Saved Object type may define model versions for its schema and are bound to a given https://github.com/elastic/kibana/blob/9b330e493216e8dde3166451e4714966f63f5ab7/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts#L22-L27[savedObject type]. Changes to a saved object type are +specified by defining a new model. -First, the current `mappings` should always reflect the latest or "target" -schema. Next, we should define a migration function for each step in the schema -evolution: +=== Defining model versions -src/plugins/my_plugin/server/saved_objects/dashboard_visualization.ts -[source,typescript] +As for old migrations, model versions are bound to a given +https://github.com/elastic/kibana/blob/9b330e493216e8dde3166451e4714966f63f5ab7/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts#L22-L27[savedObject +type] + +When registering a SO type, a new +https://github.com/elastic/kibana/blob/9a6a2ccdff619f827b31c40dd9ed30cb27203da7/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts#L138-L177[modelVersions] +property is available. This attribute is a map of +https://github.com/elastic/kibana/blob/9a6a2ccdff619f827b31c40dd9ed30cb27203da7/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts#L12-L20[SavedObjectsModelVersion] +which is the top-level type/container to define model versions. + +This map follows a similar `{ [version number] => version definition }` +format as the old migration map, however a given SO type’s model version +is now identified by a single integer. + +The first version must be numbered as version 1, incrementing by one for +each new version. + +That way: - SO type versions are decoupled from stack versioning - SO +type versions are independent between types + +_a *valid* version numbering:_ + +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + switchToModelVersionAt: '8.10.0', + modelVersions: { + 1: modelVersion1, // valid: start with version 1 + 2: modelVersion2, // valid: no gap between versions + }, + // ...other mandatory properties +}; ---- -import { SavedObjectsType, SavedObjectMigrationFn } from 'src/core/server'; -import uuid from 'uuid'; -interface DashboardVisualizationPre110 { - title: string; - subtitle: string; - panels: Array<{}>; -} -interface DashboardVisualization110 { - title: string; - panels: Array<{}>; -} +_an *invalid* version numbering:_ -interface DashboardVisualization140 { - title: string; - panels: Array<{ id: string }>; -} +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + switchToModelVersionAt: '8.10.0', + modelVersions: { + 2: modelVersion2, // invalid: first version must be 1 + 4: modelVersion3, // invalid: skipped version 3 + }, + // ...other mandatory properties +}; +---- -const migrateDashboardVisualization110: SavedObjectMigrationFn< - DashboardVisualizationPre110, // <1> - DashboardVisualization110 -> = (doc) => { - const { subtitle, ...attributesWithoutSubtitle } = doc.attributes; - return { - ...doc, // <2> - attributes: { - ...attributesWithoutSubtitle, - title: `${doc.attributes.title} - ${doc.attributes.subtitle}`, +=== Structure of a model version + +https://github.com/elastic/kibana/blob/9b330e493216e8dde3166451e4714966f63f5ab7/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts#L12-L20[Model +versions] are not just functions as the previous migrations were, but +structured objects describing how the version behaves and what changed +since the last one. + +_A base example of what a model version can look like:_ + +[source,ts] +---- +const myType: SavedObjectsType = { + name: 'test', + switchToModelVersionAt: '8.10.0', + modelVersions: { + 1: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + someNewField: { type: 'text' }, + }, + }, + { + type: 'data_backfill', + transform: someBackfillFunction, + }, + ], + schemas: { + forwardCompatibility: fcSchema, + create: createSchema, + }, }, - }; + }, + // ...other mandatory properties }; +---- + +*Note:* Having multiple changes of the same type for a given version is +supported by design to allow merging different sources (to prepare for +an eventual higher-level API) -const migrateDashboardVisualization140: SavedObjectMigrationFn< - DashboardVisualization110, - DashboardVisualization140 -> = (doc) => { - const outPanels = doc.attributes.panels?.map((panel) => { - return { ...panel, id: uuid.v4() }; - }); - return { - ...doc, - attributes: { - ...doc.attributes, - panels: outPanels, +_This definition would be perfectly valid:_ + +[source,ts] +---- +const version1: SavedObjectsModelVersion = { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + someNewField: { type: 'text' }, + }, + }, + { + type: 'mappings_addition', + addedMappings: { + anotherNewField: { type: 'text' }, + }, }, - }; + ], }; +---- -export const dashboardVisualization: SavedObjectsType = { - name: 'dashboard_visualization', // <1> - /** ... */ - migrations: { - // Takes a pre 1.1.0 doc, and converts it to 1.1.0 - '1.1.0': migrateDashboardVisualization110, +It’s currently composed of two main properties: + +==== changes + +https://github.com/elastic/kibana/blob/9b330e493216e8dde3166451e4714966f63f5ab7/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts#L21-L51[link +to the TS doc for `changes`] + +Describes the list of changes applied during this version. + +*Important:* This is the part that replaces the old migration system, +and allows defining when a version adds new mapping, mutates the +documents, or other type-related changes. + +The current types of changes are: + +===== - mappings_addition + +Used to define new mappings introduced in a given version. - // Takes a 1.1.0 doc, and converts it to 1.4.0 - '1.4.0': migrateDashboardVisualization140, // <3> +_Usage example:_ + +[source,ts] +---- +const change: SavedObjectsModelMappingsAdditionChange = { + type: 'mappings_addition', + addedMappings: { + newField: { type: 'text' }, + existingNestedField: { + properties: { + newNestedProp: { type: 'keyword' }, + }, + }, }, }; ---- -<1> It is useful to define an interface for each version of the schema. This -allows TypeScript to ensure that you are properly handling the input and output -types correctly as the schema evolves. -<2> Returning a shallow copy is necessary to avoid type errors when using -different types for the input and output shape. -<3> Migrations do not have to be defined for every version. The version number -of a migration must always be the earliest Kibana version in which this -migration was released. So if you are creating a migration which will be -part of the v7.10.0 release, but will also be backported and released as -v7.9.3, the migration version should be: 7.9.3. -Migrations should be written defensively, an exception in a migration function -will prevent a Kibana upgrade from succeeding and will cause downtime for our -users. Having said that, if a document is encountered that is not in the -expected shape, migrations are encouraged to throw an exception to abort the -upgrade. In most scenarios, it is better to fail an upgrade than to silently -ignore a corrupt document which can cause unexpected behaviour at some future -point in time. +*note:* _When adding mappings, the root `type.mappings` must also be +updated accordingly (as it was done previously)._ -WARNING: Do not attempt to change the `migrationVersion`, `id`, or `type` fields -within a migration function, this is not supported. +===== - mappings_deprecation -It is critical that you have extensive tests to ensure that migrations behave -as expected with all possible input documents. Given how simple it is to test -all the branch conditions in a migration function and the high impact of a bug -in this code, there's really no reason not to aim for 100% test code coverage. +Used to flag mappings as no longer being used and ready to be removed. -==== Type visibility -It is recommended that plugins only expose Saved Object types that are necessary. -That is so to provide better backward compatibility. +_Usage example:_ -There are two options to register a type: either as completely unexposed to the global Saved Objects HTTP APIs and client or to only expose it to the client but not to the APIs. - -===== Hidden types +[source,ts] +---- +let change: SavedObjectsModelMappingsDeprecationChange = { + type: 'mappings_deprecation', + deprecatedMappings: ['someDeprecatedField', 'someNested.deprecatedField'], +}; +---- -In case when the type is not hidden, it will be exposed via the global Saved Objects HTTP API. -That brings the limitation of introducing backward incompatible changes as there might be a service consuming the API. +*note:* _It is currently not possible to remove fields from an existing +index’s mapping (without reindexing into another index), so the mappings +flagged with this change type won’t be deleted for now, but this should +still be used to allow our system to clean the mappings once upstream +(ES) unblock us._ -This is a formal limitation not prohibiting backward incompatible changes in the migrations. -But in case of such changes on the hidden types, they can be resolved and encapsulated on the intermediate layer in the plugin API. +===== - data_backfill -Hence, the main idea is that all the interactions with the Saved Objects should be done via the plugin API rather than via the Saved Objects HTTP API. +Used to populate fields (indexed or not) added in the same version. -By default, the hidden types will not be accessible in the Saved Objects client. -To make them accessible, they should be explicitly listed in the `includedHiddenTypes` parameters upon client creation. +_Usage example:_ -[source,typescript] +[source,ts] +---- +let change: SavedObjectsModelDataBackfillChange = { + type: 'data_backfill', + transform: (document) => { + return { attributes: { someAddedField: 'defaultValue' } }; + }, +}; ---- -import { CoreStart, Plugin } from '@kbn/core/server'; -class SomePlugin implements Plugin { - // ... +*note:* _Even if no check is performed to ensure it, this type of model +change should only be used to backfill newly introduced fields._ - public start({ savedObjects }: CoreStart) { - // ... +===== - data_removal - const savedObjectsClient = savedObjects.getScopedClient( - request, - { includedHiddenTypes: ['dashboard_visualization'] } - ); +Used to remove data (unset fields) from all documents of the type. - // ... - } -} +_Usage example:_ + +[source,ts] +---- +let change: SavedObjectsModelDataRemovalChange = { + type: 'data_removal', + attributePaths: ['someRootAttributes', 'some.nested.attribute'], +}; ---- -===== Hidden from the HTTP APIs +*note:* _Due to backward compatibility, field utilization must be +stopped in a prior release before actual data removal (in case of +rollback). Please refer to the field removal migration example below in +this document_ -When a saved object is registered as hidden from the HTTP APIs, it will remain exposed to the global Saved Objects client: +===== - unsafe_transform -[source,typescript] ----- -import { SavedObjectsType } from 'src/core/server'; +Used to execute an arbitrary transformation function. -export const myCustomVisualization: SavedObjectsType = { - name: 'my_custom_visualization', // <1> - hidden: false, - hiddenFromHttpApis: true, // <2> - namespaceType: 'multiple-isolated', - mappings: { - dynamic: false, - properties: { - description: { - type: 'text', - }, - hits: { - type: 'integer', - }, - }, - }, - migrations: { - '1.0.0': migrateMyCustomVisualizationToV1, - '2.0.0': migrateMyCustomVisualizationToV2, +_Usage example:_ + +[source,ts] +---- +let change: SavedObjectsModelUnsafeTransformChange = { + type: 'unsafe_transform', + transformFn: (document) => { + document.attributes.someAddedField = 'defaultValue'; + return { document }; }, }; ---- -<1> MyCustomVisualization types have their own domain-specific HTTP API's that leverage the global Saved Objects client -<2> This field determines "hidden from http apis" behavior -- any attempts to use the global Saved Objects HTTP APIs will throw errors +*note:* _Using such transformations is potentially unsafe, given the +migration system will have no knowledge of which kind of operations will +effectively be executed against the documents. Those should only be used +when there’s no other way to cover one’s migration needs._ *Please reach +out to the development team if you think you need to use this, as you +theoretically shouldn’t.* + +==== schemas + +https://github.com/elastic/kibana/blob/9b330e493216e8dde3166451e4714966f63f5ab7/packages/core/saved-objects/core-saved-objects-server/src/model_version/schemas.ts#L11-L16[link +to the TS doc for `schemas`] + +The schemas associated with this version. Schemas are used to validate +or convert SO documents at various stages of their lifecycle. -=== Client side usage +The currently available schemas are: -==== References +===== forwardCompatibility -When a Saved Object declares `references` to other Saved Objects, the -Saved Objects Export API will automatically export the target object with all -of its references. This makes it easy for users to export the entire -reference graph of an object. +This is a new concept introduced by model versions. This schema is used +for inter-version compatibility. -If a Saved Object can't be used on its own, that is, it needs other objects -to exist for a feature to function correctly, that Saved Object should declare -references to all the objects it requires. For example, a `dashboard` -object might have panels for several `visualization` objects. When these -`visualization` objects don't exist, the dashboard cannot be rendered -correctly. The `dashboard` object should declare references to all its -visualizations. +When retrieving a savedObject document from an index, if the version of +the document is higher than the latest version known of the Kibana +instance, the document will go through the `forwardCompatibility` schema +of the associated model version. -However, `visualization` objects can continue to be rendered or embedded into -other dashboards even if the `dashboard` it was originally embedded into -doesn't exist. As a result, `visualization` objects should not declare -references to `dashboard` objects. +*Important:* These conversion mechanism shouldn’t assert the data +itself, and only strip unknown fields to convert the document to the +*shape* of the document at the given version. -For each referenced object, an `id`, `type` and `name` are added to the -`references` array: +Basically, this schema should keep all the known fields of a given +version, and remove all the unknown fields, without throwing. -[source, typescript] +Forward compatibility schema can be implemented in two different ways. + +[arabic] +. Using `config-schema` + +_Example of schema for a version having two fields: someField and +anotherField_ + +[source,ts] ---- -router.get( - { path: '/some-path', validate: false }, - async (context, req, res) => { - const object = await context.core.savedObjects.client.create( - 'dashboard', - { - title: 'my dashboard', - panels: [ - { visualization: 'vis1' }, // <1> - ], - indexPattern: 'indexPattern1' - }, - { references: [ - { id: '...', type: 'visualization', name: 'vis1' }, - { id: '...', type: 'index_pattern', name: 'indexPattern1' }, - ] - } - ) - ... - } +const versionSchema = schema.object( + { + someField: schema.maybe(schema.string()), + anotherField: schema.maybe(schema.string()), + }, + { unknowns: 'ignore' } ); ---- -<1> Note how `dashboard.panels[0].visualization` stores the `name` property of -the reference (not the `id` directly) to be able to uniquely identify this -reference. This guarantees that the id the reference points to always remains -up to date. If a visualization `id` was directly stored in -`dashboard.panels[0].visualization` there is a risk that this `id` gets -updated without updating the reference in the references array. + +*Important:* Note the `{ unknowns: 'ignore' }` in the schema’s options. +This is required when using `config-schema` based schemas, as this what +will evict the additional fields without throwing an error. + +[arabic, start=2] +. Using a plain javascript function + +_Example of schema for a version having two fields: someField and +anotherField_ + +[source,ts] +---- +const versionSchema: SavedObjectModelVersionEvictionFn = (attributes) => { + const knownFields = ['someField', 'anotherField']; + return pick(attributes, knownFields); +} +---- + +*note:* _Even if highly recommended, implementing this schema is not +strictly required. Type owners can manage unknown fields and +inter-version compatibility themselves in their service layer instead._ + +===== create + +This is a direct replacement for +https://github.com/elastic/kibana/blob/9b330e493216e8dde3166451e4714966f63f5ab7/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts#L75-L82[the +old SavedObjectType.schemas] definition, now directly included in the +model version definition. + +As a refresher the `create` schema is a `@kbn/config-schema` object-type +schema, and is used to validate the properties of the document during +`create` and `bulkCreate` operations. + +*note:* _Implementing this schema is optional, but still recommended, as +otherwise there will be no validating when importing objects_ + +For implementation examples, refer to <>. + +include::saved-objects-service-use-case-examples.asciidoc[leveloffset=+1] diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts index 716b31406649c..53b52d04f376b 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts @@ -70,6 +70,7 @@ export interface SavedObjectsType { mappings: SavedObjectsTypeMappingDefinition; /** * An optional map of {@link SavedObjectMigrationFn | migrations} or a function returning a map of {@link SavedObjectMigrationFn | migrations} to be used to migrate the type. + * @deprecated Use {@link SavedObjectsType.modelVersions | modelVersions} instead. */ migrations?: SavedObjectMigrationMap | (() => SavedObjectMigrationMap); /** @@ -78,6 +79,7 @@ export interface SavedObjectsType { * When provided, calls to {@link SavedObjectsClient.create | create} will be validated against this schema. * * See {@link SavedObjectsValidationMap} for more details. + * @deprecated Use {@link SavedObjectsType.modelVersions | modelVersions} instead. */ schemas?: SavedObjectsValidationMap | (() => SavedObjectsValidationMap); /** @@ -177,7 +179,7 @@ export interface SavedObjectsType { modelVersions?: SavedObjectsModelVersionMap | SavedObjectsModelVersionMapProvider; /** - * Allows to opt-in to the new model version API. + * Allows to opt-in to the model version API. * * Must be a valid semver version (with the patch version being necessarily 0) *