Skip to content

Commit

Permalink
Merge branch 'main' into feat-sidecar
Browse files Browse the repository at this point in the history
Signed-off-by: tygao <[email protected]>
  • Loading branch information
raintygao authored Mar 4, 2024
2 parents 93aa5cc + d407f55 commit 4250167
Show file tree
Hide file tree
Showing 41 changed files with 1,422 additions and 20 deletions.
4 changes: 3 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- [BUG][Discover] Allow saved sort from search embeddable to load in Dashboard ([#5934](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5934))
- [BUG][Discover] Add key to index pattern options for support deplicate index pattern names([#5946](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5946))
- [Discover] Fix table cell content overflowing in Safari ([#5948](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5948))
- [BUG][MD]Fix schema for test connection to separate validation based on auth type([#5997](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5997))
- [BUG][MD]Fix schema for test connection to separate validation based on auth type ([#5997](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5997))
- [Discover] Enable 'Back to Top' Feature in Discover for scrolling to top ([#6008](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6008))

### 🚞 Infrastructure

Expand Down Expand Up @@ -75,6 +76,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- [Multiple Datasource] Improved error handling for the search API when a null value is passed for the dataSourceId ([#5882](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5882))
- [Multiple Datasource] Hide/Show authentication method in multi data source plugin based on configuration ([#5916](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5916))
- [[Dynamic Configurations] Add support for dynamic application configurations ([#5855](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5855))
- [Workspace] Optional workspaces params in repository ([#5949](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5949))
- Add sidecar service ([#5920](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5920))

### 🐛 Bug Fixes
Expand Down
5 changes: 4 additions & 1 deletion config/opensearch_dashboards.yml
Original file line number Diff line number Diff line change
Expand Up @@ -292,4 +292,7 @@
# opensearchDashboards.survey.url: "https://survey.opensearch.org"

# Set the value of this setting to true to enable plugin augmentation on Dashboard
# vis_augmenter.pluginAugmentationEnabled: true
# vis_augmenter.pluginAugmentationEnabled: true

# Set the value to true to enable workspace feature
# workspace.enabled: false
1 change: 1 addition & 0 deletions src/core/public/saved_objects/saved_objects_client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,7 @@ export class SavedObjectsClient {
filter: 'filter',
namespaces: 'namespaces',
preference: 'preference',
workspaces: 'workspaces',
};

const renamedQuery = renameKeys<SavedObjectsFindOptions, any>(renameMap, options);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ export interface SavedObjectsExportOptions {
excludeExportDetails?: boolean;
/** optional namespace to override the namespace used by the savedObjectsClient. */
namespace?: string;
/** optional workspaces to override the workspaces used by the savedObjectsClient. */
workspaces?: string[];
}

/**
Expand Down Expand Up @@ -87,13 +89,15 @@ async function fetchObjectsToExport({
exportSizeLimit,
savedObjectsClient,
namespace,
workspaces,
}: {
objects?: SavedObjectsExportOptions['objects'];
types?: string[];
search?: string;
exportSizeLimit: number;
savedObjectsClient: SavedObjectsClientContract;
namespace?: string;
workspaces?: string[];
}) {
if ((types?.length ?? 0) > 0 && (objects?.length ?? 0) > 0) {
throw Boom.badRequest(`Can't specify both "types" and "objects" properties when exporting`);
Expand Down Expand Up @@ -121,6 +125,7 @@ async function fetchObjectsToExport({
search,
perPage: exportSizeLimit,
namespaces: namespace ? [namespace] : undefined,
...(workspaces ? { workspaces } : {}),
});
if (findResponse.total > exportSizeLimit) {
throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
Expand Down Expand Up @@ -153,6 +158,7 @@ export async function exportSavedObjectsToStream({
includeReferencesDeep = false,
excludeExportDetails = false,
namespace,
workspaces,
}: SavedObjectsExportOptions) {
const rootObjects = await fetchObjectsToExport({
types,
Expand All @@ -161,6 +167,7 @@ export async function exportSavedObjectsToStream({
savedObjectsClient,
exportSizeLimit,
namespace,
workspaces,
});
let exportedObjects: Array<SavedObject<unknown>> = [];
let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = [];
Expand Down
3 changes: 3 additions & 0 deletions src/core/server/saved_objects/import/check_conflicts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ interface CheckConflictsParams {
ignoreRegularConflicts?: boolean;
retries?: SavedObjectsImportRetry[];
createNewCopies?: boolean;
workspaces?: string[];
}

const isUnresolvableConflict = (error: SavedObjectError) =>
Expand All @@ -56,6 +57,7 @@ export async function checkConflicts({
ignoreRegularConflicts,
retries = [],
createNewCopies,
workspaces,
}: CheckConflictsParams) {
const filteredObjects: Array<SavedObject<{ title?: string }>> = [];
const errors: SavedObjectsImportError[] = [];
Expand All @@ -77,6 +79,7 @@ export async function checkConflicts({
});
const checkConflictsResult = await savedObjectsClient.checkConflicts(objectsToCheck, {
namespace,
workspaces,
});
const errorMap = checkConflictsResult.errors.reduce(
(acc, { type, id, error }) => acc.set(`${type}:${id}`, error),
Expand Down
3 changes: 3 additions & 0 deletions src/core/server/saved_objects/import/create_saved_objects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ interface CreateSavedObjectsParams<T> {
overwrite?: boolean;
dataSourceId?: string;
dataSourceTitle?: string;
workspaces?: string[];
}
interface CreateSavedObjectsResult<T> {
createdObjects: Array<CreatedObject<T>>;
Expand All @@ -60,6 +61,7 @@ export const createSavedObjects = async <T>({
overwrite,
dataSourceId,
dataSourceTitle,
workspaces,
}: CreateSavedObjectsParams<T>): Promise<CreateSavedObjectsResult<T>> => {
// filter out any objects that resulted in errors
const errorSet = accumulatedErrors.reduce(
Expand Down Expand Up @@ -169,6 +171,7 @@ export const createSavedObjects = async <T>({
const bulkCreateResponse = await savedObjectsClient.bulkCreate(objectsToCreate, {
namespace,
overwrite,
workspaces,
});
expectedResults = bulkCreateResponse.saved_objects;
}
Expand Down
3 changes: 3 additions & 0 deletions src/core/server/saved_objects/import/import_saved_objects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ export async function importSavedObjectsFromStream({
namespace,
dataSourceId,
dataSourceTitle,
workspaces,
}: SavedObjectsImportOptions): Promise<SavedObjectsImportResponse> {
let errorAccumulator: SavedObjectsImportError[] = [];
const supportedTypes = typeRegistry.getImportableAndExportableTypes().map((type) => type.name);
Expand Down Expand Up @@ -92,6 +93,7 @@ export async function importSavedObjectsFromStream({
savedObjectsClient,
namespace,
ignoreRegularConflicts: overwrite,
workspaces,
};

const checkConflictsResult = await checkConflicts(checkConflictsParams);
Expand Down Expand Up @@ -142,6 +144,7 @@ export async function importSavedObjectsFromStream({
namespace,
dataSourceId,
dataSourceTitle,
...(workspaces ? { workspaces } : {}),
};
const createSavedObjectsResult = await createSavedObjects(createSavedObjectsParams);
errorAccumulator = [...errorAccumulator, ...createSavedObjectsResult.errors];
Expand Down
2 changes: 2 additions & 0 deletions src/core/server/saved_objects/import/resolve_import_errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ export async function resolveSavedObjectsImportErrors({
createNewCopies,
dataSourceId,
dataSourceTitle,
workspaces,
}: SavedObjectsResolveImportErrorsOptions): Promise<SavedObjectsImportResponse> {
// throw a BadRequest error if we see invalid retries
validateRetries(retries);
Expand Down Expand Up @@ -163,6 +164,7 @@ export async function resolveSavedObjectsImportErrors({
overwrite,
dataSourceId,
dataSourceTitle,
workspaces,
};
const { createdObjects, errors: bulkCreateErrors } = await createSavedObjects(
createSavedObjectsParams
Expand Down
4 changes: 4 additions & 0 deletions src/core/server/saved_objects/import/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ export interface SavedObjectsImportOptions {
createNewCopies: boolean;
dataSourceId?: string;
dataSourceTitle?: string;
/** if specified, will import in given workspaces */
workspaces?: string[];
}

/**
Expand All @@ -212,6 +214,8 @@ export interface SavedObjectsResolveImportErrorsOptions {
createNewCopies: boolean;
dataSourceId?: string;
dataSourceTitle?: string;
/** if specified, will import in given workspaces */
workspaces?: string[];
}

export type CreatedObject<T> = SavedObject<T> & { destinationId?: string };
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

import { IndexMapping, SavedObjectsTypeMappingDefinitions } from './../../mappings';
import { buildActiveMappings, diffMappings } from './build_active_mappings';
import { configMock } from '../../../config/mocks';

describe('buildActiveMappings', () => {
test('creates a strict mapping', () => {
Expand Down Expand Up @@ -91,6 +92,12 @@ describe('buildActiveMappings', () => {
expect(hashes.aaa).toEqual(hashes.bbb);
expect(hashes.aaa).not.toEqual(hashes.ccc);
});

test('workspaces field is added when workspace feature flag is enabled', () => {
const rawConfig = configMock.create();
rawConfig.get.mockReturnValue(true);
expect(buildActiveMappings({}, rawConfig)).toHaveProperty('properties.workspaces');
});
});

describe('diffMappings', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import crypto from 'crypto';
import { cloneDeep, mapValues } from 'lodash';
import { Config } from 'packages/osd-config/target';
import {
IndexMapping,
SavedObjectsMappingProperties,
Expand All @@ -47,11 +48,20 @@ import {
* @param typeDefinitions - the type definitions to build mapping from.
*/
export function buildActiveMappings(
typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties
typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties,
opensearchDashboardsRawConfig?: Config
): IndexMapping {
const mapping = defaultMapping();

const mergedProperties = validateAndMerge(mapping.properties, typeDefinitions);
let mergedProperties = validateAndMerge(mapping.properties, typeDefinitions);
// if permission control for saved objects is enabled, the permissions field should be added to the mapping
if (opensearchDashboardsRawConfig?.get('workspace.enabled')) {
mergedProperties = validateAndMerge(mapping.properties, {
workspaces: {
type: 'keyword',
},
});
}

return cloneDeep({
...mapping,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry';
import { IndexMigrator } from './index_migrator';
import { MigrationOpts } from './migration_context';
import { loggingSystemMock } from '../../../logging/logging_system.mock';
import { configMock } from '../../../config/mocks';

describe('IndexMigrator', () => {
let testOpts: jest.Mocked<MigrationOpts> & {
Expand All @@ -59,6 +60,60 @@ describe('IndexMigrator', () => {
};
});

test('creates the index when workspaces feature flag is enabled', async () => {
const { client } = testOpts;

testOpts.mappingProperties = { foo: { type: 'long' } as any };
const rawConfig = configMock.create();
rawConfig.get.mockReturnValue(true);
testOpts.opensearchDashboardsRawConfig = rawConfig;

withIndex(client, { index: { statusCode: 404 }, alias: { statusCode: 404 } });

await new IndexMigrator(testOpts).migrate();

expect(client.indices.create).toHaveBeenCalledWith({
body: {
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: {
foo: '18c78c995965207ed3f6e7fc5c6e55fe',
migrationVersion: '4a1746014a75ade3a714e1db5763276f',
namespace: '2f4316de49999235636386fe51dc06c1',
namespaces: '2f4316de49999235636386fe51dc06c1',
originId: '2f4316de49999235636386fe51dc06c1',
references: '7997cf5a56cc02bdc9c93361bde732b0',
type: '2f4316de49999235636386fe51dc06c1',
updated_at: '00da57df13e94e9d98437d13ace4bfe0',
workspaces: '2f4316de49999235636386fe51dc06c1',
},
},
properties: {
foo: { type: 'long' },
migrationVersion: { dynamic: 'true', type: 'object' },
namespace: { type: 'keyword' },
namespaces: { type: 'keyword' },
originId: { type: 'keyword' },
type: { type: 'keyword' },
updated_at: { type: 'date' },
references: {
type: 'nested',
properties: {
name: { type: 'keyword' },
type: { type: 'keyword' },
id: { type: 'keyword' },
},
},
workspaces: { type: 'keyword' },
},
},
settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
},
index: '.kibana_1',
});
});

test('creates the index if it does not exist', async () => {
const { client } = testOpts;

Expand Down
16 changes: 12 additions & 4 deletions src/core/server/saved_objects/migrations/core/migration_context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
*/

import { Logger } from 'src/core/server/logging';
import { Config } from 'packages/osd-config/target';
import { MigrationOpenSearchClient } from './migration_opensearch_client';
import { SavedObjectsSerializer } from '../../serialization';
import {
Expand Down Expand Up @@ -65,6 +66,7 @@ export interface MigrationOpts {
* prior to running migrations. For example: 'opensearch_dashboards_index_template*'
*/
obsoleteIndexTemplatePattern?: string;
opensearchDashboardsRawConfig?: Config;
}

/**
Expand All @@ -90,10 +92,15 @@ export interface Context {
* and various info needed to migrate the source index.
*/
export async function migrationContext(opts: MigrationOpts): Promise<Context> {
const { log, client } = opts;
const { log, client, opensearchDashboardsRawConfig } = opts;
const alias = opts.index;
const source = createSourceContext(await Index.fetchInfo(client, alias), alias);
const dest = createDestContext(source, alias, opts.mappingProperties);
const dest = createDestContext(
source,
alias,
opts.mappingProperties,
opensearchDashboardsRawConfig
);

return {
client,
Expand Down Expand Up @@ -125,10 +132,11 @@ function createSourceContext(source: Index.FullIndexInfo, alias: string) {
function createDestContext(
source: Index.FullIndexInfo,
alias: string,
typeMappingDefinitions: SavedObjectsTypeMappingDefinitions
typeMappingDefinitions: SavedObjectsTypeMappingDefinitions,
opensearchDashboardsRawConfig?: Config
): Index.FullIndexInfo {
const targetMappings = disableUnknownTypeMappingFields(
buildActiveMappings(typeMappingDefinitions),
buildActiveMappings(typeMappingDefinitions, opensearchDashboardsRawConfig),
source.mappings
);

Expand Down
Loading

0 comments on commit 4250167

Please sign in to comment.