From dd3164f6d0a7599eeb20d3ace36038c79aebecd7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 Mar 2024 00:03:46 +0000 Subject: [PATCH] Add datasource field in accelerations cache (#1525) * add datasource in accelerations cache Signed-off-by: Shenoy Pratik * fixed nits Signed-off-by: Shenoy Pratik --------- Signed-off-by: Shenoy Pratik (cherry picked from commit b0f0d9b9dea6e6b853cf785fc49b2f51707e3e3e) Signed-off-by: github-actions[bot] --- common/types/data_connections.ts | 17 +- .../catalog_cache/cache_loader.test.tsx | 181 ++++++++++++++---- .../framework/catalog_cache/cache_loader.tsx | 24 +-- .../catalog_cache/cache_manager.test.tsx | 94 ++++++++- .../framework/catalog_cache/cache_manager.ts | 47 ++++- 5 files changed, 298 insertions(+), 65 deletions(-) diff --git a/common/types/data_connections.ts b/common/types/data_connections.ts index ebe48afc5..99a49b48f 100644 --- a/common/types/data_connections.ts +++ b/common/types/data_connections.ts @@ -83,19 +83,19 @@ export interface CachedColumn { export interface CachedTable { name: string; - columns: CachedColumn[]; + columns?: CachedColumn[]; } export interface CachedDatabase { name: string; tables: CachedTable[]; - lastUpdated: string; // Assuming date string in UTC format + lastUpdated: string; // date string in UTC format status: CachedDataSourceStatus; } export interface CachedDataSource { name: string; - lastUpdated: string; // Assuming date string in UTC format + lastUpdated: string; // date string in UTC format status: CachedDataSourceStatus; databases: CachedDatabase[]; } @@ -115,13 +115,18 @@ export interface CachedAccelerations { status: string; } -export interface AccelerationsCacheData { - version: string; +export interface CachedAcclerationByDataSource { + name: string; accelerations: CachedAccelerations[]; - lastUpdated: string; // Assuming date string in UTC format + lastUpdated: string; // date string in UTC format status: CachedDataSourceStatus; } +export interface AccelerationsCacheData { + version: string; + dataSources: CachedAcclerationByDataSource[]; +} + export interface PollingSuccessResult { schema: Array<{ name: string; type: string }>; datarows: Array>; diff --git a/public/framework/catalog_cache/cache_loader.test.tsx b/public/framework/catalog_cache/cache_loader.test.tsx index 063da0108..f7ad4fcbf 100644 --- a/public/framework/catalog_cache/cache_loader.test.tsx +++ b/public/framework/catalog_cache/cache_loader.test.tsx @@ -11,9 +11,11 @@ import { mockShowTablesPollingResult, } from '../../../test/datasources'; import { + createLoadQuery, updateAccelerationsToCache, updateDatabasesToCache, updateTablesToCache, + updateToCache, } from './cache_loader'; import { CatalogCacheManager } from './cache_manager'; @@ -146,10 +148,7 @@ describe('loadCacheTests', () => { dataSourceName, expect.objectContaining({ name: databaseName, - tables: [ - { name: 'Table1', columns: [] }, - { name: 'Table2', columns: [] }, - ], + tables: [{ name: 'Table1' }, { name: 'Table2' }], lastUpdated: expect.any(String), status: CachedDataSourceStatus.Updated, }) @@ -166,55 +165,169 @@ describe('loadCacheTests', () => { it('should save empty accelerations cache and status failed when polling result is null', () => { const pollingResult = null; - updateAccelerationsToCache(pollingResult); + updateAccelerationsToCache('sampleDS', pollingResult); // Verify that saveAccelerationsCache is called with the correct parameters expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({ version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: expect.any(String), - status: CachedDataSourceStatus.Failed, + dataSources: [ + { + name: 'sampleDS', + accelerations: [], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Failed, + }, + ], }); }); it('should save new accelerations cache when polling result is not null', () => { - updateAccelerationsToCache(mockShowIndexesPollingResult); + updateAccelerationsToCache('sampleDS', mockShowIndexesPollingResult); // Verify that saveAccelerationsCache is called with the correct parameters expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({ version: CATALOG_CACHE_VERSION, - accelerations: [ - { - flintIndexName: 'flint_mys3_default_http_logs_skipping_index', - type: 'skipping', - database: 'default', - table: 'http_logs', - indexName: 'skipping_index', - autoRefresh: false, - status: 'Active', - }, + dataSources: [ { - flintIndexName: 'flint_mys3_default_http_logs_status_clientip_and_day_index', - type: 'covering', - database: 'default', - table: 'http_logs', - indexName: 'status_clientip_and_day', - autoRefresh: true, - status: 'Active', + name: 'sampleDS', + accelerations: [ + { + flintIndexName: 'flint_mys3_default_http_logs_skipping_index', + type: 'skipping', + database: 'default', + table: 'http_logs', + indexName: 'skipping_index', + autoRefresh: false, + status: 'Active', + }, + { + flintIndexName: 'flint_mys3_default_http_logs_status_clientip_and_day_index', + type: 'covering', + database: 'default', + table: 'http_logs', + indexName: 'status_clientip_and_day', + autoRefresh: true, + status: 'Active', + }, + { + flintIndexName: 'flint_mys3_default_http_count_view', + type: 'materialized', + database: 'default', + table: '', + indexName: 'http_count_view', + autoRefresh: true, + status: 'Active', + }, + ], + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Updated, }, + ], + }); + }); + }); + + describe('updateToCache', () => { + it('should call updateDatabasesToCache when loadCacheType is "databases"', () => { + const loadCacheType = 'databases'; + const dataSourceName = 'TestDataSource'; + + updateToCache(mockShowDatabasesPollingResult, loadCacheType, dataSourceName); + + // Verify that addOrUpdateDataSource is called + expect(CatalogCacheManager.addOrUpdateDataSource).toHaveBeenCalled(); + expect(CatalogCacheManager.updateDatabase).not.toHaveBeenCalled(); + expect(CatalogCacheManager.saveAccelerationsCache).not.toHaveBeenCalled(); + }); + + it('should call updateTablesToCache when loadCacheType is "tables"', () => { + const loadCacheType = 'tables'; + const dataSourceName = 'TestDataSource'; + const databaseName = 'TestDatabase'; + + CatalogCacheManager.addOrUpdateDataSource({ + databases: [ { - flintIndexName: 'flint_mys3_default_http_count_view', - type: 'materialized', - database: 'default', - table: '', - indexName: 'http_count_view', - autoRefresh: true, - status: 'Active', + name: databaseName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + tables: [], }, ], - lastUpdated: expect.any(String), + name: dataSourceName, + lastUpdated: new Date().toUTCString(), status: CachedDataSourceStatus.Updated, }); + + updateToCache(mockShowTablesPollingResult, loadCacheType, dataSourceName, databaseName); + + // Verify that updateDatabase is called + expect(CatalogCacheManager.addOrUpdateDataSource).toHaveBeenCalled(); + expect(CatalogCacheManager.updateDatabase).toHaveBeenCalled(); + expect(CatalogCacheManager.saveAccelerationsCache).not.toHaveBeenCalled(); + }); + + it('should call updateAccelerationsToCache when loadCacheType is "accelerations"', () => { + const loadCacheType = 'accelerations'; + const dataSourceName = 'TestDataSource'; + + updateToCache(mockShowIndexesPollingResult, loadCacheType, dataSourceName); + + // Verify that saveAccelerationsCache is called + expect(CatalogCacheManager.addOrUpdateDataSource).not.toHaveBeenCalled(); + expect(CatalogCacheManager.updateDatabase).not.toHaveBeenCalled(); + expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalled(); + }); + + it('should not call any update function when loadCacheType is not recognized', () => { + const pollResults = {}; + const loadCacheType = ''; + const dataSourceName = 'TestDataSource'; + + updateToCache(pollResults, loadCacheType, dataSourceName); + + // Verify that no update function is called + expect(CatalogCacheManager.addOrUpdateDataSource).not.toHaveBeenCalled(); + expect(CatalogCacheManager.updateDatabase).not.toHaveBeenCalled(); + expect(CatalogCacheManager.saveAccelerationsCache).not.toHaveBeenCalled(); + }); + }); + + describe('createLoadQuery', () => { + it('should create a query for loading databases', () => { + const loadCacheType = 'databases'; + const dataSourceName = 'example'; + const expectedQuery = 'SHOW SCHEMAS IN `example`'; + expect(createLoadQuery(loadCacheType, dataSourceName)).toEqual(expectedQuery); + }); + + it('should create a query for loading tables', () => { + const loadCacheType = 'tables'; + const dataSourceName = 'example'; + const databaseName = 'test'; + const expectedQuery = 'SHOW TABLES IN `example`.`test`'; + expect(createLoadQuery(loadCacheType, dataSourceName, databaseName)).toEqual(expectedQuery); + }); + + it('should create a query for loading accelerations', () => { + const loadCacheType = 'accelerations'; + const dataSourceName = 'example'; + const expectedQuery = 'SHOW FLINT INDEX in `example`'; + expect(createLoadQuery(loadCacheType, dataSourceName)).toEqual(expectedQuery); + }); + + it('should return an empty string for unknown loadCacheType', () => { + const loadCacheType = 'unknownType'; + const dataSourceName = 'example'; + expect(createLoadQuery(loadCacheType, dataSourceName)).toEqual(''); + }); + + it('should properly handle backticks in database name', () => { + const loadCacheType = 'tables'; + const dataSourceName = 'example'; + const databaseName = '`sample`'; + const expectedQuery = 'SHOW TABLES IN `example`.`sample`'; + expect(createLoadQuery(loadCacheType, dataSourceName, databaseName)).toEqual(expectedQuery); }); }); }); diff --git a/public/framework/catalog_cache/cache_loader.tsx b/public/framework/catalog_cache/cache_loader.tsx index 60fc1175e..bc0d7869e 100644 --- a/public/framework/catalog_cache/cache_loader.tsx +++ b/public/framework/catalog_cache/cache_loader.tsx @@ -4,10 +4,7 @@ */ import { useEffect, useState } from 'react'; -import { - ASYNC_POLLING_INTERVAL, - CATALOG_CACHE_VERSION, -} from '../../../common/constants/data_sources'; +import { ASYNC_POLLING_INTERVAL } from '../../../common/constants/data_sources'; import { AsyncPollingResult, CachedDataSourceStatus, @@ -80,7 +77,6 @@ export const updateTablesToCache = ( const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); const newTables = combinedData.map((row: any) => ({ name: row.tableName, - columns: [], })); CatalogCacheManager.updateDatabase(dataSourceName, { @@ -91,12 +87,15 @@ export const updateTablesToCache = ( }); }; -export const updateAccelerationsToCache = (pollingResult: AsyncPollingResult) => { +export const updateAccelerationsToCache = ( + dataSourceName: string, + pollingResult: AsyncPollingResult +) => { const currentTime = new Date().toUTCString(); if (!pollingResult) { - CatalogCacheManager.saveAccelerationsCache({ - version: CATALOG_CACHE_VERSION, + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, accelerations: [], lastUpdated: currentTime, status: CachedDataSourceStatus.Failed, @@ -116,8 +115,8 @@ export const updateAccelerationsToCache = (pollingResult: AsyncPollingResult) => status: row.status, })); - CatalogCacheManager.saveAccelerationsCache({ - version: CATALOG_CACHE_VERSION, + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, accelerations: newAccelerations, lastUpdated: currentTime, status: CachedDataSourceStatus.Updated, @@ -138,7 +137,7 @@ export const updateToCache = ( updateTablesToCache(dataSourceName, databaseName!, pollResults); break; case 'accelerations': - updateAccelerationsToCache(pollResults); + updateAccelerationsToCache(dataSourceName, pollResults); break; default: break; @@ -189,6 +188,7 @@ export const useLoadToCache = (loadCacheType: LoadCacheType) => { }, ASYNC_POLLING_INTERVAL); const startLoading = (dataSourceName: string, databaseName?: string) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); setCurrentDataSourceName(dataSourceName); setCurrentDatabaseName(databaseName); @@ -272,7 +272,7 @@ export const useLoadTablesToCache = () => { return { loadStatus, startLoading, stopLoading }; }; -export const useAccelerationsToCache = () => { +export const useLoadAccelerationsToCache = () => { const { loadStatus, startLoading, stopLoading } = useLoadToCache('accelerations'); return { loadStatus, startLoading, stopLoading }; }; diff --git a/public/framework/catalog_cache/cache_manager.test.tsx b/public/framework/catalog_cache/cache_manager.test.tsx index 69be8be29..f73609eda 100644 --- a/public/framework/catalog_cache/cache_manager.test.tsx +++ b/public/framework/catalog_cache/cache_manager.test.tsx @@ -10,6 +10,7 @@ import { } from '../../../common/constants/shared'; import { AccelerationsCacheData, + CachedAcclerationByDataSource, CachedDataSource, CachedDataSourceStatus, CachedDatabase, @@ -134,9 +135,7 @@ describe('CatalogCacheManager', () => { it('should save accelerations cache to local storage', () => { const cacheData: AccelerationsCacheData = { version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: '2024-03-07T12:00:00Z', - status: CachedDataSourceStatus.Empty, + dataSources: [], }; CatalogCacheManager.saveAccelerationsCache(cacheData); expect(localStorage.setItem).toHaveBeenCalledWith( @@ -150,20 +149,16 @@ describe('CatalogCacheManager', () => { it('should retrieve accelerations cache from local storage', () => { const cacheData: AccelerationsCacheData = { version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: '2024-03-07T12:00:00Z', - status: CachedDataSourceStatus.Empty, + dataSources: [], }; localStorage.setItem(ASYNC_QUERY_ACCELERATIONS_CACHE, JSON.stringify(cacheData)); expect(CatalogCacheManager.getAccelerationsCache()).toEqual(cacheData); }); it('should return default cache object if cache is not found', () => { - const defaultCacheObject = { + const defaultCacheObject: AccelerationsCacheData = { version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: '', - status: CachedDataSourceStatus.Empty, + dataSources: [], }; localStorage.removeItem(ASYNC_QUERY_ACCELERATIONS_CACHE); expect(CatalogCacheManager.getAccelerationsCache()).toEqual(defaultCacheObject); @@ -396,4 +391,83 @@ describe('CatalogCacheManager', () => { expect(localStorage.removeItem).toHaveBeenCalledWith(ASYNC_QUERY_ACCELERATIONS_CACHE); }); }); + + describe('addOrUpdateAccelerationsByDataSource', () => { + it('should add a new data source to the accelerations cache', () => { + const dataSource: CachedAcclerationByDataSource = { + name: 'TestDataSource', + lastUpdated: '2024-03-08T12:00:00Z', + status: CachedDataSourceStatus.Updated, + accelerations: [], + }; + + CatalogCacheManager.addOrUpdateAccelerationsByDataSource(dataSource); + + // Verify that saveAccelerationsCache is called with the updated cache data + expect(localStorage.setItem).toHaveBeenCalledWith( + ASYNC_QUERY_ACCELERATIONS_CACHE, + JSON.stringify({ + version: '1.0', + dataSources: [{ ...dataSource }], + }) + ); + }); + + it('should update an existing data source in the accelerations cache', () => { + // Set up initial cache data + const initialDataSource: CachedAcclerationByDataSource = { + name: 'TestDataSource', + lastUpdated: '2024-03-08T12:00:00Z', + status: CachedDataSourceStatus.Updated, + accelerations: [], + }; + + // Update the data source + const updatedDataSource: CachedAcclerationByDataSource = { + ...initialDataSource, + status: CachedDataSourceStatus.Failed, + }; + + CatalogCacheManager.addOrUpdateAccelerationsByDataSource(updatedDataSource); + + // Verify that saveAccelerationsCache is called with the updated cache data + expect(localStorage.setItem).toHaveBeenCalledWith( + ASYNC_QUERY_ACCELERATIONS_CACHE, + JSON.stringify({ + version: '1.0', + dataSources: [{ ...updatedDataSource }], + }) + ); + }); + }); + + describe('getOrCreateAccelerationsByDataSource', () => { + it('should return an existing data source from the accelerations cache', () => { + // Set up initial cache data + const existingDataSource: CachedAcclerationByDataSource = { + name: 'TestDataSource', + lastUpdated: '2024-03-08T12:00:00Z', + status: CachedDataSourceStatus.Updated, + accelerations: [], + }; + + CatalogCacheManager.addOrUpdateAccelerationsByDataSource(existingDataSource); + const result = CatalogCacheManager.getOrCreateAccelerationsByDataSource('TestDataSource'); + + // Verify that the existing data source is returned + expect(result).toEqual(existingDataSource); + }); + + it('should create and return a new data source if not found in the accelerations cache', () => { + const result = CatalogCacheManager.getOrCreateAccelerationsByDataSource('TestDataSource1'); + + // Verify that the new data source is created and returned + expect(result).toEqual({ + name: 'TestDataSource1', + lastUpdated: expect.any(String), + status: CachedDataSourceStatus.Empty, + accelerations: [], + }); + }); + }); }); diff --git a/public/framework/catalog_cache/cache_manager.ts b/public/framework/catalog_cache/cache_manager.ts index b183170b0..42d5b5841 100644 --- a/public/framework/catalog_cache/cache_manager.ts +++ b/public/framework/catalog_cache/cache_manager.ts @@ -10,6 +10,7 @@ import { } from '../../../common/constants/shared'; import { AccelerationsCacheData, + CachedAcclerationByDataSource, CachedDataSource, CachedDataSourceStatus, CachedDatabase, @@ -75,15 +76,55 @@ export class CatalogCacheManager { } else { const defaultCacheObject = { version: CATALOG_CACHE_VERSION, - accelerations: [], - lastUpdated: '', - status: CachedDataSourceStatus.Empty, + dataSources: [], }; this.saveAccelerationsCache(defaultCacheObject); return defaultCacheObject; } } + /** + * Adds or updates a data source in the accelerations cache. + * @param {CachedAcclerationByDataSource} dataSource - The data source to add or update. + */ + static addOrUpdateAccelerationsByDataSource(dataSource: CachedAcclerationByDataSource): void { + const accCacheData = this.getAccelerationsCache(); + const index = accCacheData.dataSources.findIndex( + (ds: CachedAcclerationByDataSource) => ds.name === dataSource.name + ); + if (index !== -1) { + accCacheData.dataSources[index] = dataSource; + } else { + accCacheData.dataSources.push(dataSource); + } + this.saveAccelerationsCache(accCacheData); + } + + /** + * Retrieves accelerations cache from local storage by the datasource name. + * @param {string} dataSourceName - The name of the data source. + * @returns {CachedAcclerationByDataSource} The retrieved accelerations by datasource in cache. + * @throws {Error} If the data source is not found. + */ + static getOrCreateAccelerationsByDataSource( + dataSourceName: string + ): CachedAcclerationByDataSource { + const accCacheData = this.getAccelerationsCache(); + const cachedDataSource = accCacheData.dataSources.find((ds) => ds.name === dataSourceName); + + if (cachedDataSource) return cachedDataSource; + else { + const defaultDataSourceObject = { + name: dataSourceName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + accelerations: [], + }; + this.addOrUpdateAccelerationsByDataSource(defaultDataSourceObject); + return defaultDataSourceObject; + } + } + /** * Adds or updates a data source in the cache. * @param {CachedDataSource} dataSource - The data source to add or update.