Skip to content

Commit

Permalink
relational test data setup can receive multiple DB definitions (#2549)
Browse files Browse the repository at this point in the history
  • Loading branch information
@haroonsherjan
haroonsherjan authored Jan 22, 2024
1 parent 715a473 commit 7677747
Show file tree
Hide file tree
Showing 10 changed files with 47 additions and 37 deletions.
2 changes: 1 addition & 1 deletion ...engine-pure-code-compiled-core/src/main/resources/core/pure/executionPlan/extensions.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ Class meta::pure::executionPlan::extension::ExecutionPlanExtension
planGraphFetchExecution : Function<{StoreMappingLocalGraphFetchExecutionNodeGenerationInput[1] -> LocalGraphFetchExecutionNode[1]}>[0..1];
planCrossGraphFetchExecution : Function<{StoreMappingCrossLocalGraphFetchExecutionNodeGenerationInput[1] -> LocalGraphFetchExecutionNode[1]}>[0..1];

reprocessConnection : Function<{ConnectionStore[1]->Connection[1]}>[0..1];
processRuntimeTestConnections : Function<{ConnectionStore[*]->ConnectionStore[*]}>[0..1];
extractVariablesFromExecutionOption : Function<{ExecutionOption[1]->PlanVarPlaceHolder[*]}>[0..1];
crossStoreSourceSupportsBatching : Function<{InstanceSetImplementation[1], AbstractProperty<Any>[*], Map<String, String>[1] -> Boolean[1]}>[0..1];
}
8 changes: 7 additions & 1 deletion ...ine-pure-code-compiled-core/src/main/resources/core/pure/router/metamodel/clustering.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,13 @@ function meta::pure::router::clustering::buildExecutionNodeForStoreClusteredVS(s
//This is required to populate the setup sql for test connections where csv data has been supplied. It is done here to ensure that the plan generation is always
//the only place where sql is generated for this case.
let rt = $sc.runtime->match([
r:Runtime[1] | ^$r(connectionStores = $r.connectionStores->map(c | if ($sc.s.reprocessConnection->isEmpty(),|$c,|^$c(connection=$sc.s.reprocessConnection->toOne()->eval($c))))),
r:Runtime[1] | ^$r(connectionStores =
if (
$sc.s.processRuntimeTestConnections->isEmpty(),
|$r.connectionStores,
|$sc.s.processRuntimeTestConnections->toOne()->eval($r.connectionStores);
)
),
r:Runtime[0..1] | $r
]);
let query = ^meta::pure::mapping::StoreQuery(store=$sc.store, fe=$fe, inScopeVars=$inScopeVars);
Expand Down
20 changes: 12 additions & 8 deletions ...ionalStore-pure/src/main/resources/core_relational/relational/contract/storeContract.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,14 +113,16 @@ function meta::relational::contract::relationalStoreContract():StoreContract[1]
a: Any[*] | false
])
},
reprocessConnection = {c:ConnectionStore[1] |
$c.connection->match(
processRuntimeTestConnections = {connectionStores:ConnectionStore[*] |
$connectionStores.connection->distinct()->map(connection|
let specificConnectionStores = $connectionStores->filter(cs|$cs.connection==$connection);
let newConnection = $connection->match(
[
t:meta::external::store::relational::runtime::TestDatabaseConnection[1] |
^$t(testDataSetupSqls = if ($t.testDataSetupCsv->isEmpty() || !$c.element->instanceOf(meta::relational::metamodel::Database),
^$t(testDataSetupSqls = if ($t.testDataSetupCsv->isEmpty(),
| [];,
| if ($t.testDataSetupSqls->isEmpty(),
| $t.testDataSetupCsv->toOne()->meta::alloy::service::execution::setUpDataSQLs($c.element->cast(@meta::relational::metamodel::Database)),
| $t.testDataSetupCsv->toOne()->meta::alloy::service::execution::setUpDataSQLs($specificConnectionStores.element->cast(@meta::relational::metamodel::Database)),
| $t.testDataSetupSqls))),
r:meta::external::store::relational::runtime::RelationalDatabaseConnection[1] |
^$r(
Expand All @@ -130,17 +132,19 @@ function meta::relational::contract::relationalStoreContract():StoreContract[1]
(
testDataSetupSqls =
$l.testDataSetupSqls->concatenate(
if ($l.testDataSetupCsv->isEmpty() || !$c.element->instanceOf(meta::relational::metamodel::Database),
if ($l.testDataSetupCsv->isEmpty(),
| [];,
| $l.testDataSetupCsv->toOne()->meta::alloy::service::execution::setUpDataSQLs($c.element->cast(@meta::relational::metamodel::Database))
| $l.testDataSetupCsv->toOne()->meta::alloy::service::execution::setUpDataSQLs($specificConnectionStores.element->cast(@meta::relational::metamodel::Database))
)
)
),
d:meta::pure::alloy::connections::alloy::specification::DatasourceSpecification[1] | $d
])
),
c:Connection[1] | $c
]);
conn:Connection[1] | $conn
]);
$specificConnectionStores->map(cs| ^$cs(connection=$newConnection));
);
},

printPlanNodeToString = {space : String[1], extensions:meta::pure::extension::Extension[*]|
Expand Down
7 changes: 4 additions & 3 deletions ...e-pure/src/main/resources/core_relational/relational/graphFetch/relationalGraphFetch.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -612,10 +612,11 @@ function meta::relational::graphFetch::executionPlan::planCrossRootGraphFetchExe
print(if($debug.debug,|$debug.space+' >Generate cross property query (with properties - '+$simplePrimitiveProperties->map(p|$p.name)->removeDuplicates()->joinStrings('[',', ',']')+'\n',|''));
let crossQuery = generateCrossPropertyQuery($rootTree, $parentIdx, $xStorePropertyMapping, $srcProperties, $mapping, $inScopeVars, $simplePrimitiveProperties, $mcForProperty, $debug->indent(2), $extensions);
let dbConnectionPre = $runtime->connectionByElement($store)->cast(@DatabaseConnection) ;
let dbConnectionStorePre = $runtime.connectionStores->filter(connectionStore|$connectionStore.connection==$dbConnectionPre)->toOne();
let dbConnectionStorePre = $runtime.connectionStores->filter(connectionStore|$connectionStore.connection==$dbConnectionPre);

let dbConnection = if($clusteredTree.s.reprocessConnection->isEmpty(),|$dbConnectionPre,|$clusteredTree.s.reprocessConnection->toOne()->eval($dbConnectionStorePre))->cast(@DatabaseConnection) ;
let oneRuntime = ^Runtime(connectionStores = ^$dbConnectionStorePre(connection=$dbConnection));
let dbConnectionStore = if($clusteredTree.s.processRuntimeTestConnections->isEmpty(),|$dbConnectionStorePre,|$clusteredTree.s.processRuntimeTestConnections->toOne()->eval($dbConnectionStorePre));
let dbConnection = $dbConnectionStore.connection->distinct()->toOne('Expected 1 connection, but received ' + $dbConnectionStore.connection->distinct()->size()->toString())->cast(@DatabaseConnection);
let oneRuntime = ^Runtime(connectionStores = $dbConnectionStore);
let postProcessorResult = $crossQuery->postProcessSQLQuery($store, [], $mapping, $oneRuntime, $exeCtx, $extensions);
let postProcessedQuery = $postProcessorResult.query->cast(@SelectSQLQuery);
print(if($debug.debug,|$debug.space+' >Generated cross property query: '+$postProcessedQuery->meta::relational::functions::sqlQueryToString::processOperation(meta::relational::runtime::DatabaseType.H2, $extensions)+'\n',|''));
Expand Down
4 changes: 2 additions & 2 deletions ...e-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,12 +164,12 @@ function <<access.private>> meta::alloy::service::execution::replaceWithEmptyVal
if ($replaced->contains(',,'), | meta::alloy::service::execution::replaceWithEmptyValue($replaced, $delimForEmptyCsvField), | $replaced);
}

function <<doc.deprecated>> meta::alloy::service::execution::setUpDataSQLs(data:String[1], db:Database[1]) : String[*]
function <<doc.deprecated>> meta::alloy::service::execution::setUpDataSQLs(data:String[1], db:Database[*]) : String[*]
{
meta::alloy::service::execution::setUpDataSQLs($data, $db, createDbConfig(DatabaseType.H2));
}

function <<doc.deprecated>> meta::alloy::service::execution::setUpDataSQLs(records:List<String>[*], db:Database[1]) : String[*]
function <<doc.deprecated>> meta::alloy::service::execution::setUpDataSQLs(records:List<String>[*], db:Database[*]) : String[*]
{
meta::alloy::service::execution::setUpDataSQLs($records, $db, createDbConfig(DatabaseType.H2));
}
Expand Down
6 changes: 3 additions & 3 deletions ...es/core_relational/relational/postprocessor/defaultPostProcessor/dbSpecificProcessor.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ import meta::relational::functions::sqlQueryToString::*;
function meta::relational::mapping::dbSpecificProcessor(query:SQLQuery[1], runtime:Runtime[1], store:Database[0..1], exeCtx:ExecutionContext[1], extensions:Extension[*]): PostProcessorResult[1]
{
let connectionStore = $runtime.connectionStores->filter(c|$c.connection->instanceOf(DatabaseConnection));
let conn = $connectionStore.connection->cast(@DatabaseConnection);
let conn = $connectionStore.connection->distinct()->cast(@DatabaseConnection);
let postProcessorQuery = if($conn->isNotEmpty(),
| let type = $conn.type;
| let type = $conn.type->distinct();
assert(type->size()==1, 'found not enough or too many DB types');
if ($type->toOne()->createDbConfig([]).dbExtension.aliasLimit->isNotEmpty(),
| trimColumnNamePostProcessors()->fold({pp,q|$pp->eval($q,$runtime).values->toOne()}, $query->cast(@SelectSQLQuery));,
Expand All @@ -41,7 +41,7 @@ function meta::relational::mapping::dbSpecificProcessor(query:SQLQuery[1], runti

let postProcessorOutput = ^PostProcessorResult(query = $postProcessorQuery, resultPostProcessor = {result:Result<Any|*>[1] | $result});

let dbConfig = createDbConfig($conn.type->toOne());
let dbConfig = createDbConfig($conn.type->distinct()->toOne());
if($dbConfig.dbExtension.preAndFinallyExecutionSQLQuery->isNotEmpty(),
| let sqlPairs = if($conn->isNotEmpty() && $conn->toOne()->instanceOf(DatabaseConnection),
|$dbConfig.preAndFinallyExecutionSQLQuery($conn->toOne()->cast(@DatabaseConnection)),
Expand Down
3 changes: 1 addition & 2 deletions ...re_relational/relational/postprocessor/defaultPostProcessor/processObjectReferenceIn.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,8 @@ function meta::relational::postProcessor::getObjectReferenceStoreType():String[1
function meta::relational::postProcessor::processObjectReferenceInOperation(query:SQLQuery[1], runtime:Runtime[1], store:Database[0..1], exeCtx:ExecutionContext[1], extensions:Extension[*]):PostProcessorResult[1]
{
let connectionPre = $runtime->connectionByElement($store->toOne());
let connectionStorePre = $runtime.connectionStores->filter(c|$c.connection==$connectionPre)->toOne();
let connectionStorePre = $runtime.connectionStores->filter(c|$c.connection==$connectionPre);
let connection = $connectionPre->meta::relational::mapping::updateConnection($extensions)->cast(@DatabaseConnection);
let connectionStore = ^$connectionStorePre(connection=$connection);
let tempTableNamePrefix = 'tempTableForObjectReferenceIn_';
let columnName = 'ObjectReferenceInCollectionColumn';

Expand Down
2 changes: 1 addition & 1 deletion ...relational/relational/postprocessor/defaultPostProcessor/trimColumnNamePostProcessor.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ function meta::relational::postProcessor::reAliasColumnName::lengthConfig(runtim
{
let type = $runtime.connectionStores.connection->filter(c|$c->instanceOf(DatabaseConnection))->cast(@DatabaseConnection).type;

let length = $type->cast(@DatabaseType)->toOne()->createDbConfig([]).dbExtension.aliasLimit;
let length = $type->cast(@DatabaseType)->distinct()->toOne('Provided runtime has multiple database connections of different types')->createDbConfig([]).dbExtension.aliasLimit;
if($length->isEmpty(),
| fail('Db not configured for column rename');0;,
| $length
Expand Down
23 changes: 12 additions & 11 deletions ...lStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/DDL/toDDL.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ function meta::relational::functions::toDDL::createTableStatement(database:Datab
function
{doc.doc = 'Input CSV file must contain one empty line between each table and two empty lines after the last table'}
meta::relational::functions::database::loadCsvDataToDbTable(csvData:List<String>[*],
db:Database[1],
db:Database[*],
databaseConnection:DatabaseConnection[1],
tablePostProcessor:Function<{Table[1]->Table[1]}>[1]
):Any[*]
Expand All @@ -92,7 +92,7 @@ Class meta::relational::functions::database::Res
}

function meta::relational::functions::database::loadCsvDataToDbTable(csvData:List<String>[*],
db:Database[1],
inputDb:Database[*],
dbConfig:DbConfig[1],
tablePostProcessor:Function<{Table[1]->Table[1]}>[1]
) : String[*]
Expand All @@ -101,13 +101,14 @@ function meta::relational::functions::database::loadCsvDataToDbTable(csvData:Lis
| let schemaName = $tab.p.values->at(0).values->at(0);
let tableName = $tab.p.values->at(1).values->at(0);
let columnNames = $tab.p.values->at(2).values;
let schema = $db->schema($schemaName);
assertNotEmpty($schema, | 'Schema "' + $schemaName + '" not found in database "' + $db->elementToPath() + '"');
let table = $schema->toOne()->table($tableName);
assertNotEmpty($table,| 'Table "' + $tableName + '" not found in Schema "' + $schemaName + '" in Database "' + $db->elementToPath() + '"');
let dbNames = $inputDb->map(d|$d->elementToPath())->joinStrings(', ');
let db = ^meta::relational::metamodel::Database(includes=$inputDb);
let schema = $db->schema($schemaName)->toOne('There should be exactly 1 schema with name ' + $schemaName + ' found in the provided runtime database(s) ['+ $dbNames +'] at the given connection.');
let table = $schema->table($tableName);
assertNotEmpty($table,| 'Table "' + $tableName + '" not found in Schema "' + $schemaName + '" in Database(s) ' + $dbNames);
let pTable = $tablePostProcessor->eval($table)->toOne();
let columns = $columnNames->map(cName | let c = $pTable->toOne().columns->cast(@Column)->filter(tc| $tc.name->equalIgnoreCase($cName->trim()))->first();
assertNotEmpty($c,| 'Column "' + $cName + '" not found in Table "' + $tableName + '" in Database "' + $db->elementToPath() + '"');
assertNotEmpty($c,| 'Column "' + $cName + '" not found in Table "' + $tableName + '" in Database(s) ' + $dbNames);
$c; );
let sql = loadValuesToDbTable2(list($tab.p.values->slice(3,$tab.p.values->size())),
$pTable, $columns, $dbConfig);
Expand Down Expand Up @@ -143,9 +144,9 @@ function meta::alloy::service::execution::schemaAndTableSetup(db:Database[1], db
$schemaSetup->concatenate($tableSetup);
}

function meta::alloy::service::execution::setUpDataSQLs(data:String[1], db:Database[1], dbConfig:DbConfig[1]) : String[*]
function meta::alloy::service::execution::setUpDataSQLs(data:String[1], db:Database[*], dbConfig:DbConfig[1]) : String[*]
{
let schemaAndTableSetup = $db->meta::alloy::service::execution::schemaAndTableSetup($dbConfig);
let schemaAndTableSetup = $db->map(d|$d->meta::alloy::service::execution::schemaAndTableSetup($dbConfig));

let formattedData = $data->split('\n')
->map(l|list($l->meta::alloy::service::execution::splitWithEmptyValue()))
Expand All @@ -154,8 +155,8 @@ function meta::alloy::service::execution::setUpDataSQLs(data:String[1], db:Datab
$schemaAndTableSetup->concatenate(loadCsvDataToDbTable($formattedData, $db, $dbConfig, t:Table[1]|$t));
}

function meta::alloy::service::execution::setUpDataSQLs(records:List<String>[*], db:Database[1], dbConfig:DbConfig[1]) : String[*]
function meta::alloy::service::execution::setUpDataSQLs(records:List<String>[*], db:Database[*], dbConfig:DbConfig[1]) : String[*]
{
let schemaAndTableSetup = $db->meta::alloy::service::execution::schemaAndTableSetup($dbConfig);
let schemaAndTableSetup = $db->map(d|$d->meta::alloy::service::execution::schemaAndTableSetup($dbConfig));
$schemaAndTableSetup->concatenate(loadCsvDataToDbTable($records, $db, $dbConfig, t:Table[1]|$t));
}
9 changes: 4 additions & 5 deletions .../src/main/resources/core_relational/relational/testDataGeneration/testDataGeneration.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -830,14 +830,13 @@ function meta::relational::testDataGeneration::executionPlan::planTestDataGenera
{
let newParameterNameValuePairs = meta::relational::testDataGeneration::generateParametersNameValuePair($func, [], $parameterNameValuePairs);
let relationTree = if($func->functionReturnType().rawType->toOne()->_subTypeOf(TabularDataSet), | scanRelations($func, $mapping, $runtime, $newParameterNameValuePairs, $extensions),| scanRelations($func, $mapping, $extensions));
let dbConnection = $runtime.connectionStores->filter(c|$c.connection->instanceOf(DatabaseConnection))->toOne();
let updatedDbConnection = $extensions.executionPlanExtensions().reprocessConnection->toOne()->eval($dbConnection)->cast(@DatabaseConnection);
let dbConnectionStore = ^$dbConnection(connection=$updatedDbConnection);
let newRuntime = ^$runtime(connectionStores=$dbConnectionStore);
let dbConnectionStore = $runtime.connectionStores->filter(c|$c.connection->instanceOf(DatabaseConnection));
let updatedDbConnectionStore = $extensions.executionPlanExtensions().processRuntimeTestConnections->toOne()->eval($dbConnectionStore);
let newRuntime = ^$runtime(connectionStores=$updatedDbConnectionStore);

let relationColumnMap = $relationTree->generateRelationColumnMap(^Map<NamedRelation, List<Column>>());

let node = meta::relational::testDataGeneration::executionPlan::planTestDataGeneration($relationTree, $newRuntime, $exeCtx, $updatedDbConnection, $relationColumnMap, [], ^TestDataGenerationConfig(rowIdentifiers=$rowIdentifiers, generateWithDefaultPKs=false, hashStrings=$hashStrings, temporalMilestoningDates=$temporalMilestoningDates), $extensions);
let node = meta::relational::testDataGeneration::executionPlan::planTestDataGeneration($relationTree, $newRuntime, $exeCtx, $updatedDbConnectionStore.connection->distinct()->toOne()->cast(@DatabaseConnection), $relationColumnMap, [], ^TestDataGenerationConfig(rowIdentifiers=$rowIdentifiers, generateWithDefaultPKs=false, hashStrings=$hashStrings, temporalMilestoningDates=$temporalMilestoningDates), $extensions);
^ExecutionPlan
(
func = $func,
Expand Down

0 comments on commit 7677747

Please sign in to comment.