feat: #217 migrate with transparent approach

.vscode/launch.json

@@ -4,7 +4,7 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "run integration",
+      "name": "run test app integration",
       "runtimeExecutable": "npx",
       "runtimeArgs": [
         "cds-ts",
@@ -19,20 +19,35 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "run build",
+      "name": "run build - transparent",
       "runtimeExecutable": "npx",
       "runtimeArgs": [
         "ts-node",
         "-e",
         "import {build} from \"${workspaceFolder}/src/scripts/build\"; build().catch(console.error);",
       ],
-      "cwd": "${workspaceFolder}/test/resources/integration",
+      "cwd": "${workspaceFolder}/test/resources/transparent",
       "skipFiles": [
         "<node_internals>/**"
       ],
       "sourceMaps": true,
       "envFile": "${workspaceFolder}/.env"
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "run test app transparent",
+      "runtimeExecutable": "npx",
+      "runtimeArgs": [
+        "cds-ts",
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/test/resources/transparent",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "envFile": "${workspaceFolder}/.env"
+    },
     {
       "type": "node",
       "name": "vscode-jest-tests",

src/AdminTool.ts

@@ -9,6 +9,8 @@ import { csnToEntity } from "./typeorm/entity";
 import { TypeORMLogger } from "./typeorm/logger";
 import { CDSMySQLDataSource } from "./typeorm/mysql";
 import { MysqlDatabaseOptions } from "./types";
+import fs from "fs/promises";
+import { migration_tool } from "./utils";
 
 
 /**
@@ -260,10 +262,25 @@ export class AdminTool {
   public async deploy(model: CSN, tenant: string = TENANT_DEFAULT) {
     try {
       this._logger.info("migrating schema for tenant", tenant.green);
+
       if (tenant !== TENANT_DEFAULT) { await this.createDatabase(tenant); }
-      const entities = csnToEntity(model);
       const migrateOptions = await this.getDataSourceOption(tenant);
-      await migrate({ ...migrateOptions, entities });
+
+      if (this._options?.tenant?.deploy?.transparent === true) {
+        this._logger.info("migrate with transparent approach");
+        const migrations = migration_tool.parse(
+          await fs.readFile(
+            path.join(this._cds.root, "db/migrations.sql"),
+            { encoding: "utf-8" }
+          )
+        );
+        await migrate({ ...migrateOptions }, false, migrations);
+      }
+      else {
+        const entities = csnToEntity(model);
+        await migrate({ ...migrateOptions, entities });
+      }
+
       if (this._options?.csv?.migrate !== false) {
         await this.deployCSV(tenant);
       }

src/constants.ts

@@ -43,4 +43,4 @@ export const DEFAULT_CSV_IDENTITY_CONCURRENCY = 10;
  */
 export const DEFAULT_MAX_ALLOWED_PACKED_MB = 512;
 
-export const MIGRATION_VERSION_PREFIX = "-- version number: ";
+export const MIGRATION_VERSION_PREFIX = "-- version: ";

src/scripts/build.ts

@@ -1,7 +1,7 @@
 // @ts-nocheck
 import { cwdRequireCDS } from "cds-internal-tool";
 import { readFile, mkdir, writeFile } from "fs/promises";
-import { Table } from "typeorm";
+import { EntitySchema, Table } from "typeorm";
 import { existsSync } from "fs";
 import path from "path";
 import { csnToEntity } from "../typeorm/entity";
@@ -11,11 +11,19 @@ import { View } from "typeorm/schema-builder/view/View";
 import { CDSMySQLDataSource } from "../typeorm/mysql";
 import { Query } from "typeorm/driver/Query";
 import { MigrationHistory, Query, Migration } from "../types";
-import { migration as migrationScript } from "../utils";
+import { migration_tool as migrationScript } from "../utils";
+
+
+const INITIAL_VERSION = 100;
 
 export async function build() {
   const cds = cwdRequireCDS();
   const logger = cds.log("build|cds|deploy");
+
+  if (cds.env.requires.db?.tenant?.deploy?.transparent !== true) {
+    logger.error("must enable transport deployment for 'cds-mysql'");
+    return;
+  }
   const base = path.join(cds.root, "db");
   const mysql_last_file_path = path.join(base, "last-dev", "mysql.json");
   const mysql_migration_file_path = path.join(base, "migrations.sql");
@@ -25,7 +33,7 @@ export async function build() {
   const current_hash = sha256(current_entities);
   const statements: Array<Query> = [];
   const migrations: Array<Migration> = [];
-  let nextVersion = 100;
+  let nextVersion = INITIAL_VERSION;
 
   const last_version_views: Array<View> = [];
   const last_version_tables: Array<Table> = [];
@@ -67,34 +75,13 @@ export async function build() {
 
   // >> typeorm internal hack
 
-  const ds = new CDSMySQLDataSource({
-    type: "mysql",
-    database: "__database_placeholder__", // dummy, it should never use
-    entities: current_entities,
-  });
-
-  await ds.buildMetadatas();
-  const queryRunner = ds.createQueryRunner();
-  queryRunner.loadedTables = last_version_tables;
-  queryRunner.loadedViews = last_version_views;
-  queryRunner.enableSqlMemory();
-  queryRunner.getCurrentDatabase = function () { return ds.options.database; };
-  queryRunner.getCurrentSchema = function () { return ds.options.database; };
-  queryRunner.insertViewDefinitionSql = function () { return undefined; };
-  queryRunner.deleteViewDefinitionSql = function () { return undefined; };
-
-  const builder = ds.driver.createSchemaBuilder();
-  builder.queryRunner = queryRunner;
-  await builder.executeSchemaSyncOperationsInProperOrder();
+  const queries = await build_migration_scripts(current_entities, last_version_tables, last_version_views);
 
   // << typeorm internal hack
 
-  statements.push(...queryRunner.getMemorySql().upQueries.filter(query => query !== undefined));
+  statements.push(...queries);
 
-  migrations.push({
-    version: nextVersion,
-    statements: statements
-  });
+  migrations.push({ version: nextVersion, at: new Date(), statements: statements, hash: current_hash });
 
   await writeFile(
     mysql_migration_file_path,
@@ -119,4 +106,43 @@ export async function build() {
   );
 
   logger.info("Written last dev json", mysql_last_file_path);
-}
+}
+
+/**
+ * 
+ * @ignore
+ * @internal
+ * @private
+ * @param current_entities 
+ * @param last_version_tables 
+ * @param last_version_views 
+ * @returns 
+ */
+export async function build_migration_scripts(
+  current_entities: Array<EntitySchema>,
+  last_version_tables: Table[],
+  last_version_views: View[]
+) {
+  const ds = new CDSMySQLDataSource({
+    type: "mysql",
+    database: "__database_placeholder__",
+    entities: current_entities,
+  });
+
+  await ds.buildMetadatas();
+  const queryRunner = ds.createQueryRunner();
+  queryRunner.loadedTables = last_version_tables;
+  queryRunner.loadedViews = last_version_views;
+  queryRunner.enableSqlMemory();
+  queryRunner.getCurrentDatabase = function () { return ds.options.database; };
+  queryRunner.getCurrentSchema = function () { return ds.options.database; };
+  queryRunner.insertViewDefinitionSql = function () { return undefined; };
+  queryRunner.deleteViewDefinitionSql = function () { return undefined; };
+
+  const builder = ds.driver.createSchemaBuilder();
+  builder.queryRunner = queryRunner;
+  await builder.executeSchemaSyncOperationsInProperOrder();
+
+  const queries = queryRunner.getMemorySql().upQueries.filter(query => query !== undefined);
+  return queries;
+}

src/typeorm/migrate.ts

@@ -7,6 +7,7 @@ import { TypeORMLogger } from "./logger";
 import { createHash } from "crypto";
 import { CDSMySQLDataSource } from "./mysql";
 import { last6Chars } from "../utils";
+import { Migration } from "../types";
 
 
 const MigrationHistory = new EntitySchema({
@@ -67,69 +68,93 @@ export function sha256(entities: Array<EntitySchema>) {
   return hash.digest("hex");
 }
 
+async function migrateMetadata(connectionOptions: DataSourceOptions): Promise<void> {
+  const ds = new CDSMySQLDataSource({
+    ...connectionOptions,
+    entities: CDSMysqlMetaTables,
+    logger: TypeORMLogger,
+  });
+  await ds.initialize();
+  const builder = ds.driver.createSchemaBuilder();
+  return await builder.build();
+}
+
 
 export async function migrate(connectionOptions: DataSourceOptions, dryRun: true): Promise<SqlInMemory>;
+export async function migrate(connectionOptions: DataSourceOptions, dryRun: false, migrations: Array<Migration>): Promise<void>;
 export async function migrate(connectionOptions: DataSourceOptions, dryRun?: false): Promise<void>;
-export async function migrate(connectionOptions: DataSourceOptions, dryRun = false): Promise<any> {
-
-  const isMetaMigration = connectionOptions.entities === CDSMysqlMetaTables;
-  const isTenantMigration = dryRun === false && !isMetaMigration;
+export async function migrate(connectionOptions: DataSourceOptions, dryRun = false, migrations?: Array<Migration>): Promise<any> {
 
   const logger = cwdRequireCDS().log("db|deploy|mysql|migrate|typeorm");
 
-  const entityHash = sha256(connectionOptions.entities as any as Array<any>);
-
-  if (isTenantMigration) {
-    logger.debug("start migrate meta tables for cds-mysql");
-    await migrate({ ...connectionOptions, entities: CDSMysqlMetaTables }, false);
-    logger.debug("migrate meta tables for cds-mysql successful");
-  }
+  logger.debug("start migrate meta tables for cds-mysql");
+  await migrateMetadata(connectionOptions);
+  logger.debug("migrate meta tables for cds-mysql successful");
 
   if (connectionOptions.entities === undefined || connectionOptions.entities?.length === 0) {
-    logger.warn("there is no entities provided, skip processing");
+    if (migrations === undefined) {
+      logger.warn("there is no entities provided, skip processing");
+      return;
+    }
   }
 
   const ds = new CDSMySQLDataSource({
     ...connectionOptions,
-    entities: connectionOptions.entities,
-    logging: true,
     logger: TypeORMLogger,
   });
 
   try {
-    (isTenantMigration ? logger.info : logger.debug)(
-      "migrate database",
-      String(connectionOptions.database).green,
-      "with hash",
-      last6Chars(entityHash).green,
-    );
 
     await ds.initialize();
 
-    // not dry run and not meta table migration
-    if (isTenantMigration) {
-      const [record] = await ds.query("SELECT HASH, MIGRATED_AT FROM cds_mysql_migration_history ORDER BY MIGRATED_AT DESC LIMIT 1 FOR UPDATE");
-      if (record?.HASH === entityHash) {
-        logger.debug(
-          "database model with hash",
-          last6Chars(entityHash).green,
-          "was ALREADY migrated at", String(record.MIGRATED_AT).green,
-        );
+    return await ds.transaction(async tx => {
+      const [record] = await tx.query("SELECT ID, HASH, MIGRATED_AT FROM cds_mysql_migration_history ORDER BY MIGRATED_AT DESC LIMIT 1 FOR UPDATE");
+
+      if (migrations !== undefined) {
+        // transparent migration
+        for (const m of migrations.filter(migration => migration.version > (record?.ID ?? Number.MIN_VALUE))) {
+          logger.info("migrate version", m.version, "generated at", m.at, "to hash", last6Chars(m.hash).green);
+          for (const ddl of m.statements) { await tx.query(ddl.query); }
+          await tx.createQueryBuilder()
+            .insert()
+            .into(MigrationHistory)
+            .values({ id: m.version, hash: m.hash })
+            .execute();
+        }
         return;
       }
-      await ds.createQueryBuilder()
-        .insert()
-        .into(MigrationHistory)
-        .values({ hash: entityHash })
-        .execute();
-    }
+      else {
+        // traditional way
+        const entityHash = sha256(connectionOptions.entities as any as Array<any>);
 
-    const builder = ds.driver.createSchemaBuilder();
+        logger.info(
+          "migrate database", String(connectionOptions.database).green,
+          "with hash", last6Chars(entityHash).green,
+        );
+
+        if (!dryRun) {
+          if (record?.HASH === entityHash) {
+            logger.debug(
+              "database model with hash", last6Chars(entityHash).green,
+              "was ALREADY migrated at", String(record.MIGRATED_AT).green,
+            );
+            return;
+          }
+          await tx.createQueryBuilder()
+            .insert()
+            .into(MigrationHistory)
+            .values({ hash: entityHash })
+            .execute();
+        }
+
+        const builder = ds.driver.createSchemaBuilder();
+        // dry run and return the DDL SQL
+        if (dryRun) { return await builder.log(); }
+        // perform DDL
+        return await builder.build();
+      }
+    });
 
-    // dry run and return the DDL SQL
-    if (dryRun) { return await builder.log(); }
-    // perform DDL
-    await builder.build();
   }
   catch (error) {
     logger.error("migrate database failed:", error);