From c2a6fe8484d15173d1ffd79c20fea7e1174d9864 Mon Sep 17 00:00:00 2001 From: Rico Hermans Date: Thu, 19 Oct 2023 10:32:23 +0200 Subject: [PATCH] feat: diff spec databases (#596) Adds a diffing tool to the service spec repository. Its goal is to print the differences between 2 database files, in terms of services and their properties added, removed and updated. By default, the diff is printed in a nice ASCII art tree, but it can also be printed in JSON format by passing the `--json` flag. This diff tool will be used in the future to make sure that when we refactor how the spec database is loaded, we can ensure that the database before the refactor and after the refactor is the same (because the diff will be empty). We've decided that while the `tskb` project could potentially support a generic database-level diff, it's probably easiest for now to just diff at the application level (i.e., the code doing the diffing knows about services, resources, properties and attributes). * Diff types have been added to the `service-spec-types` package. * The diff tool itself lives in the `service-spec-build` package. NOTE: the ASCII tree print is currently broken, but we are merging this early because we need the diffing functionality itself to validate other changes. --------- Co-authored-by: Kaizen Conroy --- .projenrc.ts | 6 +- .../service-spec-build/.projen/deps.json | 9 + .../service-spec-build/.projen/tasks.json | 11 +- .../@aws-cdk/service-spec-build/package.json | 5 +- .../service-spec-build/src/cli/diff-db.ts | 42 +++ .../service-spec-build/src/db-diff.ts | 150 +++++++++ .../service-spec-build/src/diff-fmt.ts | 284 ++++++++++++++++++ .../service-spec-build/src/diff-helpers.ts | 150 +++++++++ .../@aws-cdk/service-spec-build/src/index.ts | 1 + .../service-spec-build/src/tree-emitter.ts | 104 +++++++ .../service-spec-build/tsconfig.dev.json | 4 +- .../@aws-cdk/service-spec-build/tsconfig.json | 4 +- .../@aws-cdk/service-spec-types/src/index.ts | 1 + .../service-spec-types/src/types/database.ts | 5 +- .../service-spec-types/src/types/diff.ts | 61 ++++ .../service-spec-types/src/types/resource.ts | 10 +- yarn.lock | 7 +- 17 files changed, 840 insertions(+), 14 deletions(-) create mode 100644 packages/@aws-cdk/service-spec-build/src/cli/diff-db.ts create mode 100644 packages/@aws-cdk/service-spec-build/src/db-diff.ts create mode 100644 packages/@aws-cdk/service-spec-build/src/diff-fmt.ts create mode 100644 packages/@aws-cdk/service-spec-build/src/diff-helpers.ts create mode 100644 packages/@aws-cdk/service-spec-build/src/tree-emitter.ts create mode 100644 packages/@aws-cdk/service-spec-types/src/types/diff.ts diff --git a/.projenrc.ts b/.projenrc.ts index 4705ab316..bee2c1578 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -113,7 +113,7 @@ const serviceSpecBuild = new TypeScriptWorkspace({ parent: repo, name: '@aws-cdk/service-spec-build', description: 'Build the service spec from service-spec-sources to service-spec', - deps: [tsKb, serviceSpecSources, serviceSpecTypes], + deps: [tsKb, serviceSpecTypes, 'commander', 'chalk@^4', serviceSpecSources], devDeps: ['source-map-support'], private: true, }); @@ -125,6 +125,10 @@ serviceSpecBuild.tasks.addTask('analyze:db', { exec: 'ts-node src/cli/analyze-db', receiveArgs: true, }); +serviceSpecBuild.tasks.addTask('diff:db', { + exec: 'ts-node src/cli/diff-db', + receiveArgs: true, +}); serviceSpecBuild.gitignore.addPatterns('db.json'); serviceSpecBuild.gitignore.addPatterns('build-report'); diff --git a/packages/@aws-cdk/service-spec-build/.projen/deps.json b/packages/@aws-cdk/service-spec-build/.projen/deps.json index df45f1629..5b282e303 100644 --- a/packages/@aws-cdk/service-spec-build/.projen/deps.json +++ b/packages/@aws-cdk/service-spec-build/.projen/deps.json @@ -89,6 +89,15 @@ { "name": "@cdklabs/tskb", "type": "runtime" + }, + { + "name": "chalk", + "version": "^4", + "type": "runtime" + }, + { + "name": "commander", + "type": "runtime" } ], "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"npx projen\"." diff --git a/packages/@aws-cdk/service-spec-build/.projen/tasks.json b/packages/@aws-cdk/service-spec-build/.projen/tasks.json index ee5a0d6d5..1b59a6fa5 100644 --- a/packages/@aws-cdk/service-spec-build/.projen/tasks.json +++ b/packages/@aws-cdk/service-spec-build/.projen/tasks.json @@ -54,7 +54,7 @@ }, "steps": [ { - "exec": "npm-check-updates --upgrade --target=minor --peer --dep=dev,peer,prod,optional --filter=@types/jest,@types/node,@typescript-eslint/eslint-plugin,@typescript-eslint/parser,eslint-config-prettier,eslint-import-resolver-node,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,eslint,jest,jest-junit,npm-check-updates,prettier,projen,source-map-support,ts-jest,typescript" + "exec": "npm-check-updates --upgrade --target=minor --peer --dep=dev,peer,prod,optional --filter=@types/jest,@types/node,@typescript-eslint/eslint-plugin,@typescript-eslint/parser,eslint-config-prettier,eslint-import-resolver-node,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,eslint,jest,jest-junit,npm-check-updates,prettier,projen,source-map-support,ts-jest,typescript,chalk,commander" } ] }, @@ -77,6 +77,15 @@ } ] }, + "diff:db": { + "name": "diff:db", + "steps": [ + { + "exec": "ts-node src/cli/diff-db", + "receiveArgs": true + } + ] + }, "eslint": { "name": "eslint", "description": "Runs eslint against the codebase", diff --git a/packages/@aws-cdk/service-spec-build/package.json b/packages/@aws-cdk/service-spec-build/package.json index 338918274..4c0902030 100644 --- a/packages/@aws-cdk/service-spec-build/package.json +++ b/packages/@aws-cdk/service-spec-build/package.json @@ -9,6 +9,7 @@ "check-for-updates": "npx projen check-for-updates", "compile": "npx projen compile", "default": "npx projen default", + "diff:db": "npx projen diff:db", "eslint": "npx projen eslint", "gather-versions": "npx projen gather-versions", "nx": "npx projen nx", @@ -44,7 +45,9 @@ "dependencies": { "@aws-cdk/service-spec-sources": "^0.0.0", "@aws-cdk/service-spec-types": "^0.0.0", - "@cdklabs/tskb": "^0.0.0" + "@cdklabs/tskb": "^0.0.0", + "chalk": "^4", + "commander": "^11.1.0" }, "main": "lib/index.js", "license": "Apache-2.0", diff --git a/packages/@aws-cdk/service-spec-build/src/cli/diff-db.ts b/packages/@aws-cdk/service-spec-build/src/cli/diff-db.ts new file mode 100644 index 000000000..bae844674 --- /dev/null +++ b/packages/@aws-cdk/service-spec-build/src/cli/diff-db.ts @@ -0,0 +1,42 @@ +import { loadDatabase } from '@aws-cdk/service-spec-types'; +import { Command } from 'commander'; +import { DbDiff } from '../db-diff'; +import { DiffFormatter } from '../diff-fmt'; + +async function main() { + const program = new Command(); + + program + .name('diff-db') + .description('Calculate differences between two databases') + .argument('', 'First database file') + .argument('', 'Second database file') + .option('-j, --json', 'Output json', false) + .parse(); + const options = program.opts(); + const args = program.args; + + const db1 = await loadDatabase(args[0]); + const db2 = await loadDatabase(args[1]); + + const result = new DbDiff(db1, db2).diff(); + + const hasChanges = + Object.keys(result.services.added ?? {}).length + + Object.keys(result.services.removed ?? {}).length + + Object.keys(result.services.updated ?? {}).length > + 0; + + if (options.json) { + console.log(JSON.stringify(result, undefined, 2)); + } else { + console.log(new DiffFormatter(db1, db2).format(result)); + } + + process.exitCode = hasChanges ? 1 : 0; +} + +main().catch((e) => { + console.error(e); + process.exitCode = 1; +}); diff --git a/packages/@aws-cdk/service-spec-build/src/db-diff.ts b/packages/@aws-cdk/service-spec-build/src/db-diff.ts new file mode 100644 index 000000000..78a6cce79 --- /dev/null +++ b/packages/@aws-cdk/service-spec-build/src/db-diff.ts @@ -0,0 +1,150 @@ +import { + Attribute, + Property, + PropertyType, + Resource, + RichPropertyType, + Service, + SpecDatabase, + SpecDatabaseDiff, + TypeDefinition, + UpdatedAttribute, + UpdatedProperty, + UpdatedResource, + UpdatedService, + UpdatedTypeDefinition, +} from '@aws-cdk/service-spec-types'; +import { + diffByKey, + collapseUndefined, + diffScalar, + collapseEmptyDiff, + jsonEq, + diffMap, + diffList, + diffField, +} from './diff-helpers'; + +export class DbDiff { + constructor(private readonly db1: SpecDatabase, private readonly db2: SpecDatabase) {} + + public diff(): SpecDatabaseDiff { + return { + services: diffByKey( + this.db1.all('service'), + this.db2.all('service'), + (service) => service.name, + (a, b) => this.diffService(a, b), + ), + }; + } + + private diffService(a: Service, b: Service): UpdatedService | undefined { + return collapseUndefined({ + capitalized: diffScalar(a, b, 'capitalized'), + cloudFormationNamespace: diffScalar(a, b, 'cloudFormationNamespace'), + name: diffScalar(a, b, 'name'), + shortName: diffScalar(a, b, 'shortName'), + resourceDiff: this.diffServiceResources(a, b), + }); + } + + private diffServiceResources(a: Service, b: Service): UpdatedService['resourceDiff'] { + const aRes = this.db1.follow('hasResource', a).map((r) => r.entity); + const bRes = this.db2.follow('hasResource', b).map((r) => r.entity); + + return collapseEmptyDiff( + diffByKey( + aRes, + bRes, + (resource) => resource.cloudFormationType, + (x, y) => this.diffResource(x, y), + ), + ); + } + + private diffResource(a: Resource, b: Resource): UpdatedResource | undefined { + return collapseUndefined({ + cloudFormationTransform: diffScalar(a, b, 'cloudFormationTransform'), + documentation: diffScalar(a, b, 'documentation'), + cloudFormationType: diffScalar(a, b, 'cloudFormationType'), + isStateful: diffScalar(a, b, 'isStateful'), + identifier: diffField(a, b, 'identifier', jsonEq), + name: diffScalar(a, b, 'name'), + scrutinizable: diffScalar(a, b, 'scrutinizable'), + tagInformation: diffField(a, b, 'tagInformation', jsonEq), + attributes: collapseEmptyDiff(diffMap(a.attributes, b.attributes, (x, y) => this.diffAttribute(x, y))), + properties: collapseEmptyDiff(diffMap(a.properties, b.properties, (x, y) => this.diffProperty(x, y))), + typeDefinitionDiff: this.diffResourceTypeDefinitions(a, b), + }); + } + + private diffAttribute(a: Attribute, b: Attribute): UpdatedAttribute | undefined { + const eqType = this.eqType.bind(this); + + const anyDiffs = collapseUndefined({ + documentation: diffScalar(a, b, 'documentation'), + previousTypes: collapseEmptyDiff(diffList(a.previousTypes ?? [], b.previousTypes ?? [], eqType)), + type: diffField(a, b, 'type', eqType), + }); + + if (anyDiffs) { + return { old: a, new: b }; + } + return undefined; + } + + private diffProperty(a: Property, b: Property): UpdatedProperty | undefined { + const eqType = this.eqType.bind(this); + + const anyDiffs = collapseUndefined({ + documentation: diffScalar(a, b, 'documentation'), + defaultValue: diffScalar(a, b, 'defaultValue'), + deprecated: diffScalar(a, b, 'deprecated'), + required: diffScalar(a, b, 'required'), + scrutinizable: diffScalar(a, b, 'scrutinizable'), + previousTypes: collapseEmptyDiff(diffList(a.previousTypes ?? [], b.previousTypes ?? [], eqType)), + type: diffField(a, b, 'type', eqType), + }); + + if (anyDiffs) { + return { old: a, new: b }; + } + return undefined; + } + + private diffResourceTypeDefinitions(a: Resource, b: Resource): UpdatedResource['typeDefinitionDiff'] { + const aTypes = this.db1.follow('usesType', a).map((r) => r.entity); + const bTypes = this.db2.follow('usesType', b).map((r) => r.entity); + + return collapseEmptyDiff( + diffByKey( + aTypes, + bTypes, + (type) => type.name, + (x, y) => this.diffTypeDefinition(x, y), + ), + ); + } + + private diffTypeDefinition(a: TypeDefinition, b: TypeDefinition): UpdatedTypeDefinition | undefined { + return collapseUndefined({ + documentation: diffScalar(a, b, 'documentation'), + name: diffScalar(a, b, 'name'), + mustRenderForBwCompat: diffScalar(a, b, 'mustRenderForBwCompat'), + properties: collapseEmptyDiff(diffMap(a.properties, b.properties, (x, y) => this.diffProperty(x, y))), + }); + } + + /** + * Tricky -- we have to deep-compare all the type references which will have different ids in + * different databases. + * + * Solve it by doing a string-render and comparing those (for now). + */ + private eqType(a: PropertyType, b: PropertyType): boolean { + const s1 = new RichPropertyType(a).stringify(this.db1, false); + const s2 = new RichPropertyType(b).stringify(this.db2, false); + return s1 === s2; + } +} diff --git a/packages/@aws-cdk/service-spec-build/src/diff-fmt.ts b/packages/@aws-cdk/service-spec-build/src/diff-fmt.ts new file mode 100644 index 000000000..549521b1b --- /dev/null +++ b/packages/@aws-cdk/service-spec-build/src/diff-fmt.ts @@ -0,0 +1,284 @@ +import { + Attribute, + Deprecation, + MapDiff, + Property, + Resource, + RichPropertyType, + ScalarDiff, + Service, + SpecDatabase, + SpecDatabaseDiff, + TypeDefinition, + UpdatedAttribute, + UpdatedProperty, + UpdatedResource, + UpdatedService, + UpdatedTypeDefinition, +} from '@aws-cdk/service-spec-types'; +import chalk from 'chalk'; +import { TreeEmitter } from './tree-emitter'; + +const ADDITION = '[+]'; +const UPDATE = '[~]'; +const REMOVAL = '[-]'; +const META_INDENT = ' '; + +const [OLD_DB, NEW_DB] = [0, 1]; + +export class DiffFormatter { + private readonly tree = new TreeEmitter(); + private readonly dbs: SpecDatabase[]; + + constructor(db1: SpecDatabase, db2: SpecDatabase) { + this.dbs = [db1, db2]; + } + + public format(diff: SpecDatabaseDiff): string { + this.tree.clear(); + + this.renderMapDiff( + 'service', + diff.services, + (s, db) => this.renderService(s, db), + (u) => this.renderUpdatedService(u), + ); + + return this.tree.toString(); + } + + private renderService(s: Service, db: number) { + this.tree.plainStringBlock( + META_INDENT, + listFromProps(s, ['capitalized', 'cloudFormationNamespace', 'name', 'shortName']), + ); + + this.tree.emitList( + this.dbs[db].follow('hasResource', s).map((x) => x.entity), + (resource, last) => + this.tree.withBullet(last, () => { + this.renderResource(resource, db); + }), + ); + } + + private renderUpdatedService(s: UpdatedService) { + const d = pick(s, ['capitalized', 'cloudFormationNamespace', 'name', 'shortName']); + this.tree.plainStringBlock(META_INDENT, listFromDiffs(d)); + + this.renderMapDiff( + 'resource', + s.resourceDiff, + (r, db) => this.renderResource(r, db), + (u) => this.renderUpdatedResource(u), + ); + } + + private renderResource(r: Resource, db: number) { + this.tree.plainStringBlock( + META_INDENT, + listFromProps(r, [ + 'name', + 'identifier', + 'cloudFormationType', + 'cloudFormationTransform', + 'documentation', + 'identifier', + 'isStateful', + 'scrutinizable', + 'tagInformation', + ]), + ); + + // FIXME: props, attributes + + this.tree.emitList( + this.dbs[db].follow('usesType', r).map((x) => x.entity), + (typeDef, last) => + this.tree.withBullet(last, () => { + this.renderTypeDefinition(typeDef, db); + }), + ); + } + + private renderUpdatedResource(s: UpdatedResource) { + const d = pick(s, [ + 'name', + 'identifier', + 'cloudFormationType', + 'cloudFormationTransform', + 'documentation', + 'identifier', + 'isStateful', + 'scrutinizable', + 'tagInformation', + ]); + this.tree.plainStringBlock(META_INDENT, listFromDiffs(d)); + + this.tree.withPrefix(META_INDENT, () => { + if (s.properties) { + this.tree.emit('properties\n'); + this.renderMapDiff( + 'prop', + s.properties, + (p, db) => this.renderProperty(p, db), + (u) => this.renderUpdatedProperty(u), + ); + } + + if (s.attributes) { + this.tree.emit('attributes\n'); + this.renderMapDiff( + 'attr', + s.attributes, + (p, db) => this.renderAttribute(p, db), + (u) => this.renderUpdatedAttribute(u), + ); + } + }); + + this.renderMapDiff( + 'type', + s.typeDefinitionDiff, + (p, db) => this.renderTypeDefinition(p, db), + (u) => this.renderUpdatedTypeDefinition(u), + ); + } + + private renderTypeDefinition(r: TypeDefinition, db: number) { + this.tree.plainStringBlock(META_INDENT, listFromProps(r, ['documentation', 'mustRenderForBwCompat', 'name'])); + + // Properties + this.tree.emitList(Object.entries(r.properties), ([name, p], last) => { + this.tree.withBullet(last, () => { + this.tree.emit(`${name}: `); + this.renderProperty(p, db); + }); + }); + } + + private renderUpdatedTypeDefinition(t: UpdatedTypeDefinition) { + const d = pick(t, ['documentation', 'mustRenderForBwCompat', 'name']); + this.tree.plainStringBlock(META_INDENT, listFromDiffs(d)); + + this.tree.withPrefix(' ', () => { + this.renderMapDiff( + 'prop', + t.properties, + (p, db) => this.renderProperty(p, db), + (u) => this.renderUpdatedProperty(u), + ); + }); + } + + private renderProperty(p: Property, db: number) { + const types = [p.type, ...(p.previousTypes ?? []).reverse()]; + this.tree.emit(types.map((type) => new RichPropertyType(type).stringify(this.dbs[db], false)).join(' ⇐ ')); + + const attributes = []; + if (p.defaultValue) { + attributes.push(`default=${render(p.defaultValue)}`); + } + if (p.deprecated && p.deprecated !== Deprecation.NONE) { + attributes.push(`deprecated=${p.deprecated}`); + } + // FIXME: Documentation? + + if (attributes.length) { + this.tree.emit(` (${attributes.join(', ')})`); + } + } + + private renderUpdatedProperty(t: UpdatedProperty) { + this.tree.withColor(chalk.red, () => { + this.renderProperty(t.old, OLD_DB); + }); + this.tree.emit('\n'); + this.tree.withColor(chalk.green, () => { + this.renderProperty(t.new, NEW_DB); + }); + } + + private renderAttribute(a: Attribute, db: number) { + const types = [a.type, ...(a.previousTypes ?? []).reverse()]; + this.tree.emit(types.map((type) => new RichPropertyType(type).stringify(this.dbs[db], false)).join(' ⇐ ')); + } + + private renderUpdatedAttribute(t: UpdatedAttribute) { + this.tree.withColor(chalk.red, () => { + this.renderAttribute(t.old, OLD_DB); + }); + this.tree.emit('\n'); + this.tree.withColor(chalk.green, () => { + this.renderAttribute(t.new, NEW_DB); + }); + } + + private renderMapDiff( + type: string, + diff: MapDiff | undefined, + renderEl: (x: E, dbI: number) => void, + renderUpdated: (x: U) => void, + ) { + if (!diff) { + return; + } + + // Turn the lists into maps + const keys = Array.from( + new Set([ + ...Object.keys(diff.added ?? {}), + ...Object.keys(diff.removed ?? {}), + ...Object.keys(diff.updated ?? {}), + ]), + ); + keys.sort((a, b) => a.localeCompare(b)); + + this.tree.emitList(keys, (key, last) => { + if (diff.added?.[key]) { + this.tree.withColor(chalk.green, () => + this.tree.withBullet(last, () => { + this.tree.emit(ADDITION); + this.tree.emit(` ${type} ${key}\n`); + renderEl(diff.added?.[key]!, NEW_DB); + }), + ); + } else if (diff.removed?.[key]) { + this.tree.withColor(chalk.green, () => + this.tree.withBullet(last, () => { + this.tree.emit(REMOVAL); + this.tree.emit(` ${type} ${key}\n`); + renderEl(diff.removed?.[key]!, OLD_DB); + }), + ); + } else if (diff.updated?.[key]) { + this.tree.withBullet(last, () => { + this.tree.emit(chalk.yellow(UPDATE)); + this.tree.emit(` ${type} ${key}\n`); + renderUpdated(diff.updated?.[key]!); + }); + } + }); + } +} + +function listFromProps(a: A, ks: K[]) { + return listFromObj(pick(a, ks)); +} + +function listFromObj(xs: Record): string[] { + return Object.entries(xs).map(([k, v]) => `${String(k)}: ${render(v)}`); +} + +function pick(a: A, ks: K[]): Pick { + const pairs = ks.flatMap((k) => (a[k] !== undefined ? [[k, a[k]] as const] : [])); + return Object.fromEntries(pairs) as any; +} + +function listFromDiffs(xs: Record | undefined>): string[] { + return Object.entries(xs).map(([key, diff]) => `${String(key)}: ${render(diff?.old)} → ${render(diff?.new)}`); +} + +function render(x: unknown) { + return typeof x === 'object' ? JSON.stringify(x) : `${x}`; +} diff --git a/packages/@aws-cdk/service-spec-build/src/diff-helpers.ts b/packages/@aws-cdk/service-spec-build/src/diff-helpers.ts new file mode 100644 index 000000000..3100fb81a --- /dev/null +++ b/packages/@aws-cdk/service-spec-build/src/diff-helpers.ts @@ -0,0 +1,150 @@ +import { MapDiff, ListDiff, ScalarDiff } from '@aws-cdk/service-spec-types'; + +export function diffByKey(a: A[], b: A[], keyFn: (x: A) => string, updatedDiff: (a: A, b: A) => B | undefined) { + return diffMap( + Object.fromEntries(a.map((x) => [keyFn(x), x])), + Object.fromEntries(b.map((x) => [keyFn(x), x])), + updatedDiff, + ); +} + +export function diffMap( + a: Record, + b: Record, + updatedDiff: (a: A, b: A) => B | undefined, +): MapDiff { + const added = new Set(Object.keys(b)); + + const ret: Required> = { + added: {}, + removed: {}, + updated: {}, + }; + + for (const [key, value] of Object.entries(a)) { + if (key in b) { + const deep = updatedDiff(value, b[key]); + if (deep) { + ret.updated[key] = deep; + } + added.delete(key); + } else { + ret.removed[key] = value; + } + } + for (const key of added) { + ret.added[key] = b[key]; + } + + return { + ...(Object.keys(ret.added).length > 0 ? { added: ret.added } : {}), + ...(Object.keys(ret.removed).length > 0 ? { removed: ret.removed } : {}), + ...(Object.keys(ret.updated).length > 0 ? { updated: ret.updated } : {}), + }; +} + +/** + * Diff a list by quadratically comparing all elements + */ +export function diffList(as: A[], bs: A[], eq: Eq, updatedDiff: (a: A, b: A) => B | undefined): ListDiff; +export function diffList(as: A[], bs: A[], eq: Eq): ListDiff; +export function diffList( + as: A[], + bs: A[], + eq: Eq, + updatedDiff?: (a: A, b: A) => B | undefined, +): ListDiff { + const added: Array = [...bs]; + + const ret: Required> = { + added: [], + removed: [], + updated: [], + }; + + for (const a of as) { + let found = false; + for (let i = 0; i < added.length; i++) { + const b = added[i]; + if (b === undefined) { + continue; + } + + if (eq(a, b)) { + found = true; + const deep = updatedDiff?.(a, b); + if (deep) { + ret.updated.push(deep); + } + // Mark off + added[i] = undefined; + break; + } + } + + if (!found) { + ret.removed.push(a); + } + } + + for (const b of added) { + if (b !== undefined) { + ret.added.push(b); + } + } + + return { + ...(ret.added.length > 0 ? { added: ret.added } : {}), + ...(ret.removed.length > 0 ? { removed: ret.removed } : {}), + ...(ret.updated.length > 0 ? { updated: ret.updated } : {}), + }; +} + +export function tripleEq(a: A, b: A): boolean { + return a === b; +} + +export function jsonEq(a: A, b: A): boolean { + return JSON.stringify(a) === JSON.stringify(b); +} + +export function diffScalar( + a: A, + b: A, + k: K, +): A[K] extends string | number | boolean | undefined ? ScalarDiff> | undefined : void { + // Complex return type makes it so that the === comparison only works on scalars, and for other types + // the user must use diffField with a custom equality function + return diffField(a, b, k, tripleEq) as any; +} + +export function diffField( + a: A, + b: A, + k: K, + eq: Eq, +): ScalarDiff> | undefined { + if (eq(a[k], b[k])) { + return undefined; + } + return { + old: a[k]!, + new: b[k]!, + }; +} + +/** + * Return the object if it has any defined fields, otherwise undefined + */ +export function collapseUndefined(x: A): A | undefined { + return Object.keys(x).some((key) => (x as any)[key] !== undefined) ? x : undefined; +} + +export function collapseEmptyDiff | MapDiff>(x: A): A | undefined { + return Object.keys(x.added ?? {}).length + Object.keys(x.removed ?? {}).length + Object.keys(x.updated ?? {}).length > + 0 + ? x + : undefined; +} + +export type Eq = (x: A, y: A) => boolean; diff --git a/packages/@aws-cdk/service-spec-build/src/index.ts b/packages/@aws-cdk/service-spec-build/src/index.ts index a1b082de7..a1a2007c4 100644 --- a/packages/@aws-cdk/service-spec-build/src/index.ts +++ b/packages/@aws-cdk/service-spec-build/src/index.ts @@ -1 +1,2 @@ export * from './build-database'; +export * from './db-diff'; diff --git a/packages/@aws-cdk/service-spec-build/src/tree-emitter.ts b/packages/@aws-cdk/service-spec-build/src/tree-emitter.ts new file mode 100644 index 000000000..5da9f8d9d --- /dev/null +++ b/packages/@aws-cdk/service-spec-build/src/tree-emitter.ts @@ -0,0 +1,104 @@ +type Colorizer = (x: string) => string; + +/** + * A class to emit ASCII art tree structures + */ +export class TreeEmitter { + private readonly buffer = new Array(); + private readonly prefix = new Array(); + private readonly colors: Colorizer[] = [ident]; + + public clear() { + this.buffer.splice(0, this.buffer.length); + } + + public toString() { + return this.buffer.join(''); + } + + public emitList(as: A[], block: (a: A, last: boolean) => void) { + for (let i = 0; i < as.length; i++) { + if (i > 0) { + this.emit('\n'); + } + const last = i === as.length - 1; + block(as[i], last); + } + } + + public withBullet(last: boolean, block: () => void): void; + public withBullet(last: boolean, additionalIndent: string, block: () => void): void; + public withBullet(last: boolean, blockOrIndent: string | (() => void), block?: () => void) { + const header = last ? '└' : '├'; + const indent = (last ? ' ' : '│') + (typeof blockOrIndent === 'string' ? blockOrIndent : ' '); + const theBlock = typeof blockOrIndent === 'function' ? blockOrIndent : block; + + this.emit(header); + this.withPrefix(indent, theBlock!); + } + + public withHeader(header: string, indent: string, block: () => void) { + this.emit(header); + this.withPrefix(indent, block); + } + + public plainStringBlock(indent: string, as: string[]) { + if (as.length === 0) { + return; + } + + this.withHeader(indent, indent, () => { + this.emitList(as, (a) => { + this.emit(a); + }); + }); + } + + public emit(x: string) { + if (this.buffer.length && this.buffer[this.buffer.length - 1].endsWith('\n')) { + this.buffer.push(this.currentPrefix); + } + + // Replace newlines with the prefix, except if the string ends in one + const parts = x.split('\n'); + for (let i = 0; i < parts.length; i++) { + if (i > 0) { + this.buffer.push('\n'); + if (i < parts.length || parts[i] !== '') { + this.buffer.push(this.currentPrefix); + } + } + this.buffer.push(this.currentColor(parts[i])); + } + } + + private get currentPrefix() { + return this.prefix.join(''); + } + + private get currentColor() { + return this.colors[this.colors.length - 1]; + } + + public withPrefix(x: string, block: () => void) { + this.prefix.push(this.currentColor(x)); + try { + block(); + } finally { + this.prefix.pop(); + } + } + + public withColor(col: Colorizer, block: () => void) { + this.colors.push(col); + try { + block(); + } finally { + this.colors.pop(); + } + } +} + +function ident(x: string) { + return x; +} diff --git a/packages/@aws-cdk/service-spec-build/tsconfig.dev.json b/packages/@aws-cdk/service-spec-build/tsconfig.dev.json index 82d895d12..bb5b2a70f 100644 --- a/packages/@aws-cdk/service-spec-build/tsconfig.dev.json +++ b/packages/@aws-cdk/service-spec-build/tsconfig.dev.json @@ -40,10 +40,10 @@ "path": "../../@cdklabs/tskb" }, { - "path": "../service-spec-sources" + "path": "../service-spec-types" }, { - "path": "../service-spec-types" + "path": "../service-spec-sources" } ] } diff --git a/packages/@aws-cdk/service-spec-build/tsconfig.json b/packages/@aws-cdk/service-spec-build/tsconfig.json index 6d9d0016f..965ce942d 100644 --- a/packages/@aws-cdk/service-spec-build/tsconfig.json +++ b/packages/@aws-cdk/service-spec-build/tsconfig.json @@ -37,10 +37,10 @@ "path": "../../@cdklabs/tskb" }, { - "path": "../service-spec-sources" + "path": "../service-spec-types" }, { - "path": "../service-spec-types" + "path": "../service-spec-sources" } ] } diff --git a/packages/@aws-cdk/service-spec-types/src/index.ts b/packages/@aws-cdk/service-spec-types/src/index.ts index 60ae3e390..567d20206 100644 --- a/packages/@aws-cdk/service-spec-types/src/index.ts +++ b/packages/@aws-cdk/service-spec-types/src/index.ts @@ -2,3 +2,4 @@ export * from './types/database'; export * from './types/resource'; export * from './types/augmentations'; export * from './types/metrics'; +export * from './types/diff'; diff --git a/packages/@aws-cdk/service-spec-types/src/types/database.ts b/packages/@aws-cdk/service-spec-types/src/types/database.ts index 2167bf9a1..fabceb55d 100644 --- a/packages/@aws-cdk/service-spec-types/src/types/database.ts +++ b/packages/@aws-cdk/service-spec-types/src/types/database.ts @@ -1,4 +1,5 @@ import { promises as fs } from 'fs'; +import { gunzipSync } from 'zlib'; import { Database, entityCollection, fieldIndex, stringCmp } from '@cdklabs/tskb'; import { IsAugmentedResource, ResourceAugmentation } from './augmentations'; import { @@ -61,7 +62,9 @@ export function emptyDatabase() { export async function loadDatabase(pathToDb: string) { const db = emptyDatabase(); - db.load(JSON.parse(await fs.readFile(pathToDb, { encoding: 'utf-8' }))); + const contents = await fs.readFile(pathToDb); + const json = pathToDb.endsWith('.gz') ? gunzipSync(contents).toString('utf-8') : contents.toString('utf-8'); + db.load(JSON.parse(json)); return db; } diff --git a/packages/@aws-cdk/service-spec-types/src/types/diff.ts b/packages/@aws-cdk/service-spec-types/src/types/diff.ts new file mode 100644 index 000000000..bd5b6bdc9 --- /dev/null +++ b/packages/@aws-cdk/service-spec-types/src/types/diff.ts @@ -0,0 +1,61 @@ +import { Attribute, Property, Resource, Service, TypeDefinition } from './resource'; + +export interface SpecDatabaseDiff { + services: MapDiff; +} + +export interface ListDiff { + readonly added?: E[]; + readonly removed?: E[]; + readonly updated?: ED[]; +} + +export interface MapDiff { + readonly added?: Record; + readonly removed?: Record; + readonly updated?: Record; +} + +export interface UpdatedService { + readonly name?: ScalarDiff; + readonly shortName?: ScalarDiff; + readonly capitalized?: ScalarDiff; + readonly cloudFormationNamespace?: ScalarDiff; + readonly resourceDiff?: MapDiff; +} + +export interface UpdatedResource { + readonly name?: ScalarDiff; + readonly cloudFormationType?: ScalarDiff; + readonly cloudFormationTransform?: ScalarDiff; + readonly documentation?: ScalarDiff; + readonly properties?: MapDiff; + readonly attributes?: MapDiff; + readonly identifier?: ScalarDiff; + readonly isStateful?: ScalarDiff; + readonly tagInformation?: ScalarDiff; + readonly scrutinizable?: ScalarDiff; + readonly typeDefinitionDiff?: MapDiff; +} + +export interface UpdatedProperty { + readonly old: Property; + readonly new: Property; +} + +export interface UpdatedAttribute { + readonly old: Attribute; + readonly new: Attribute; +} + +export interface UpdatedTypeDefinition { + readonly name?: ScalarDiff; + readonly documentation?: ScalarDiff; + readonly properties?: MapDiff; + readonly mustRenderForBwCompat?: ScalarDiff; +} + +export interface ScalarDiff { + readonly old?: A; + readonly new?: A; +} diff --git a/packages/@aws-cdk/service-spec-types/src/types/resource.ts b/packages/@aws-cdk/service-spec-types/src/types/resource.ts index b1eeb7db4..2702a57b3 100644 --- a/packages/@aws-cdk/service-spec-types/src/types/resource.ts +++ b/packages/@aws-cdk/service-spec-types/src/types/resource.ts @@ -458,7 +458,7 @@ export class RichPropertyType { } } - public stringify(db: SpecDatabase): string { + public stringify(db: SpecDatabase, withId = true): string { switch (this.type.type) { case 'integer': case 'boolean': @@ -470,14 +470,14 @@ export class RichPropertyType { case 'tag': return this.type.type; case 'array': - return `Array<${new RichPropertyType(this.type.element).stringify(db)}>`; + return `Array<${new RichPropertyType(this.type.element).stringify(db, withId)}>`; case 'map': - return `Map`; + return `Map`; case 'ref': const type = db.get('typeDefinition', this.type.reference); - return `${type.name}(${this.type.reference.$ref})`; + return withId ? `${type.name}(${this.type.reference.$ref})` : type.name; case 'union': - return this.type.types.map((t) => new RichPropertyType(t).stringify(db)).join(' | '); + return this.type.types.map((t) => new RichPropertyType(t).stringify(db, withId)).join(' | '); } } diff --git a/yarn.lock b/yarn.lock index 5cb1e690c..5f83fffbf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1682,7 +1682,7 @@ chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: +chalk@^4, chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -1811,6 +1811,11 @@ commander@^11.0.0: resolved "https://registry.yarnpkg.com/commander/-/commander-11.0.0.tgz#43e19c25dbedc8256203538e8d7e9346877a6f67" integrity sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ== +commander@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-11.1.0.tgz#62fdce76006a68e5c1ab3314dc92e800eb83d906" + integrity sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ== + comment-json@4.2.2: version "4.2.2" resolved "https://registry.yarnpkg.com/comment-json/-/comment-json-4.2.2.tgz#5fae70a94e0c8f84a077bd31df5aa5269252f293"