From 9dc048123e449626777c52da01ac4a007d99a440 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Thu, 21 Mar 2024 21:24:29 +0200 Subject: [PATCH] fix(containers)!: migrate to `sim.Container` Use the shiny new `sim.Container` to implement the simulator version of `Workload`. BREAKING CHANGE: `sources` is a glob string instead of an array of strings. --- containers/api.w | 5 +- containers/helm.extern.d.ts | 282 +++++++++++++++++++++++++++++++++++ containers/package-lock.json | 10 +- containers/package.json | 7 +- containers/utils.extern.d.ts | 3 + containers/utils.js | 28 ---- containers/utils.w | 26 +--- containers/workload.sim.w | 190 +++++++---------------- containers/workload.tfaws.w | 17 ++- 9 files changed, 358 insertions(+), 210 deletions(-) create mode 100644 containers/helm.extern.d.ts create mode 100644 containers/utils.extern.d.ts diff --git a/containers/api.w b/containers/api.w index 6777719f..62e658d2 100644 --- a/containers/api.w +++ b/containers/api.w @@ -12,10 +12,9 @@ pub struct ContainerOpts { args: Array?; // container arguments /** - * A list of globs of local files to consider as input sources for the container. - * By default, the entire build context directory will be included. + * The glob to use in order to calculate the content hash (default: all files) */ - sources: Array?; + sources: str?; /** * a hash that represents the container source. if not set, diff --git a/containers/helm.extern.d.ts b/containers/helm.extern.d.ts new file mode 100644 index 00000000..4b12c1b3 --- /dev/null +++ b/containers/helm.extern.d.ts @@ -0,0 +1,282 @@ +export default interface extern { + toHelmChart: (wingdir: string, chart: Chart) => string, +} +/** Trait marker for classes that can be depended upon. +The presence of this interface indicates that an object has +an `IDependable` implementation. + +This interface can be used to take an (ordering) dependency on a set of +constructs. An ordering dependency implies that the resources represented by +those constructs are deployed before the resources depending ON them are +deployed. */ +export interface IDependable { +} +/** Options for `construct.addMetadata()`. */ +export interface MetadataOptions { + /** Include stack trace with metadata entry. */ + readonly stackTrace?: (boolean) | undefined; + /** A JavaScript function to begin tracing from. + This option is ignored unless `stackTrace` is `true`. */ + readonly traceFromFunction?: (any) | undefined; +} +/** Implement this interface in order for the construct to be able to validate itself. */ +export interface IValidation { + /** Validate the current construct. + This method can be implemented by derived constructs in order to perform + validation logic. It is called on all constructs before synthesis. + @returns An array of validation error messages, or an empty array if there the construct is valid. */ + readonly validate: () => (readonly (string)[]); +} +/** In what order to return constructs. */ +export enum ConstructOrder { + PREORDER = 0, + POSTORDER = 1, +} +/** An entry in the construct metadata table. */ +export interface MetadataEntry { + /** The data. */ + readonly data?: any; + /** Stack trace at the point of adding the metadata. + Only available if `addMetadata()` is called with `stackTrace: true`. */ + readonly trace?: ((readonly (string)[])) | undefined; + /** The metadata entry type. */ + readonly type: string; +} +/** Represents the construct node in the scope tree. */ +export class Node { + /** Add an ordering dependency on another construct. + An `IDependable` */ + readonly addDependency: (deps?: ((readonly (IDependable)[])) | undefined) => void; + /** Adds a metadata entry to this construct. + Entries are arbitrary values and will also include a stack trace to allow tracing back to + the code location for when the entry was added. It can be used, for example, to include source + mapping in CloudFormation templates to improve diagnostics. */ + readonly addMetadata: (type: string, data?: any, options?: (MetadataOptions) | undefined) => void; + /** Adds a validation to this construct. + When `node.validate()` is called, the `validate()` method will be called on + all validations and all errors will be returned. */ + readonly addValidation: (validation: IValidation) => void; + /** Returns an opaque tree-unique address for this construct. + Addresses are 42 characters hexadecimal strings. They begin with "c8" + followed by 40 lowercase hexadecimal characters (0-9a-f). + + Addresses are calculated using a SHA-1 of the components of the construct + path. + + To enable refactorings of construct trees, constructs with the ID `Default` + will be excluded from the calculation. In those cases constructs in the + same tree may have the same addreess. + c83a2846e506bcc5f10682b564084bca2d275709ee */ + readonly addr: string; + /** All direct children of this construct. */ + readonly children: (readonly (IConstruct)[]); + /** Returns the child construct that has the id `Default` or `Resource"`. + This is usually the construct that provides the bulk of the underlying functionality. + Useful for modifications of the underlying construct that are not available at the higher levels. + Override the defaultChild property. + + This should only be used in the cases where the correct + default child is not named 'Resource' or 'Default' as it + should be. + + If you set this to undefined, the default behavior of finding + the child named 'Resource' or 'Default' will be used. + @returns a construct or undefined if there is no default child */ + defaultChild?: (IConstruct) | undefined; + /** Return all dependencies registered on this node (non-recursive). */ + readonly dependencies: (readonly (IConstruct)[]); + /** Return this construct and all of its children in the given order. */ + readonly findAll: (order?: (ConstructOrder) | undefined) => (readonly (IConstruct)[]); + /** Return a direct child by id. + Throws an error if the child is not found. + @returns Child with the given id. */ + readonly findChild: (id: string) => IConstruct; + /** Retrieves the all context of a node from tree context. + Context is usually initialized at the root, but can be overridden at any point in the tree. + @returns The context object or an empty object if there is discovered context */ + readonly getAllContext: (defaults?: (Readonly) | undefined) => any; + /** Retrieves a value from tree context if present. Otherwise, would throw an error. + Context is usually initialized at the root, but can be overridden at any point in the tree. + @returns The context value or throws error if there is no context value for this key */ + readonly getContext: (key: string) => any; + /** The id of this construct within the current scope. + This is a scope-unique id. To obtain an app-unique id for this construct, use `addr`. */ + readonly id: string; + /** Locks this construct from allowing more children to be added. + After this + call, no more children can be added to this construct or to any children. */ + readonly lock: () => void; + /** Returns true if this construct or the scopes in which it is defined are locked. */ + readonly locked: boolean; + /** An immutable array of metadata objects associated with this construct. + This can be used, for example, to implement support for deprecation notices, source mapping, etc. */ + readonly metadata: (readonly (MetadataEntry)[]); + /** The full, absolute path of this construct in the tree. + Components are separated by '/'. */ + readonly path: string; + /** Returns the root of the construct tree. + @returns The root of the construct tree. */ + readonly root: IConstruct; + /** Returns the scope in which this construct is defined. + The value is `undefined` at the root of the construct scope tree. */ + readonly scope?: (IConstruct) | undefined; + /** All parent scopes of this construct. + @returns a list of parent scopes. The last element in the list will always + be the current construct and the first element will be the root of the + tree. */ + readonly scopes: (readonly (IConstruct)[]); + /** This can be used to set contextual values. + Context must be set before any children are added, since children may consult context info during construction. + If the key already exists, it will be overridden. */ + readonly setContext: (key: string, value?: any) => void; + /** Return a direct child by id, or undefined. + @returns the child if found, or undefined */ + readonly tryFindChild: (id: string) => (IConstruct) | undefined; + /** Retrieves a value from tree context. + Context is usually initialized at the root, but can be overridden at any point in the tree. + @returns The context value or `undefined` if there is no context value for this key. */ + readonly tryGetContext: (key: string) => any; + /** Remove the child with the given name, if present. + @returns Whether a child with the given name was deleted. */ + readonly tryRemoveChild: (childName: string) => boolean; + /** Validates this construct. + Invokes the `validate()` method on all validations added through + `addValidation()`. + @returns an array of validation error messages associated with this + construct. */ + readonly validate: () => (readonly (string)[]); +} +/** Represents a construct. */ +export interface IConstruct extends IDependable { + /** The tree node. */ + readonly node: Node; +} +/** Represents the building block of the construct graph. +All constructs besides the root construct must be created within the scope of +another construct. */ +export class Construct implements IConstruct { + /** The tree node. */ + readonly node: Node; + /** Returns a string representation of this construct. */ + readonly toString: () => string; +} +/** Utility for applying RFC-6902 JSON-Patch to a document. +Use the the `JsonPatch.apply(doc, ...ops)` function to apply a set of +operations to a JSON document and return the result. + +Operations can be created using the factory methods `JsonPatch.add()`, +`JsonPatch.remove()`, etc. +const output = JsonPatch.apply(input, + JsonPatch.replace('/world/hi/there', 'goodbye'), + JsonPatch.add('/world/foo/', 'boom'), + JsonPatch.remove('/hello')); */ +export class JsonPatch { +} +/** OwnerReference contains enough information to let you identify an owning object. +An owning object must be in the same namespace as the dependent, or +be cluster-scoped, so there is no namespace field. */ +export interface OwnerReference { + /** API version of the referent. */ + readonly apiVersion: string; + /** If true, AND if the owner has the "foregroundDeletion" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. + Defaults to false. To set this field, a user needs "delete" + permission of the owner, otherwise 422 (Unprocessable Entity) will be + returned. */ + readonly blockOwnerDeletion?: (boolean) | undefined; + /** If true, this reference points to the managing controller. */ + readonly controller?: (boolean) | undefined; + /** Kind of the referent. + @see https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds */ + readonly kind: string; + /** Name of the referent. + @see http://kubernetes.io/docs/user-guide/identifiers#names */ + readonly name: string; + /** UID of the referent. + @see http://kubernetes.io/docs/user-guide/identifiers#uids */ + readonly uid: string; +} +/** Object metadata. */ +export class ApiObjectMetadataDefinition { + /** Adds an arbitrary key/value to the object metadata. */ + readonly add: (key: string, value?: any) => void; + /** Add an annotation. */ + readonly addAnnotation: (key: string, value: string) => void; + /** Add one or more finalizers. */ + readonly addFinalizers: (finalizers?: ((readonly (string)[])) | undefined) => void; + /** Add a label. */ + readonly addLabel: (key: string, value: string) => void; + /** Add an owner. */ + readonly addOwnerReference: (owner: OwnerReference) => void; + /** + @returns a value of a label or undefined */ + readonly getLabel: (key: string) => (string) | undefined; + /** The name of the API object. + If a name is specified in `metadata.name` this will be the name returned. + Otherwise, a name will be generated by calling + `Chart.of(this).generatedObjectName(this)`, which by default uses the + construct path to generate a DNS-compatible name for the resource. */ + readonly name?: (string) | undefined; + /** The object's namespace. */ + readonly namespace?: (string) | undefined; + /** Synthesizes a k8s ObjectMeta for this metadata set. */ + readonly toJson: () => any; +} +export class ApiObject extends Construct { + /** Create a dependency between this ApiObject and other constructs. + These can be other ApiObjects, Charts, or custom. */ + readonly addDependency: (dependencies?: ((readonly (IConstruct)[])) | undefined) => void; + /** Applies a set of RFC-6902 JSON-Patch operations to the manifest synthesized for this API object. + kubePod.addJsonPatch(JsonPatch.replace('/spec/enableServiceLinks', true)); */ + readonly addJsonPatch: (ops?: ((readonly (JsonPatch)[])) | undefined) => void; + /** The group portion of the API version (e.g. `authorization.k8s.io`). */ + readonly apiGroup: string; + /** The object's API version (e.g. `authorization.k8s.io/v1`). */ + readonly apiVersion: string; + /** The chart in which this object is defined. */ + readonly chart: Chart; + /** The object kind. */ + readonly kind: string; + /** Metadata associated with this API object. */ + readonly metadata: ApiObjectMetadataDefinition; + /** The name of the API object. + If a name is specified in `metadata.name` this will be the name returned. + Otherwise, a name will be generated by calling + `Chart.of(this).generatedObjectName(this)`, which by default uses the + construct path to generate a DNS-compatible name for the resource. */ + readonly name: string; + /** Renders the object to Kubernetes JSON. + To disable sorting of dictionary keys in output object set the + `CDK8S_DISABLE_SORT` environment variable to any non-empty value. */ + readonly toJson: () => any; +} +export class Chart extends Construct { + /** Create a dependency between this Chart and other constructs. + These can be other ApiObjects, Charts, or custom. */ + readonly addDependency: (dependencies?: ((readonly (IConstruct)[])) | undefined) => void; + /** Returns all the included API objects. */ + readonly apiObjects: (readonly (ApiObject)[]); + /** Generates a app-unique name for an object given it's construct node path. + Different resource types may have different constraints on names + (`metadata.name`). The previous version of the name generator was + compatible with DNS_SUBDOMAIN but not with DNS_LABEL. + + For example, `Deployment` names must comply with DNS_SUBDOMAIN while + `Service` names must comply with DNS_LABEL. + + Since there is no formal specification for this, the default name + generation scheme for kubernetes objects in cdk8s was changed to DNS_LABEL, + since it’s the common denominator for all kubernetes resources + (supposedly). + + You can override this method if you wish to customize object names at the + chart level. */ + readonly generateObjectName: (apiObject: ApiObject) => string; + /** Labels applied to all resources in this chart. + This is an immutable copy. */ + readonly labels: Readonly>; + /** The default namespace for all objects in this chart. */ + readonly namespace?: (string) | undefined; + /** Renders this chart to a set of Kubernetes JSON resources. + @returns array of resource manifests */ + readonly toJson: () => (readonly (any)[]); +} \ No newline at end of file diff --git a/containers/package-lock.json b/containers/package-lock.json index ea40dc32..4fbd300b 100644 --- a/containers/package-lock.json +++ b/containers/package-lock.json @@ -1,12 +1,12 @@ { "name": "@winglibs/containers", - "version": "0.0.23", + "version": "0.0.25", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@winglibs/containers", - "version": "0.0.23", + "version": "0.0.25", "license": "MIT", "dependencies": { "@cdktf/provider-aws": "^18.0.5", @@ -4252,14 +4252,10 @@ } }, "node_modules/@winglang/sdk/node_modules/constructs": { - "version": "10.2.70", "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.2.70.tgz", "integrity": "sha512-z6zr1E8K/9tzJbCQzY0UGX0/oVKPFKu9C/mzEnghCG6TAJINnvlq0CMKm63XqqeMleadZYm5T3sZGJKcxJS/Pg==", "inBundle": true, - "peer": true, - "engines": { - "node": ">= 16.14.0" - } + "peer": true }, "node_modules/@winglang/sdk/node_modules/content-disposition": { "version": "0.5.4", diff --git a/containers/package.json b/containers/package.json index cc12b9c1..9eba55a6 100644 --- a/containers/package.json +++ b/containers/package.json @@ -20,15 +20,12 @@ "license": "MIT", "peerDependencies": { "cdktf": "*", - "winglang": "*" - }, - "dependencies": { + "winglang": "*", "@cdktf/provider-aws": "^18.0.5", "@cdktf/provider-docker": "^10.0.0", "@cdktf/provider-helm": "^9.0.0", "@cdktf/provider-kubernetes": "^10.0.0", "@cdktf/provider-null": "^9.0.0", - "cdk8s-plus-27": "^2.7.33", - "glob": "^10.3.10" + "cdk8s-plus-27": "^2.7.33" } } diff --git a/containers/utils.extern.d.ts b/containers/utils.extern.d.ts new file mode 100644 index 00000000..5c1a51c5 --- /dev/null +++ b/containers/utils.extern.d.ts @@ -0,0 +1,3 @@ +export default interface extern { + dirname: () => string, +} diff --git a/containers/utils.js b/containers/utils.js index 0481b993..6df39c34 100644 --- a/containers/utils.js +++ b/containers/utils.js @@ -1,22 +1,3 @@ -const child_process = require("child_process"); -const fs = require('fs'); -const crypto = require('crypto'); -const glob = require('glob'); -const path = require('path'); - -exports.shell = async function (command, args, cwd) { - return new Promise((resolve, reject) => { - child_process.execFile(command, args, { cwd }, (error, stdout, stderr) => { - if (error) { - console.error(stderr); - return reject(error); - } - - return resolve(stdout ? stdout : stderr); - }); - }); -}; - exports.entrypointDir = function (scope) { if (typeof(scope.entrypointDir) == "string") { return scope.entrypointDir; @@ -29,12 +10,3 @@ exports.dirname = function() { return __dirname; }; -exports.contentHash = function(patterns, cwd) { - const hash = crypto.createHash('md5'); - const files = glob.sync(patterns, { nodir: true, cwd }); - for (const f of files) { - const data = fs.readFileSync(path.join(cwd, f)); - hash.update(data); - } - return hash.digest('hex'); -}; \ No newline at end of file diff --git a/containers/utils.w b/containers/utils.w index 1357b3b7..13e9aeb9 100644 --- a/containers/utils.w +++ b/containers/utils.w @@ -1,30 +1,6 @@ -bring "./api.w" as api; -bring fs; - pub class Util { - extern "./utils.js" pub static inflight shell(command: str, args: Array, cwd: str?): str; - extern "./utils.js" pub static contentHash(files: Array, cwd: str): str; - extern "./utils.js" pub static dirname(): str; - + pub static extern "./utils.js" dirname(): str; pub static entrypointDir(scope: std.IResource): str { return std.Node.of(scope).app.entrypointDir; } - - pub static isPath(s: str): bool { - return s.startsWith("/") || s.startsWith("./"); - } - - pub static inflight isPathInflight(s: str): bool { - return s.startsWith("/") || s.startsWith("./"); - } - - pub static resolveContentHash(scope: std.IResource, props: api.WorkloadProps): str? { - if !Util.isPath(props.image) { - return nil; - } - - let sources = props.sources ?? ["**/*"]; - let imageDir = props.image; - return props.sourceHash ?? Util.contentHash(sources, imageDir); - } } \ No newline at end of file diff --git a/containers/workload.sim.w b/containers/workload.sim.w index d8aba665..04c64721 100644 --- a/containers/workload.sim.w +++ b/containers/workload.sim.w @@ -2,151 +2,49 @@ bring http; bring util; bring cloud; bring sim; +bring ui; bring "./api.w" as api; bring "./utils.w" as utils; pub class Workload_sim { - publicUrlKey: str?; - internalUrlKey: str?; - pub publicUrl: str?; pub internalUrl: str?; - props: api.WorkloadProps; - appDir: str; - imageTag: str; - public: bool; - state: sim.State; - new(props: api.WorkloadProps) { - this.appDir = utils.entrypointDir(this); - this.props = props; - this.state = new sim.State(); - let containerName = util.uuidv4(); - - let hash = utils.resolveContentHash(this, props); - if let hash = hash { - this.imageTag = "{props.name}:{hash}"; - } else { - this.imageTag = props.image; - } - - this.public = props.public ?? false; - - if this.public { - if !props.port? { - throw "'port' is required if 'public' is enabled"; - } - - let key = "public_url"; - this.publicUrl = this.state.token(key); - this.publicUrlKey = key; - } - - if props.port? { - let key = "internal_url"; - this.internalUrl = this.state.token(key); - this.internalUrlKey = key; - } - - let containerService = new cloud.Service(inflight () => { - log("starting workload..."); - - let opts = this.props; - - // if this a reference to a local directory, build the image from a docker file - if utils.isPathInflight(opts.image) { - // check if the image is already built - try { - utils.shell("docker", ["inspect", this.imageTag]); - log("image {this.imageTag} already exists"); - } catch { - log("building locally from {opts.image} and tagging {this.imageTag}..."); - utils.shell("docker", ["build", "-t", this.imageTag, opts.image], this.appDir); - } - } else { - try { - utils.shell("docker", ["inspect", this.imageTag]); - log("image {this.imageTag} already exists"); - } catch { - log("pulling {this.imageTag}"); - utils.shell("docker", ["pull", this.imageTag]); - } - } - - // start the new container - let dockerRun = MutArray[]; - dockerRun.push("run"); - dockerRun.push("--detach"); - dockerRun.push("--rm"); - - dockerRun.push("--name", containerName); - - if let port = opts.port { - dockerRun.push("-p"); - dockerRun.push("{port}"); - } - - if let env = opts.env { - if env.size() > 0 { - dockerRun.push("-e"); - for k in env.keys() { - dockerRun.push("{k}={env.get(k)!}"); - } - } - } - - dockerRun.push(this.imageTag); - - if let runArgs = this.props.args { - for a in runArgs { - dockerRun.push(a); - } - } - - log("starting container from image {this.imageTag}"); - log("docker {dockerRun.join(" ")}"); - utils.shell("docker", dockerRun.copy()); - - log("containerName={containerName}"); - - return () => { - utils.shell("docker", ["rm", "-f", containerName]); - }; - }) as "ContainerService"; - std.Node.of(containerService).hidden = true; - - let readinessService = new cloud.Service(inflight () => { - let opts = this.props; - let var out: Json? = nil; - util.waitUntil(inflight () => { - try { - out = Json.parse(utils.shell("docker", ["inspect", containerName])); - return true; - } catch { - log("something went wrong"); - return false; - } - }, interval: 0.1s); - - if let port = opts.port { - let hostPort = out?.tryGetAt(0)?.tryGet("NetworkSettings")?.tryGet("Ports")?.tryGet("{port}/tcp")?.tryGetAt(0)?.tryGet("HostPort")?.tryAsStr(); - if !hostPort? { - throw "Container does not listen to port {port}"; - } - - let publicUrl = "http://localhost:{hostPort!}"; - - if let k = this.publicUrlKey { - this.state.set(k, publicUrl); - } - - if let k = this.internalUrlKey { - this.state.set(k, "http://host.docker.internal:{hostPort!}"); - } - - if let readiness = opts.readiness { - let readinessUrl = "{publicUrl}{readiness}"; + let state = new sim.State(); + nodeof(state).hidden = true; + + let publicUrlKey = "public_url"; + let internalUrlKey = "internal_url"; + + let c = new sim.Container( + name: props.name, + image: props.image, + args: props.args, + containerPort: props.port, + env: this.toEnv(props.env), + sourceHash: props.sourceHash, + sourcePattern: props.sources, + ); + + nodeof(c).hidden = true; + + if props.port != nil { + this.publicUrl = state.token(publicUrlKey); + this.internalUrl = state.token(internalUrlKey); + + new ui.Field("url", inflight () => { + return this.publicUrl!; + }); + + let s1 = new cloud.Service(inflight () => { + state.set(publicUrlKey, "http://localhost:{c.hostPort!}"); + state.set(internalUrlKey, "http://host.docker.internal:{c.hostPort!}"); + }) as "urls"; + + let s2 = new cloud.Service(inflight () => { + if let readiness = props.readiness { + let readinessUrl = "{this.publicUrl!}{readiness}"; log("waiting for container to be ready: {readinessUrl}..."); util.waitUntil(inflight () => { try { @@ -156,10 +54,22 @@ pub class Workload_sim { } }, interval: 0.1s); } + }) as "readiness"; + + nodeof(s1).hidden = true; + nodeof(s2).hidden = true; + } + } + + toEnv(input: Map?): Map { + let env = MutMap{}; + let i = input ?? {}; + for e in i.entries() { + if e.value != nil { + env.set(e.key, e.value!); } - }) as "ReadinessService"; - std.Node.of(readinessService).hidden = true; + } - std.Node.of(this.state).hidden = true; + return env.copy(); } } diff --git a/containers/workload.tfaws.w b/containers/workload.tfaws.w index f349d093..63580314 100644 --- a/containers/workload.tfaws.w +++ b/containers/workload.tfaws.w @@ -7,6 +7,7 @@ bring "./tfaws-ecr.w" as ecr; bring "./utils.w" as utils; bring "@cdktf/provider-kubernetes" as k8s; bring "@cdktf/provider-helm" as helm; +bring fs; pub class Workload_tfaws { pub internalUrl: str?; @@ -18,8 +19,8 @@ pub class Workload_tfaws { let var image = props.image; let var deps = MutArray[]; - if utils.isPath(props.image) { - let hash = utils.resolveContentHash(this, props) ?? props.image; + if this.isPath(props.image) { + let hash = this.resolveContentHash(props) ?? props.image; let appDir = utils.entrypointDir(this); let repository = new ecr.Repository( name: props.name, @@ -148,5 +149,17 @@ pub class Workload_tfaws { this.publicUrl = "http://{hostname}"; } } + + isPath(s: str): bool { + return s.startsWith("/") || s.startsWith("./"); + } + + resolveContentHash(props: api.WorkloadProps): str? { + if !this.isPath(props.image) { + return nil; + } + + return props.sourceHash ?? fs.md5(props.image, props.sources); + } }