From 457ee3ae6b82104f0c1493df02cd84c6a6d2ee67 Mon Sep 17 00:00:00 2001 From: eladcon Date: Thu, 13 Jun 2024 16:02:29 +0300 Subject: [PATCH 1/5] chore(python): use `sim.Container` (#261) Use the `sim.Container` to show containers logs. Also use docker image instead of local python to build the python dependencies. --- python/builder/Dockerfile | 5 + python/package-lock.json | 8 + python/package.json | 2 +- python/sim/containers.w | 299 ---------------------------- python/sim/function.js | 32 ++- python/sim/inflight.w | 182 +++++++++-------- python/test-assets/main.py | 4 +- python/test-assets/requirements.txt | 2 +- python/tfaws/function.js | 15 +- python/util.extern.d.ts | 3 +- python/util.js | 60 +++++- python/util.w | 3 +- python/wplatform.js | 6 +- 13 files changed, 200 insertions(+), 421 deletions(-) create mode 100644 python/builder/Dockerfile delete mode 100644 python/sim/containers.w diff --git a/python/builder/Dockerfile b/python/builder/Dockerfile new file mode 100644 index 00000000..37c75d93 --- /dev/null +++ b/python/builder/Dockerfile @@ -0,0 +1,5 @@ +FROM public.ecr.aws/lambda/python:3.12 + +COPY requirements.txt /app/requirements.txt + +RUN pip install -r /app/requirements.txt diff --git a/python/package-lock.json b/python/package-lock.json index eed2052a..22d15e2d 100644 --- a/python/package-lock.json +++ b/python/package-lock.json @@ -1,12 +1,20 @@ { "name": "@winglibs/python", +<<<<<<< py-use-sim-container + "version": "0.0.8", +======= "version": "0.0.7", +>>>>>>> main "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@winglibs/python", +<<<<<<< py-use-sim-container + "version": "0.0.8", +======= "version": "0.0.7", +>>>>>>> main "license": "MIT", "peerDependencies": { "@aws-sdk/client-lambda": "^3.549.0", diff --git a/python/package.json b/python/package.json index ff482ee5..89f40842 100644 --- a/python/package.json +++ b/python/package.json @@ -1,7 +1,7 @@ { "name": "@winglibs/python", "description": "python library for Wing", - "version": "0.0.7", + "version": "0.0.8", "repository": { "type": "git", "url": "https://github.com/winglang/winglibs.git", diff --git a/python/sim/containers.w b/python/sim/containers.w deleted file mode 100644 index 9e5383d5..00000000 --- a/python/sim/containers.w +++ /dev/null @@ -1,299 +0,0 @@ -bring http; -bring util; -bring cloud; -bring sim; -bring fs; - -pub class Util { - pub static entrypointDir(scope: std.IResource): str { - return std.Node.of(scope).app.entrypointDir; - } - - pub static isPath(s: str): bool { - return s.startsWith("/") || s.startsWith("./"); - } - - pub static inflight isPathInflight(s: str): bool { - return s.startsWith("/") || s.startsWith("./"); - } - - pub static resolveContentHash(scope: std.IResource, props: ContainerOpts): str? { - if !Util.isPath(props.image) { - return nil; - } - - if let hash = props.sourceHash { - return hash; - } - - let var hash = ""; - let imageDir = props.image; - let sources = props.sources ?? ["**/*"]; - for source in sources { - hash += fs.md5(imageDir, source); - } - - return util.sha256(hash); - } -} - -pub struct ContainerOpts { - name: str; - image: str; - - /** Internal container port to expose */ - flags: Map?; // flags to pass to the docker run command - port: num?; - exposedPort: num?; // internal port to expose - env: Map?; - readiness: str?; // http get - replicas: num?; // number of replicas - public: bool?; // whether the container should have a public url (default: false) - args: Array?; // container arguments - volumes: Map?; // volumes to mount - network: str?; // network to connect to - entrypoint: str?; // entrypoint to run - - /** - * A list of globs of local files to consider as input sources for the container. - * By default, the entire build context directory will be included. - */ - sources: Array?; - - /** - * a hash that represents the container source. if not set, - * and `sources` is set, the hash will be calculated based on the content of the - * source files. - */ - sourceHash: str?; -} - -pub class Container { - publicUrlKey: str?; - internalUrlKey: str?; - - pub publicUrl: str?; - pub internalUrl: str?; - - props: ContainerOpts; - appDir: str; - imageTag: str; - public: bool; - state: sim.State; - - runtimeEnv: cloud.Bucket; - containerService: cloud.Service; - readinessService: cloud.Service; - - new(props: ContainerOpts) { - this.appDir = Util.entrypointDir(this); - this.props = props; - this.state = new sim.State(); - this.runtimeEnv = new cloud.Bucket(); - let containerName = util.uuidv4(); - - let hash = Util.resolveContentHash(this, props); - if let hash = hash { - this.imageTag = "{props.name}:{hash}"; - } else { - this.imageTag = props.image; - } - - this.public = props.public ?? false; - - if this.public { - if !props.port? { - throw "'port' is required if 'public' is enabled"; - } - - let key = "public_url"; - this.publicUrl = this.state.token(key); - this.publicUrlKey = key; - } - - if props.port? { - let key = "internal_url"; - this.internalUrl = this.state.token(key); - this.internalUrlKey = key; - } - - let pathEnv = util.tryEnv("PATH") ?? ""; - - this.containerService = new cloud.Service(inflight () => { - log("starting container..."); - - let opts = this.props; - - // if this a reference to a local directory, build the image from a docker file - if Util.isPathInflight(opts.image) { - // check if the image is already built - try { - util.exec("docker", ["inspect", this.imageTag], { env: { PATH: pathEnv } }); - log("image {this.imageTag} already exists"); - } catch { - log("building locally from {opts.image} and tagging {this.imageTag}..."); - util.exec("docker", ["build", "-t", this.imageTag, opts.image], { env: { PATH: pathEnv }, cwd: this.appDir }); - } - } else { - try { - util.exec("docker", ["inspect", this.imageTag], { env: { PATH: pathEnv } }); - log("image {this.imageTag} already exists"); - } catch { - log("pulling {this.imageTag}"); - try { - util.exec("docker", ["pull", this.imageTag], { env: { PATH: pathEnv } }); - } catch e { - log("failed to pull image {this.imageTag} {e}"); - throw e; - } - log("image pulled"); - } - } - - // start the new container - let dockerRun = MutArray[]; - dockerRun.push("run"); - dockerRun.push("--detach"); - dockerRun.push("--rm"); - - dockerRun.push("--name", containerName); - - if let flags = opts.flags { - if flags.size() > 0 { - for k in flags.keys() { - dockerRun.push("{k}={flags.get(k)}"); - } - } - } - - if let network = opts.network { - dockerRun.push("--network={network}"); - } - - if let port = opts.port { - dockerRun.push("-p"); - if let exposedPort = opts.exposedPort { - dockerRun.push("{exposedPort}:{port}"); - } else { - dockerRun.push("{port}"); - } - } - - if let env = opts.env { - if env.size() > 0 { - for k in env.keys() { - dockerRun.push("-e"); - dockerRun.push("{k}={env.get(k)!}"); - } - } - } - - for key in this.runtimeEnv.list() { - dockerRun.push("-e"); - dockerRun.push("{key}={this.runtimeEnv.get(key)}"); - } - - if let volumes = opts.volumes { - if volumes.size() > 0 { - dockerRun.push("-v"); - for volume in volumes.entries() { - dockerRun.push("{volume.value}:{volume.key}"); - } - } - } - - if let entrypoint = opts.entrypoint { - dockerRun.push("--entrypoint"); - dockerRun.push(entrypoint); - } - - dockerRun.push(this.imageTag); - - if let runArgs = this.props.args { - for a in runArgs { - dockerRun.push(a); - } - } - - log("starting container from image {this.imageTag}"); - log("docker {dockerRun.join(" ")}"); - util.exec("docker", dockerRun.copy(), { env: { PATH: pathEnv } }); - - log("containerName={containerName}"); - - return () => { - util.exec("docker", ["rm", "-f", containerName], { env: { PATH: pathEnv } }); - }; - }, {autoStart: false}) as "ContainerService"; - std.Node.of(this.containerService).hidden = true; - - this.readinessService = new cloud.Service(inflight () => { - let opts = this.props; - let var out: Json? = nil; - util.waitUntil(inflight () => { - try { - out = Json.parse(util.exec("docker", ["inspect", containerName], { env: { PATH: pathEnv } }).stdout); - return true; - } catch { - log("something went wrong"); - return false; - } - }, interval: 3s); - - if let network = opts.network { - if network == "host" { - if let k = this.publicUrlKey { - this.state.set(k, "http://localhost:{opts.port!}"); - } - - if let k = this.internalUrlKey { - this.state.set(k, "http://localhost:{opts.port!}"); - } - - return () => {}; - } - } - - if let port = opts.port { - let hostPort = out?.tryGetAt(0)?.tryGet("NetworkSettings")?.tryGet("Ports")?.tryGet("{port}/tcp")?.tryGetAt(0)?.tryGet("HostPort")?.tryAsStr(); - if !hostPort? { - throw "Container does not listen to port {port}"; - } - - let publicUrl = "http://localhost:{hostPort!}"; - - if let k = this.publicUrlKey { - this.state.set(k, publicUrl); - } - - if let k = this.internalUrlKey { - this.state.set(k, "http://host.docker.internal:{hostPort!}"); - } - - if let readiness = opts.readiness { - let readinessUrl = "{publicUrl}{readiness}"; - log("waiting for container to be ready: {readinessUrl}..."); - util.waitUntil(inflight () => { - try { - return http.get(readinessUrl).ok; - } catch { - return false; - } - }, interval: 0.1s); - } - } - }, {autoStart: false}) as "ReadinessService"; - std.Node.of(this.readinessService).hidden = true; - - std.Node.of(this.state).hidden = true; - } - - pub inflight start(env: Map) { - for entry in env.entries() { - this.runtimeEnv.put(entry.key, entry.value); - } - - this.containerService.start(); - this.readinessService.start(); - } -} diff --git a/python/sim/function.js b/python/sim/function.js index 8c696e23..10b42352 100644 --- a/python/sim/function.js +++ b/python/sim/function.js @@ -1,16 +1,30 @@ const { App } = require("@winglang/sdk/lib/core"); +const { Function: SimFunction } = require("@winglang/sdk/lib/target-sim/function.js"); -module.exports.handleSimInflight = (inflight, props) => { - for (let e in props.env) { - inflight.inner.service.addEnvironment(e, props.env[e]); - } +module.exports.Function = class Function extends SimFunction { + constructor( + scope, + id, + inflight, + props = {}, + pythonInflight, + ) { + super(scope, id, inflight, props); + + this.pythonInflight = pythonInflight; - if (!App.of(inflight).isTestEnvironment) { - for (let key of ["AWS_REGION", "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]) { - let value = process.env[key]; - if (value) { - inflight.inner.service.addEnvironment(key, value); + if (!App.of(this).isTestEnvironment) { + for (let key of ["AWS_REGION", "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]) { + let value = process.env[key]; + if (value) { + this.addEnvironment(key, value); + } } } } + + _preSynthesize() { + this.pythonInflight.inner.preLift(this); + return super._preSynthesize(); + } } diff --git a/python/sim/inflight.w b/python/sim/inflight.w index 03812d8f..0a398c74 100644 --- a/python/sim/inflight.w +++ b/python/sim/inflight.w @@ -2,20 +2,21 @@ bring cloud; bring util; bring http; bring math; -bring "./containers.w" as containers; +bring sim; bring "../types.w" as types; bring "../util.w" as libutil; pub class Inflight impl cloud.IFunctionHandler { pub url: str; - service: cloud.Service; + network: str?; + lifts: MutMap; clients: MutMap; new(props: types.InflightProps) { let homeEnv = util.tryEnv("HOME") ?? ""; let pathEnv = util.tryEnv("PATH") ?? ""; - let outdir = libutil.build( + let outdir = libutil.buildSim( nodePath: nodeof(this).path, path: props.path, handler: props.handler, @@ -34,107 +35,104 @@ pub class Inflight impl cloud.IFunctionHandler { "127.0.0.1:{runtimePort}" ]; let flags = MutMap{}; - let var network: str? = nil; let platform = Inflight.os(); if platform != "darwin" && platform != "win32" { - network = "host"; + this.network = "host"; } - let runner = new containers.Container( - image: "public.ecr.aws/lambda/python:3.12", + let runner = new sim.Container( + image: outdir, name: "python-runner", - volumes: { - "/var/task:ro,delegated": outdir, - }, + volumes: [ + "{props.path}:/var/task:ro", + ], env: { DOCKER_LAMBDA_STAY_OPEN: "1", + WING_TARGET: util.env("WING_TARGET"), }, - public: true, - port: port, - exposedPort: port, + containerPort: port, args: args, - flags: flags.copy(), - network: network, + network: this.network, entrypoint: "/usr/local/bin/aws-lambda-rie", ); - - this.service = new cloud.Service(inflight () => { - let clients = MutMap{}; - for client in this.clients.entries() { - let value = client.value; - - // TODO: move it to a function (there's a weird bug going on here) - let collect = inflight (value: types.LiftedSim) => { - let c = MutMap{}; - if let children = value.children { - for child in children.entries() { - c.set(child.key, collect(child.value)); - } - } - if let handle = util.tryEnv(value.handle) { - // sdk resources - return { - id: value.id, - type: value.type, - path: value.path, - target: value.target, - props: value.props, - children: c.copy(), - handle: handle, - }; - } else { - // custom resources - return { - id: value.id, - type: value.type, - path: value.path, - target: value.target, - props: value.props, - children: c.copy(), - handle: value.handle, - }; - } - }; - clients.set(client.key, collect(value)); - } + this.url = "http://localhost:{runner.hostPort!}/2015-03-31/functions/function/invocations"; + this.clients = MutMap{}; - let env = MutMap{ - "WING_CLIENTS" => Json.stringify(clients), - }; + if let lift = props.lift { + this.lifts = lift.copyMut(); + } else { + this.lifts = MutMap{}; + } + } - let var host = "http://host.docker.internal"; - if let network = network { - if network == "host" { - host = "http://127.0.0.1"; + inflight context(): Map { + let clients = MutMap{}; + for client in this.clients.entries() { + let value = client.value; + + // TODO: move it to a function (there's a weird bug going on here) + let collect = inflight (value: types.LiftedSim) => { + let c = MutMap{}; + if let children = value.children { + for child in children.entries() { + c.set(child.key, collect(child.value)); + } } - } - - for e in Inflight.env().entries() { - env.set(e.key, e.value); - } - for e in env.entries() { - let var value = e.value; - if value.contains("http://localhost") { - value = value.replaceAll("http://localhost", host); - } elif value.contains("http://127.0.0.1") { - value = value.replaceAll("http://127.0.0.1", host); + if let handle = util.tryEnv(value.handle) { + // sdk resources + return { + id: value.id, + type: value.type, + path: value.path, + target: value.target, + props: value.props, + children: c.copy(), + handle: handle, + }; + } else { + // custom resources + return { + id: value.id, + type: value.type, + path: value.path, + target: value.target, + props: value.props, + children: c.copy(), + handle: value.handle, + }; } - env.set(e.key, value); - } + }; + clients.set(client.key, collect(value)); + } - runner.start(env.copy()); - }); + let env = MutMap{ + "WING_CLIENTS" => Json.stringify(clients), + }; - this.url = "{runner.publicUrl!}/2015-03-31/functions/function/invocations"; - this.clients = MutMap{}; + let var host = "http://host.docker.internal"; + if let network = this.network { + if network == "host" { + host = "http://127.0.0.1"; + } + } - if let lifts = props.lift { - for lift in lifts.entries() { - this.lift(lift.value.obj, { id: lift.key, allow: lift.value.allow }); + for e in Inflight.env().entries() { + env.set(e.key, e.value); + } + + for e in env.entries() { + let var value = e.value; + if value.contains("http://localhost") { + value = value.replaceAll("http://localhost", host); + } elif value.contains("http://127.0.0.1") { + value = value.replaceAll("http://127.0.0.1", host); } + env.set(e.key, value); } + + return env.copy(); } pub inflight handle(event: str?): str? { @@ -142,14 +140,15 @@ pub class Inflight impl cloud.IFunctionHandler { } protected inflight _handle(event: str?): str? { + let context = Json.stringify(this.context()); let var body = event; if event == nil || event == "" { - body = Json.stringify({ payload: "" }); + body = Json.stringify({ payload: "", context: context }); } else { if let json = Json.tryParse(event) { - body = Json.stringify({ payload: json }); + body = Json.stringify({ payload: json, context: context }); } else { - body = Json.stringify({ payload: event }); + body = Json.stringify({ payload: event, context: context }); } } @@ -158,10 +157,21 @@ pub class Inflight impl cloud.IFunctionHandler { } pub lift(obj: std.Resource, options: types.LiftOptions): cloud.IFunctionHandler { - libutil.liftSim(obj, options, this.service, this.clients); + this.lifts.set(options.id, { obj: obj, allow: options.allow }); return this; } + pub preLift(host: std.IInflightHost) { + for lift in this.lifts.entries() { + libutil.liftSim( + lift.value.obj, + { id: lift.key, allow: lift.value.allow }, + host, + this.clients + ); + } + } + extern "./util.js" inflight static env(): Map; extern "./util.js" static os(): str; } diff --git a/python/test-assets/main.py b/python/test-assets/main.py index 19034767..f6a24976 100644 --- a/python/test-assets/main.py +++ b/python/test-assets/main.py @@ -5,8 +5,8 @@ def handler(event, context): print(event) print(context) - foo_env = os.getenv("FOO") payload = from_function_event(event) + foo_env = os.getenv("FOO") email_client = lifted("email") email_client.send_email(Source="bot@wing.cloud", Destination={'ToAddresses': ['bot@monada.co',],},Message={'Subject': {'Data': 'Winglang Test Email!',},'Body': {'Text': {'Data': 'Hello from Python!',},}},) @@ -77,9 +77,9 @@ def api_handler(event, context): print(event) print(context) + req = from_api_event(event) foo = os.getenv("FOO") - req = from_api_event(event) client_put = lifted("bucket") client_put.put(req["path"], json.dumps(req)) diff --git a/python/test-assets/requirements.txt b/python/test-assets/requirements.txt index 6ebdcbc2..bca16bb7 100644 --- a/python/test-assets/requirements.txt +++ b/python/test-assets/requirements.txt @@ -1,4 +1,4 @@ -wingsdk == 0.0.2 +wingsdk == 0.0.3 requests boto3 Faker diff --git a/python/tfaws/function.js b/python/tfaws/function.js index 091a36f8..0499634b 100644 --- a/python/tfaws/function.js +++ b/python/tfaws/function.js @@ -11,7 +11,7 @@ const { ResourceNames } = require("@winglang/sdk/lib/shared/resource-names"); const { DEFAULT_MEMORY_SIZE } = require("@winglang/sdk/lib/shared/function"); const cdktf = require("cdktf"); const awsProvider = require("@cdktf/provider-aws"); -const { build } = require("../util.js"); +const { buildAws } = require("../util.js"); const FUNCTION_NAME_OPTS = { maxLen: 64, @@ -24,6 +24,7 @@ module.exports.Function = class Function extends Construct { id, inflight, props = {}, + pythonInflight, ) { super(scope, id); @@ -31,10 +32,10 @@ module.exports.Function = class Function extends Construct { const pathEnv = process.env["PATH"] || ""; const homeEnv = process.env["HOME"] || ""; - const outdir = build({ + const outdir = buildAws({ nodePath: Node.of(handler).path, - path: inflight.inner.props.path, - handler: inflight.inner.props.handler, + path: pythonInflight.inner.props.path, + handler: pythonInflight.inner.props.handler, homeEnv: homeEnv, pathEnv: pathEnv, }); @@ -57,8 +58,8 @@ module.exports.Function = class Function extends Construct { const roleName = this.dummy.role.name; const clients = {}; - for (let clientId of Object.keys(inflight.inner.lifts)) { - const { client, options } = inflight.inner.lifts[clientId]; + for (let clientId of Object.keys(pythonInflight.inner.lifts)) { + const { client, options } = pythonInflight.inner.lifts[clientId]; const allow = options.allow; // SDK resources @@ -122,7 +123,7 @@ module.exports.Function = class Function extends Construct { this.lambda = new awsProvider.lambdaFunction.LambdaFunction(this, "PyFunction", { functionName: this.name, role: roleArn, - handler: inflight.inner.props.handler, + handler: pythonInflight.inner.props.handler, runtime: "python3.11", s3Bucket: bucket.bucket, s3Key: lambdaArchive.key, diff --git a/python/util.extern.d.ts b/python/util.extern.d.ts index fce3a49d..37d6369b 100644 --- a/python/util.extern.d.ts +++ b/python/util.extern.d.ts @@ -1,5 +1,6 @@ export default interface extern { - build: (options: BuildOptions) => string, + buildAws: (options: BuildOptions) => string, + buildSim: (options: BuildOptions) => string, dirname: () => string, liftSim: (obj: Resource, options: LiftOptions, host: IInflightHost, clients: Record) => void, liftTfAws: (id: string, client: Resource) => string, diff --git a/python/util.js b/python/util.js index 359e62fd..c7f85c92 100644 --- a/python/util.js +++ b/python/util.js @@ -1,20 +1,19 @@ const { join } = require("node:path"); -const { cpSync, existsSync, mkdtempSync, mkdirSync, readFileSync } = require("node:fs"); -const { execSync } = require("node:child_process"); +const { cpSync, existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync } = require("node:fs"); +const { execSync, spawnSync } = require("node:child_process"); const { tmpdir } = require("node:os"); const crypto = require("node:crypto"); const glob = require("glob"); const { App, Lifting } = require("@winglang/sdk/lib/core"); const { Node } = require("@winglang/sdk/lib/std"); +const { Util } = require("@winglang/sdk/lib/util"); -const createMD5ForProject = (nodePath, filePath, path, handler) => { +const createMD5ForProject = (requirementsFile, nodePath = "", path = "", handler = "") => { const hash = crypto.createHash('md5'); hash.update(nodePath); hash.update(path); hash.update(handler); - - const file = readFileSync(filePath, "utf8"); - hash.update(file); + hash.update(requirementsFile); return hash.digest("hex"); }; @@ -26,7 +25,41 @@ exports.resolve = (path1, path2) => { return join(path1, path2); }; -exports.build = (options) => { +exports.buildSim = (options) => { + const { nodePath, path, handler, homeEnv, pathEnv } = options; + + const requirementsPath = join(path, "requirements.txt"); + let requirements = ""; + if (existsSync(requirementsPath)) { + requirements = readFileSync(requirementsPath, "utf8"); + } + const md5 = createMD5ForProject(requirements, nodePath, path, handler); + const imageName = `wing-py:${md5}`; + execSync(`docker build -t ${imageName} -f ${join(__dirname, "./builder/Dockerfile")} ${path}`, + { + cwd: __dirname, + env: { HOME: homeEnv, PATH: pathEnv } + } + ); + + const forceReloadImage = () => { + try { + execSync(`docker inspect ${imageName}`); + execSync(`docker save ${imageName} -o ${join(tmpdir(), imageName)}`); + execSync(`docker load -i ${join(tmpdir(), imageName)}`); + rmSync(join(tmpdir(), imageName)); + return true; + } catch {} + }; + + if (process.env.CI) { + Util.waitUntil(forceReloadImage); + } + + return imageName; +}; + +exports.buildAws = (options) => { const { nodePath, path, handler, homeEnv, pathEnv } = options; const copyFiles = (src, dest) => { @@ -40,13 +73,18 @@ exports.build = (options) => { // if there is a requirements.txt file, install the dependencies const requirementsPath = join(path, "requirements.txt"); if (existsSync(requirementsPath)) { - const md5 = createMD5ForProject(nodePath, requirementsPath, path, handler); + const requirements = readFileSync(requirementsPath, "utf8"); + const md5 = createMD5ForProject(requirements, nodePath, path, handler); const outdir = join(tmpdir(), "py-func-", md5); if (!existsSync(outdir)) { mkdirSync(outdir, { recursive: true }); cpSync(requirementsPath, join(outdir, "requirements.txt")); - execSync(`python -m pip install -r ${join(outdir, "requirements.txt")} -t python`, - { cwd: outdir, env: { HOME: homeEnv, PATH: pathEnv } }); + execSync(`docker run --rm -v ${outdir}:/var/task:rw --entrypoint python python:3.12 -m pip install -r /var/task/requirements.txt -t /var/task/python`, + { + cwd: outdir, + env: { HOME: homeEnv, PATH: pathEnv } + } + ); } copyFiles(path, join(outdir, "python")); return join(outdir, "python"); @@ -61,7 +99,7 @@ exports.liftTfAws = (id, resource) => { }; -exports.liftSim = (resource, options, host, clients, wingClients) => { +exports.liftSim = (resource, options, host, clients) => { let lifted = getLifted(resource, options.id); if (lifted) { diff --git a/python/util.w b/python/util.w index fbbf4948..c03d7d07 100644 --- a/python/util.w +++ b/python/util.w @@ -2,7 +2,8 @@ bring "./types.w" as types; pub class Util { extern "./util.js" pub static dirname(): str; - extern "./util.js" pub static build(options: types.BuildOptions): str; + extern "./util.js" pub static buildSim(options: types.BuildOptions): str; + extern "./util.js" pub static buildAws(options: types.BuildOptions): str; extern "./util.js" pub static liftTfAws(id: str, client: std.Resource): str; extern "./util.js" pub static liftSim( obj: std.Resource, diff --git a/python/wplatform.js b/python/wplatform.js index 1ffecd30..61f19174 100644 --- a/python/wplatform.js +++ b/python/wplatform.js @@ -1,4 +1,4 @@ -const { handleSimInflight } = require("./sim/function.js"); +const { Function: SimFunction } = require("./sim/function.js"); const { Function: TfAwsFunction } = require("./tfaws/function.js"); const { Queue: TfAwsQueue } = require("./tfaws/queue.js"); const { Topic: TfAwsTopic } = require("./tfaws/topic.js"); @@ -29,9 +29,9 @@ const createFunction = (target, scope, id, inflight, props) => { const pythonInflight = tryGetPythonInflight(inflight); if (pythonInflight) { if (target === "tf-aws") { - return new TfAwsFunction(scope, id, pythonInflight, props); + return new TfAwsFunction(scope, id, inflight, props); } else if (target === "sim") { - handleSimInflight(pythonInflight, props); + return new SimFunction(scope, id, inflight, props, pythonInflight); } } }; From 5258012c8dd80fd9cae8aac0b3eb6aadfd4a7e60 Mon Sep 17 00:00:00 2001 From: eladcon Date: Thu, 13 Jun 2024 16:12:43 +0300 Subject: [PATCH 2/5] fix(python): dockerfile is not packed (#262) * wip * wip --- python/builder/Dockerfile | 5 ----- python/package-lock.json | 12 ++---------- python/package.json | 2 +- python/util.js | 16 +++++++++++++--- 4 files changed, 16 insertions(+), 19 deletions(-) delete mode 100644 python/builder/Dockerfile diff --git a/python/builder/Dockerfile b/python/builder/Dockerfile deleted file mode 100644 index 37c75d93..00000000 --- a/python/builder/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM public.ecr.aws/lambda/python:3.12 - -COPY requirements.txt /app/requirements.txt - -RUN pip install -r /app/requirements.txt diff --git a/python/package-lock.json b/python/package-lock.json index 22d15e2d..b2539a74 100644 --- a/python/package-lock.json +++ b/python/package-lock.json @@ -1,20 +1,12 @@ { "name": "@winglibs/python", -<<<<<<< py-use-sim-container - "version": "0.0.8", -======= - "version": "0.0.7", ->>>>>>> main + "version": "0.0.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@winglibs/python", -<<<<<<< py-use-sim-container - "version": "0.0.8", -======= - "version": "0.0.7", ->>>>>>> main + "version": "0.0.9", "license": "MIT", "peerDependencies": { "@aws-sdk/client-lambda": "^3.549.0", diff --git a/python/package.json b/python/package.json index 89f40842..bd7f999c 100644 --- a/python/package.json +++ b/python/package.json @@ -1,7 +1,7 @@ { "name": "@winglibs/python", "description": "python library for Wing", - "version": "0.0.8", + "version": "0.0.9", "repository": { "type": "git", "url": "https://github.com/winglang/winglibs.git", diff --git a/python/util.js b/python/util.js index c7f85c92..730c541e 100644 --- a/python/util.js +++ b/python/util.js @@ -1,6 +1,6 @@ const { join } = require("node:path"); -const { cpSync, existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync } = require("node:fs"); -const { execSync, spawnSync } = require("node:child_process"); +const { cpSync, existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } = require("node:fs"); +const { execSync } = require("node:child_process"); const { tmpdir } = require("node:os"); const crypto = require("node:crypto"); const glob = require("glob"); @@ -35,7 +35,17 @@ exports.buildSim = (options) => { } const md5 = createMD5ForProject(requirements, nodePath, path, handler); const imageName = `wing-py:${md5}`; - execSync(`docker build -t ${imageName} -f ${join(__dirname, "./builder/Dockerfile")} ${path}`, + + const dockerfile = join(tmpdir(), `Dockerfile-${md5}`); + if (!existsSync(dockerfile)) { + const dockerfileContent = ` +FROM public.ecr.aws/lambda/python:3.12 +COPY requirements.txt /app/requirements.txt +RUN pip install -r /app/requirements.txt` + writeFileSync(dockerfile, dockerfileContent); + } + + execSync(`docker build -t ${imageName} -f ${dockerfile} ${path}`, { cwd: __dirname, env: { HOME: homeEnv, PATH: pathEnv } From d7581cbb57f210d1753f206c90840baeaedf5326 Mon Sep 17 00:00:00 2001 From: Chris Rybicki Date: Fri, 14 Jun 2024 16:56:16 -0400 Subject: [PATCH 3/5] chore(tf): add missing docs for tf.Provider (#260) --- tf/README.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++++ tf/package.json | 2 +- tf/provider.w | 5 +++ 3 files changed, 88 insertions(+), 1 deletion(-) diff --git a/tf/README.md b/tf/README.md index 4c541527..2ac4bc76 100644 --- a/tf/README.md +++ b/tf/README.md @@ -103,6 +103,88 @@ And the output will be: } ``` +## `tf.Provider` + +Represents an arbitrary Terraform provider. + +> `tf.Provider` can only be used when compiling your Wing program to a `tf-*` target. + +It takes `name`, `source`, `version`, and `attributes` properties: + +```js +bring tf; + +new tf.Provider({ + name: "dnsimple", + source: "dnsimple/dnsimple", + version: "1.6.0", + attributes: { + token: "dnsimple_token", + } +}) as "DnsimpleProvider"; +``` + +Now, we can compile this to Terraform: + +```sh +wing compile -t tf-aws +``` + +And the output will be: + +```json +{ + "provider": { + "aws": [{}], + "dnsimple": [ + { + "token": "dnsimple_token" + } + ] + }, + "terraform": { + "backend": { + "local": { + "path": "./terraform.tfstate" + } + }, + "required_providers": { + "aws": { + "source": "aws", + "version": "5.31.0" + }, + "dnsimple": { + "source": "dnsimple/dnsimple", + "version": "1.6.0" + } + } + } +} +``` + +You can create a singleton provider like so: + +```js +class DnsimpleProvider { + pub static getOrCreate(scope: std.IResource): tf.Provider { + let root = nodeof(scope).root; + let singletonKey = "WingDnsimpleProvider"; + let existing = root.node.tryFindChild(singletonKey); + if existing? { + return unsafeCast(existing); + } + + return new tf.Provider( + name: "dnsimple", + source: "dnsimple/dnsimple", + version: "1.6.0", + ) as singletonKey in root; + } +} +``` + +Use `DnsimpleProvider.getOrCreate(scope)` to get the provider instance. + ## Maintainers * [Elad Ben-Israel](@eladb) diff --git a/tf/package.json b/tf/package.json index 2d2e8028..1346ed6f 100644 --- a/tf/package.json +++ b/tf/package.json @@ -5,7 +5,7 @@ "email": "eladb@wing.cloud", "name": "Elad Ben-Israel" }, - "version": "0.0.2", + "version": "0.0.3", "repository": { "type": "git", "url": "https://github.com/winglang/winglibs.git", diff --git a/tf/provider.w b/tf/provider.w index bd5fcd31..5d0ea6e4 100644 --- a/tf/provider.w +++ b/tf/provider.w @@ -1,4 +1,5 @@ bring "cdktf" as cdktf; +bring util; pub struct ProviderProps { /// The name of the provider in Terraform - this is the prefix used for all resources in the provider. @@ -28,6 +29,10 @@ pub class Provider extends cdktf.TerraformProvider { terraformProviderSource: props.source, ); + if !util.env("WING_TARGET").startsWith("tf") { + throw "tf.Provider can only be used in a Terraform target."; + } + this.attributes = props.attributes ?? {}; } From 15694084f569f58070286f2119dc06c7f4e1935f Mon Sep 17 00:00:00 2001 From: eladcon Date: Sun, 16 Jun 2024 10:42:21 +0300 Subject: [PATCH 4/5] chore(python): restore local containers implementation (#263) * wi * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip --- python/package-lock.json | 4 +- python/package.json | 2 +- python/sim/containers.w | 311 ++++++++++++++++++++++++++++ python/sim/function.js | 11 +- python/sim/inflight.w | 174 ++++++++-------- python/sim/util.extern.d.ts | 1 + python/sim/util.js | 21 ++ python/test-assets/requirements.txt | 2 +- python/util.js | 34 ++- 9 files changed, 449 insertions(+), 111 deletions(-) create mode 100644 python/sim/containers.w diff --git a/python/package-lock.json b/python/package-lock.json index b2539a74..14dc75af 100644 --- a/python/package-lock.json +++ b/python/package-lock.json @@ -1,12 +1,12 @@ { "name": "@winglibs/python", - "version": "0.0.9", + "version": "0.0.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@winglibs/python", - "version": "0.0.9", + "version": "0.0.10", "license": "MIT", "peerDependencies": { "@aws-sdk/client-lambda": "^3.549.0", diff --git a/python/package.json b/python/package.json index bd7f999c..39404ebd 100644 --- a/python/package.json +++ b/python/package.json @@ -1,7 +1,7 @@ { "name": "@winglibs/python", "description": "python library for Wing", - "version": "0.0.9", + "version": "0.0.10", "repository": { "type": "git", "url": "https://github.com/winglang/winglibs.git", diff --git a/python/sim/containers.w b/python/sim/containers.w new file mode 100644 index 00000000..bc17f6f3 --- /dev/null +++ b/python/sim/containers.w @@ -0,0 +1,311 @@ +bring http; +bring util; +bring cloud; +bring sim; +bring fs; + +pub class Util { + pub extern "./util.js" inflight static _spawn(command: str, args: Array, options: Map): void; + + pub static entrypointDir(scope: std.IResource): str { + return std.Node.of(scope).app.entrypointDir; + } + + pub static isPath(s: str): bool { + return s.startsWith("/") || s.startsWith("./"); + } + + pub static inflight isPathInflight(s: str): bool { + return s.startsWith("/") || s.startsWith("./"); + } + + pub static resolveContentHash(scope: std.IResource, props: ContainerOpts): str? { + if !Util.isPath(props.image) { + return nil; + } + + if let hash = props.sourceHash { + return hash; + } + + let var hash = ""; + let imageDir = props.image; + let sources = props.sources ?? ["**/*"]; + for source in sources { + hash += fs.md5(imageDir, source); + } + + return util.sha256(hash); + } +} + +pub struct ContainerOpts { + name: str; + image: str; + + /** Internal container port to expose */ + flags: Map?; // flags to pass to the docker run command + port: num?; + exposedPort: num?; // internal port to expose + env: Map?; + readiness: str?; // http get + replicas: num?; // number of replicas + public: bool?; // whether the container should have a public url (default: false) + args: Array?; // container arguments + volumes: Map?; // volumes to mount + network: str?; // network to connect to + entrypoint: str?; // entrypoint to run + + /** + * A list of globs of local files to consider as input sources for the container. + * By default, the entire build context directory will be included. + */ + sources: Array?; + + /** + * a hash that represents the container source. if not set, + * and `sources` is set, the hash will be calculated based on the content of the + * source files. + */ + sourceHash: str?; +} + +pub class Container { + publicUrlKey: str?; + internalUrlKey: str?; + + pub publicUrl: str?; + pub internalUrl: str?; + + props: ContainerOpts; + appDir: str; + imageTag: str; + public: bool; + state: sim.State; + + runtimeEnv: cloud.Bucket; + containerService: cloud.Service; + readinessService: cloud.Service; + + new(props: ContainerOpts) { + this.appDir = Util.entrypointDir(this); + this.props = props; + this.state = new sim.State(); + this.runtimeEnv = new cloud.Bucket(); + let containerName = util.uuidv4(); + + let hash = Util.resolveContentHash(this, props); + if let hash = hash { + this.imageTag = "{props.name}:{hash}"; + } else { + this.imageTag = props.image; + } + + this.public = props.public ?? false; + + if this.public { + if !props.port? { + throw "'port' is required if 'public' is enabled"; + } + + let key = "public_url"; + this.publicUrl = this.state.token(key); + this.publicUrlKey = key; + } + + if props.port? { + let key = "internal_url"; + this.internalUrl = this.state.token(key); + this.internalUrlKey = key; + } + + let pathEnv = util.tryEnv("PATH") ?? ""; + + this.containerService = new cloud.Service(inflight () => { + log("starting container..."); + + let opts = this.props; + + // if this a reference to a local directory, build the image from a docker file + if Util.isPathInflight(opts.image) { + // check if the image is already built + try { + util.exec("docker", ["inspect", this.imageTag], { env: { PATH: pathEnv } }); + log("image {this.imageTag} already exists"); + } catch { + log("building locally from {opts.image} and tagging {this.imageTag}..."); + util.exec("docker", ["build", "-t", this.imageTag, opts.image], { env: { PATH: pathEnv }, cwd: this.appDir }); + } + } else { + try { + util.exec("docker", ["inspect", this.imageTag], { env: { PATH: pathEnv } }); + log("image {this.imageTag} already exists"); + } catch { + log("pulling {this.imageTag}"); + try { + util.exec("docker", ["pull", this.imageTag], { env: { PATH: pathEnv } }); + } catch e { + log("failed to pull image {this.imageTag} {e}"); + throw e; + } + log("image pulled"); + } + } + + // start the new container + let dockerRun = MutArray[]; + dockerRun.push("run"); + dockerRun.push("-i"); + dockerRun.push("--rm"); + + dockerRun.push("--name", containerName); + + if let flags = opts.flags { + if flags.size() > 0 { + for k in flags.keys() { + dockerRun.push("{k}={flags.get(k)}"); + } + } + } + + if let network = opts.network { + dockerRun.push("--network={network}"); + } + + if let port = opts.port { + dockerRun.push("-p"); + if let exposedPort = opts.exposedPort { + dockerRun.push("{exposedPort}:{port}"); + } else { + dockerRun.push("{port}"); + } + } + + if let env = opts.env { + if env.size() > 0 { + for k in env.keys() { + dockerRun.push("-e"); + dockerRun.push("{k}={env.get(k)!}"); + } + } + } + + for key in this.runtimeEnv.list() { + dockerRun.push("-e"); + dockerRun.push("{key}={this.runtimeEnv.get(key)}"); + } + + if let volumes = opts.volumes { + if volumes.size() > 0 { + dockerRun.push("-v"); + for volume in volumes.entries() { + dockerRun.push("{volume.value}:{volume.key}"); + } + } + } + + if let entrypoint = opts.entrypoint { + dockerRun.push("--entrypoint"); + dockerRun.push(entrypoint); + } + + dockerRun.push(this.imageTag); + + if let runArgs = this.props.args { + for a in runArgs { + dockerRun.push(a); + } + } + + log("starting container from image {this.imageTag}"); + log("docker {dockerRun.join(" ")}"); + Util._spawn("docker", dockerRun.copy(), { env: { PATH: pathEnv } }); + // util.exec("docker", dockerRun.copy(), { env: { PATH: pathEnv } }); + + log("containerName={containerName}"); + + return () => { + util.exec("docker", ["rm", "-f", containerName], { env: { PATH: pathEnv } }); + }; + }, {autoStart: false}) as "ContainerService"; + std.Node.of(this.containerService).hidden = true; + + this.readinessService = new cloud.Service(inflight () => { + let opts = this.props; + let var out: Json? = nil; + util.waitUntil(inflight () => { + try { + out = Json.parse(util.exec("docker", ["inspect", containerName], { env: { PATH: pathEnv } }).stdout); + + if let port = opts.port { + if let network = opts.network { + if network == "host" { + return out?.tryGetAt(0)?.tryGet("Config")?.tryGet("ExposedPorts")?.tryGet("{port}/tcp") != nil; + } + } + return out?.tryGetAt(0)?.tryGet("NetworkSettings")?.tryGet("Ports")?.tryGet("{port}/tcp")?.tryGetAt(0)?.tryGet("HostPort")?.tryAsStr() != nil; + } + return true; + } catch { + log("something went wrong"); + return false; + } + }, interval: 3s); + + if let network = opts.network { + if network == "host" { + if let k = this.publicUrlKey { + this.state.set(k, "http://localhost:{opts.port!}"); + } + + if let k = this.internalUrlKey { + this.state.set(k, "http://localhost:{opts.port!}"); + } + + return () => {}; + } + } + + if let port = opts.port { + let hostPort = out?.tryGetAt(0)?.tryGet("NetworkSettings")?.tryGet("Ports")?.tryGet("{port}/tcp")?.tryGetAt(0)?.tryGet("HostPort")?.tryAsStr(); + if !hostPort? { + throw "Container does not listen to port {port}"; + } + + let publicUrl = "http://localhost:{hostPort!}"; + + if let k = this.publicUrlKey { + this.state.set(k, publicUrl); + } + + if let k = this.internalUrlKey { + this.state.set(k, "http://host.docker.internal:{hostPort!}"); + } + + if let readiness = opts.readiness { + let readinessUrl = "{publicUrl}{readiness}"; + log("waiting for container to be ready: {readinessUrl}..."); + util.waitUntil(inflight () => { + try { + return http.get(readinessUrl).ok; + } catch { + return false; + } + }, interval: 0.1s); + } + } + }, {autoStart: false}) as "ReadinessService"; + std.Node.of(this.readinessService).hidden = true; + + std.Node.of(this.state).hidden = true; + } + + pub inflight start(env: Map) { + for entry in env.entries() { + this.runtimeEnv.put(entry.key, entry.value); + } + + this.containerService.start(); + this.readinessService.start(); + } +} \ No newline at end of file diff --git a/python/sim/function.js b/python/sim/function.js index 10b42352..6453b935 100644 --- a/python/sim/function.js +++ b/python/sim/function.js @@ -13,18 +13,17 @@ module.exports.Function = class Function extends SimFunction { this.pythonInflight = pythonInflight; + for (let e in props.env) { + this.pythonInflight.inner.service.addEnvironment(e, props.env[e]); + } + if (!App.of(this).isTestEnvironment) { for (let key of ["AWS_REGION", "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]) { let value = process.env[key]; if (value) { - this.addEnvironment(key, value); + this.pythonInflight.inner.service.addEnvironment(key, value); } } } } - - _preSynthesize() { - this.pythonInflight.inner.preLift(this); - return super._preSynthesize(); - } } diff --git a/python/sim/inflight.w b/python/sim/inflight.w index 0a398c74..2d59df05 100644 --- a/python/sim/inflight.w +++ b/python/sim/inflight.w @@ -3,13 +3,14 @@ bring util; bring http; bring math; bring sim; +bring "./containers.w" as containers; bring "../types.w" as types; bring "../util.w" as libutil; pub class Inflight impl cloud.IFunctionHandler { pub url: str; + service: cloud.Service; network: str?; - lifts: MutMap; clients: MutMap; new(props: types.InflightProps) { @@ -40,99 +41,101 @@ pub class Inflight impl cloud.IFunctionHandler { this.network = "host"; } - let runner = new sim.Container( + let runner = new containers.Container( image: outdir, name: "python-runner", - volumes: [ - "{props.path}:/var/task:ro", - ], + volumes: { + "/var/task:ro,delegated": props.path, + }, env: { DOCKER_LAMBDA_STAY_OPEN: "1", - WING_TARGET: util.env("WING_TARGET"), }, - containerPort: port, + public: true, + port: port, + exposedPort: port, args: args, + flags: flags.copy(), network: this.network, entrypoint: "/usr/local/bin/aws-lambda-rie", ); + + this.service = new cloud.Service(inflight () => { + let clients = MutMap{}; + for client in this.clients.entries() { + let value = client.value; + + // TODO: move it to a function (there's a weird bug going on here) + let collect = inflight (value: types.LiftedSim) => { + let c = MutMap{}; + if let children = value.children { + for child in children.entries() { + c.set(child.key, collect(child.value)); + } + } - this.url = "http://localhost:{runner.hostPort!}/2015-03-31/functions/function/invocations"; - this.clients = MutMap{}; - - if let lift = props.lift { - this.lifts = lift.copyMut(); - } else { - this.lifts = MutMap{}; - } - } - - inflight context(): Map { - let clients = MutMap{}; - for client in this.clients.entries() { - let value = client.value; - - // TODO: move it to a function (there's a weird bug going on here) - let collect = inflight (value: types.LiftedSim) => { - let c = MutMap{}; - if let children = value.children { - for child in children.entries() { - c.set(child.key, collect(child.value)); + if let handle = util.tryEnv(value.handle) { + // sdk resources + return { + id: value.id, + type: value.type, + path: value.path, + target: value.target, + props: value.props, + children: c.copy(), + handle: handle, + }; + } else { + // custom resources + return { + id: value.id, + type: value.type, + path: value.path, + target: value.target, + props: value.props, + children: c.copy(), + handle: value.handle, + }; } - } + }; + clients.set(client.key, collect(value)); + } - if let handle = util.tryEnv(value.handle) { - // sdk resources - return { - id: value.id, - type: value.type, - path: value.path, - target: value.target, - props: value.props, - children: c.copy(), - handle: handle, - }; - } else { - // custom resources - return { - id: value.id, - type: value.type, - path: value.path, - target: value.target, - props: value.props, - children: c.copy(), - handle: value.handle, - }; - } + let env = MutMap{ + "WING_CLIENTS" => Json.stringify(clients), }; - clients.set(client.key, collect(value)); - } - let env = MutMap{ - "WING_CLIENTS" => Json.stringify(clients), - }; + let var host = "http://host.docker.internal"; + if let network = this.network { + if network == "host" { + host = "http://127.0.0.1"; + } + } - let var host = "http://host.docker.internal"; - if let network = this.network { - if network == "host" { - host = "http://127.0.0.1"; + for e in Inflight.env().entries() { + env.set(e.key, e.value); } - } - for e in Inflight.env().entries() { - env.set(e.key, e.value); - } + for e in env.entries() { + let var value = e.value; + if value.contains("http://localhost") { + value = value.replaceAll("http://localhost", host); + } elif value.contains("http://127.0.0.1") { + value = value.replaceAll("http://127.0.0.1", host); + } + env.set(e.key, value); + } + + runner.start(env.copy()); + }); - for e in env.entries() { - let var value = e.value; - if value.contains("http://localhost") { - value = value.replaceAll("http://localhost", host); - } elif value.contains("http://127.0.0.1") { - value = value.replaceAll("http://127.0.0.1", host); + this.url = "{runner.publicUrl!}/2015-03-31/functions/function/invocations"; + this.clients = MutMap{}; + + if let lifts = props.lift { + for lift in lifts.entries() { + this.lift(lift.value.obj, { id: lift.key, allow: lift.value.allow }); } - env.set(e.key, value); } - - return env.copy(); } pub inflight handle(event: str?): str? { @@ -140,38 +143,29 @@ pub class Inflight impl cloud.IFunctionHandler { } protected inflight _handle(event: str?): str? { - let context = Json.stringify(this.context()); let var body = event; if event == nil || event == "" { - body = Json.stringify({ payload: "", context: context }); + body = Json.stringify({ payload: "" }); } else { if let json = Json.tryParse(event) { - body = Json.stringify({ payload: json, context: context }); + body = Json.stringify({ payload: json }); } else { - body = Json.stringify({ payload: event, context: context }); + body = Json.stringify({ payload: event }); } } let res = http.post(this.url, { body: body }); + if !res.ok { + log("Failed to invoke the function: {Json.stringify(res)}"); + } return res.body; } pub lift(obj: std.Resource, options: types.LiftOptions): cloud.IFunctionHandler { - this.lifts.set(options.id, { obj: obj, allow: options.allow }); + libutil.liftSim(obj, options, this.service, this.clients); return this; } - pub preLift(host: std.IInflightHost) { - for lift in this.lifts.entries() { - libutil.liftSim( - lift.value.obj, - { id: lift.key, allow: lift.value.allow }, - host, - this.clients - ); - } - } - extern "./util.js" inflight static env(): Map; extern "./util.js" static os(): str; } diff --git a/python/sim/util.extern.d.ts b/python/sim/util.extern.d.ts index 9b356f20..8080ff65 100644 --- a/python/sim/util.extern.d.ts +++ b/python/sim/util.extern.d.ts @@ -1,4 +1,5 @@ export default interface extern { + _spawn: (command: string, args: (readonly (string)[]), options: Readonly>>) => Promise, env: () => Promise>>, os: () => string, } diff --git a/python/sim/util.js b/python/sim/util.js index 0c4b22b9..a41b2689 100644 --- a/python/sim/util.js +++ b/python/sim/util.js @@ -1,3 +1,5 @@ +const { spawn } = require("node:child_process"); + exports.env = () => { return process.env; }; @@ -5,3 +7,22 @@ exports.env = () => { exports.os = function() { return process.platform; }; + +exports._spawn = function(command, args, options) { + const child = spawn(command, args, { + stdio: "pipe", + ...options, + }); + + child.stdout.on("data", (data) => + console.log(data.toString().trim()) + ); + + child.stderr.on("data", (data) => + console.log(data.toString().trim()) + ); + + child.once("error", (err) => { + console.error(err); + }); +} diff --git a/python/test-assets/requirements.txt b/python/test-assets/requirements.txt index bca16bb7..f74c2c75 100644 --- a/python/test-assets/requirements.txt +++ b/python/test-assets/requirements.txt @@ -1,4 +1,4 @@ -wingsdk == 0.0.3 +wingsdk == 0.0.4 requests boto3 Faker diff --git a/python/util.js b/python/util.js index 730c541e..941eafeb 100644 --- a/python/util.js +++ b/python/util.js @@ -18,6 +18,24 @@ const createMD5ForProject = (requirementsFile, nodePath = "", path = "", handler return hash.digest("hex"); }; +const tryInspect = (imageName) => { + try { + execSync(`docker inspect ${imageName}`); + return true; + } catch {} +}; + +const forceReloadImage = (imageName) => { + try { + tryInspect(imageName); + execSync(`docker inspect ${imageName}`); + execSync(`docker save ${imageName} -o ${join(tmpdir(), imageName)}`); + execSync(`docker load -i ${join(tmpdir(), imageName)}`); + rmSync(join(tmpdir(), imageName)); + return true; + } catch {} +}; + exports.dirname = () => __dirname; exports.resolve = (path1, path2) => { @@ -36,6 +54,10 @@ exports.buildSim = (options) => { const md5 = createMD5ForProject(requirements, nodePath, path, handler); const imageName = `wing-py:${md5}`; + if (tryInspect(imageName)) { + return imageName; + } + const dockerfile = join(tmpdir(), `Dockerfile-${md5}`); if (!existsSync(dockerfile)) { const dockerfileContent = ` @@ -52,18 +74,8 @@ RUN pip install -r /app/requirements.txt` } ); - const forceReloadImage = () => { - try { - execSync(`docker inspect ${imageName}`); - execSync(`docker save ${imageName} -o ${join(tmpdir(), imageName)}`); - execSync(`docker load -i ${join(tmpdir(), imageName)}`); - rmSync(join(tmpdir(), imageName)); - return true; - } catch {} - }; - if (process.env.CI) { - Util.waitUntil(forceReloadImage); + Util.waitUntil(() => forceReloadImage(imageName)); } return imageName; From 066409a1bd6cb0b4b75646fafa0a7e59230ed975 Mon Sep 17 00:00:00 2001 From: Chris Rybicki Date: Sun, 16 Jun 2024 06:41:44 -0400 Subject: [PATCH 5/5] chore: add .gitattributes file (#264) --- .gitattributes | 7 +++++++ .mkrepo/gitattributes.w | 16 ++++++++++++++++ .mkrepo/main.w | 2 ++ 3 files changed, 25 insertions(+) create mode 100644 .gitattributes create mode 100644 .mkrepo/gitattributes.w diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f6948154 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Auto-generated by ./mkrepo.sh + +/.github/workflows/*-pull.yaml linguist-generated +/.github/workflows/*-release.yaml linguist-generated +/**/package-lock.json linguist-generated +/**/*.extern.d.ts linguist-generated +/package-lock.json linguist-generated diff --git a/.mkrepo/gitattributes.w b/.mkrepo/gitattributes.w new file mode 100644 index 00000000..347470d0 --- /dev/null +++ b/.mkrepo/gitattributes.w @@ -0,0 +1,16 @@ +bring fs; + +pub class GitAttributes { + new() { + let lines = MutArray[]; + lines.push("# Auto-generated by ./mkrepo.sh"); + lines.push(""); + lines.push("/.github/workflows/*-pull.yaml linguist-generated"); + lines.push("/.github/workflows/*-release.yaml linguist-generated"); + lines.push("/**/package-lock.json linguist-generated"); + lines.push("/**/*.extern.d.ts linguist-generated"); + lines.push("/package-lock.json linguist-generated"); + lines.push(""); + fs.writeFile(".gitattributes", lines.join("\n")); + } +} diff --git a/.mkrepo/main.w b/.mkrepo/main.w index 45597571..3dfdf9be 100644 --- a/.mkrepo/main.w +++ b/.mkrepo/main.w @@ -1,5 +1,6 @@ bring fs; bring "./canary.w" as canary; +bring "./gitattributes.w" as gitattributes; bring "./library.w" as l; bring "./mergify.w" as mergify; bring "./pr-lint.w" as prlint; @@ -34,6 +35,7 @@ new stale.StaleWorkflow(workflowdir); new mergify.MergifyWorkflow(libs.copy()); new prdiff.PullRequestDiffWorkflow(workflowdir); new prlint.PullRequestLintWorkflow(workflowdir, libs.copy()); +new gitattributes.GitAttributes(); let skipCanaryTests = [ "containers" // https://github.com/winglang/wing/issues/5716