Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/add cruntime for machine sdk 17 use tar stream #17

Closed
wants to merge 3 commits into from
Closed

Feat/add cruntime for machine sdk 17 use tar stream #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4e4692e
fixup! feat(cli): use cruntime to build snapshot
endersonmaia May 6, 2024
45e6dc9
fixup! fixup! feat(cli): use cruntime to build snapshot
endersonmaia May 7, 2024
7b68190
wip
endersonmaia May 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/cli/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"ora": "^8.0.1",
"progress-stream": "^2.0",
"semver": "^7.5",
"tar-stream": "^3.1.7",
"tmp": "^0.2.3",
"viem": "^2.9.29"
},
Expand All @@ -52,6 +53,7 @@
"@types/progress-stream": "^2.0",
"@types/prompts": "^2.4",
"@types/semver": "^7.5.8",
"@types/tar-stream": "^3.1.3",
"@types/tmp": "^0.2",
"@wagmi/cli": "^2.1.4",
"copyfiles": "^2",
Expand Down
157 changes: 71 additions & 86 deletions apps/cli/src/commands/build.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ import { Flags } from "@oclif/core";
import bytes from "bytes";
import { execa } from "execa";
import fs from "fs-extra";
import { createTar } from "nanotar";
import path from "path";
import semver from "semver";
import tar from "tar-stream";
import tmp from "tmp";

import { BaseCommand } from "../baseCommand.js";
Expand Down Expand Up @@ -166,13 +167,7 @@ Update your application Dockerfile using one of the templates at https://github.
outputFilePath: string,
): Promise<void> {
// create docker tarball from app image
const { stdout: appCid } = await execa("docker", [
"image",
"save",
image,
"-o",
outputFilePath,
]);
await execa("docker", ["image", "save", image, "-o", outputFilePath]);
}

// this wraps the call to the sdk image with a one-shot approach
Expand Down Expand Up @@ -284,26 +279,13 @@ Update your application Dockerfile using one of the templates at https://github.
private async createMachineSnapshotCommand(
info: ImageInfo,
): Promise<string[]> {
const { flags } = await this.parse(BuildApplication);
const driveType = flags["drive-type"] as DriveType;

const ramSize = info.ramSize;
const entrypoint = [
"rollup-init",
"crun",
"run",
"--config",
"/run/cruntime/config/config.json",
"--bundle",
"/run/cruntime",
"app",
].join(" ");
const entrypoint = ["rollup-init", "crun", "run", "app"].join(" ");
const cwd = "--append-init=WORKDIR=/run/cruntime";

const flashDriveArgs: string[] = [
`--flash-drive=label:root,filename:/tmp/input0`,
`--flash-drive=label:config,filename:/tmp/input1,mount:/run/cruntime/config`,
`--flash-drive=label:dapp,filename:/tmp/input2,mount:/run/cruntime/rootfs`,
`--flash-drive=label:dapp,filename:/tmp/input1,mount:/run/cruntime`,
];

const result = [
Expand All @@ -314,25 +296,21 @@ Update your application Dockerfile using one of the templates at https://github.
"--final-hash",
"--store=/tmp/output",
"--append-bootargs=no4lvl",
"--append-bootargs=rootfstype=squashfs",
"--append-init=/bin/sh /etc/cartesi-init.d/cruntime-init",
cwd,
`--append-entrypoint=${entrypoint}`,
];

if (driveType === "sqfs")
result.push("--append-bootargs=rootfstype=squashfs");

return result;
}

//TODO: embedd cruntime.sqfs into cartesi/sdk image
// move the packages/cruntime code to packages/sdk
private async createCruntimeDrive(
image: string,
imageInfo: ImageInfo,
sdkImage: string,
): Promise<void> {
const { flags } = await this.parse(BuildApplication);
const driveType = flags["drive-type"] as DriveType;

const cruntimeTarPath = this.getContextPath("cruntime.tar");
const cruntimeGnutarPath = this.getContextPath("cruntime.gnutar");
const cruntimeDrivePath = this.getContextPath("cruntime.drive");
Expand All @@ -349,10 +327,7 @@ Update your application Dockerfile using one of the templates at https://github.

await this.sdkRun(
sdkImage,
BuildApplication.createDriveCommand(
driveType,
bytes.parse(imageInfo.dataSize),
),
BuildApplication.createDriveCommand("sqfs", 0),
[cruntimeGnutarPath],
cruntimeDrivePath,
);
Expand All @@ -362,40 +337,7 @@ Update your application Dockerfile using one of the templates at https://github.
}
}

private async createOCIConfigeDrive(
imageInfo: ImageInfo,
sdkImage: string,
): Promise<void> {
const { flags } = await this.parse(BuildApplication);
const driveType = flags["drive-type"] as DriveType;

const ociConfigTarPath = this.getContextPath("ociconfig.tar");
const ociConfigDrivePath = this.getContextPath("ociconfig.drive");

try {
const configTar = createTar([
{
name: "config.json",
data: JSON.stringify(createConfig(imageInfo)),
},
]);
fs.writeFileSync(ociConfigTarPath, configTar);

await this.sdkRun(
sdkImage,
BuildApplication.createDriveCommand(
driveType,
bytes.parse(imageInfo.dataSize),
),
[ociConfigTarPath],
ociConfigDrivePath,
);
} finally {
await fs.remove(ociConfigTarPath);
}
}

private async createAppDrive(
private async createAppOCIBundle(
image: string,
imageInfo: ImageInfo,
sdkImage: string,
Expand All @@ -405,6 +347,7 @@ Update your application Dockerfile using one of the templates at https://github.

const appTarPath = this.getContextPath("app.tar");
const appGnutarPath = this.getContextPath("app.gnutar");
const appOCIBundlePath = this.getContextPath("app.ocibundle.tar");
const appDrivePath = this.getContextPath("app.drive");

try {
Expand All @@ -419,29 +362,81 @@ Update your application Dockerfile using one of the templates at https://github.
appGnutarPath,
);

const ociConfigJSON = JSON.stringify(createConfig(imageInfo));
const rootfsPrefix = "rootfs/";

// extract pipe
const extract = tar.extract();
extract.on("error", (err: Error) => {
throw err;
});

// pack pipe
const pack = tar.pack();
pack.on("error", (err: Error) => {
throw err;
});

// add config.json
pack.entry({ name: "config.json" }, ociConfigJSON, function (err) {
if (err) throw err;
});
// add rootfs/ directory
pack.entry({ name: rootfsPrefix, type: "directory" });

// readStream for appGnutarPath
const appGnutarStream = fs.createReadStream(appGnutarPath);
appGnutarStream.on("error", (err: Error) => {
throw err;
});

// writeStream for appOCIBundlePath
const appOCIBundleStream = fs.createWriteStream(appOCIBundlePath);
appOCIBundleStream.on("error", (err: Error) => {
throw err;
});

// for every entry on extract add rootfs/ prefix into pack
extract.on("entry", function (header, stream, callback) {
header.name = path.join(rootfsPrefix, header.name);
stream.pipe(pack.entry(header, callback));
});

extract.on("finish", function () {
pack.finalize();
console.log('extract.on("finish") -> pack.finilize()');
});

appOCIBundleStream.on("close", function () {
console.log(path + " has been written");
});

// save tarball for OCI Bundle
appGnutarStream.pipe(extract);
pack.pipe(appOCIBundleStream);

// create drive
await this.sdkRun(
sdkImage,
BuildApplication.createDriveCommand(
driveType,
bytes.parse(imageInfo.dataSize),
),
[appGnutarPath],
[appOCIBundlePath],
appDrivePath,
);
} finally {
await fs.remove(appGnutarPath);
await fs.remove(appTarPath);
//await fs.remove(appTarPath);
//await fs.remove(appGnutarPath);
//await fs.remove(appOCIBundlePath);
}
}

public async run(): Promise<void> {
const { flags } = await this.parse(BuildApplication);

const tarPath = this.getContextPath("image.tar");
const snapshotPath = this.getContextPath("image");
const cruntimeDrivePath = this.getContextPath("cruntime.drive");
const ociConfigDrivePath = this.getContextPath("ociconfig.drive");
const appDrivePath = this.getContextPath("app.drive");

// clean up temp files we create along the process
Expand All @@ -460,28 +455,18 @@ Update your application Dockerfile using one of the templates at https://github.
const sdkImage = `cartesi/sdk:${imageInfo.sdkVersion}`;

try {
// create docker tarball for image specified
await this.createTarball(appImage, tarPath);

// create cruntime drive
await this.createCruntimeDrive(
CARTESI_CRUNTIME_IMAGE,
imageInfo,
sdkImage,
);

// create oci config drive
await this.createOCIConfigeDrive(imageInfo, sdkImage);
await this.createCruntimeDrive(CARTESI_CRUNTIME_IMAGE, sdkImage);

// create app drive
await this.createAppDrive(appImage, imageInfo, sdkImage);
await this.createAppOCIBundle(appImage, imageInfo, sdkImage);

// create machine snapshot
if (!flags["skip-snapshot"]) {
await this.sdkRun(
sdkImage,
await this.createMachineSnapshotCommand(imageInfo),
[cruntimeDrivePath, ociConfigDrivePath, appDrivePath],
[cruntimeDrivePath, appDrivePath],
snapshotPath,
);
await fs.chmod(snapshotPath, 0o755);
Expand Down
2 changes: 1 addition & 1 deletion apps/cli/src/commands/shell.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ export default class Shell extends BaseCommand<typeof Shell> {
args.push("-it");
}

await execa("docker", [...args, "--", "/bin/bash"], {
await execa("docker", [...args, "--", "/bin/sh"], {
stdio: "inherit",
});
}
Expand Down
47 changes: 47 additions & 0 deletions pnpm-lock.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading