From 2db31303e7d251f8555ce2d7da255cecb5af2b5c Mon Sep 17 00:00:00 2001
From: Philippe Scorsolini
Date: Tue, 13 Jun 2023 09:40:26 +0200
Subject: [PATCH 01/18] chore(Dockerfile): use COPY instead of ADD
Co-authored-by: Philippe Scorsolini
Co-authored-by: AdamKorcz
Signed-off-by: Philippe Scorsolini
(cherry picked from commit b89eafb431e28a9d8bea1653c4d5ab0405f78e6e)
---
cluster/images/crossplane/Dockerfile | 8 ++++----
cluster/images/xfn/Dockerfile | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/cluster/images/crossplane/Dockerfile b/cluster/images/crossplane/Dockerfile
index 71934c167..8e30eb9c1 100644
--- a/cluster/images/crossplane/Dockerfile
+++ b/cluster/images/crossplane/Dockerfile
@@ -3,9 +3,9 @@ FROM gcr.io/distroless/static@sha256:7198a357ff3a8ef750b041324873960cf2153c11cc5
ARG TARGETOS
ARG TARGETARCH
-ADD bin/$TARGETOS\_$TARGETARCH/crossplane /usr/local/bin/
-ADD crds /crds
-ADD webhookconfigurations /webhookconfigurations
+COPY bin/$TARGETOS\_$TARGETARCH/crossplane /usr/local/bin/
+COPY crds /crds
+COPY webhookconfigurations /webhookconfigurations
EXPOSE 8080
USER 65532
-ENTRYPOINT ["crossplane"]
\ No newline at end of file
+ENTRYPOINT ["crossplane"]
diff --git a/cluster/images/xfn/Dockerfile b/cluster/images/xfn/Dockerfile
index 8966515db..7c509fc88 100644
--- a/cluster/images/xfn/Dockerfile
+++ b/cluster/images/xfn/Dockerfile
@@ -11,7 +11,7 @@ ARG TARGETARCH
# time.
RUN apt-get update && apt-get install -y ca-certificates crun && rm -rf /var/lib/apt/lists/*
-ADD bin/${TARGETOS}\_${TARGETARCH}/xfn /usr/local/bin/
+COPY bin/${TARGETOS}\_${TARGETARCH}/xfn /usr/local/bin/
# We run xfn as root in order to grant it CAP_SETUID and CAP_SETGID, which are
# required in order to create a user namespace with more than one available UID
From 8386b6bd029d96eb9b8151adabbe279db5dab1fe Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Mon, 19 Jun 2023 00:22:26 +0000
Subject: [PATCH 02/18] Update debian:bookworm-slim Docker digest to d8f9d38
---
cluster/images/xfn/Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cluster/images/xfn/Dockerfile b/cluster/images/xfn/Dockerfile
index 7c509fc88..0d6c9b2d0 100644
--- a/cluster/images/xfn/Dockerfile
+++ b/cluster/images/xfn/Dockerfile
@@ -1,5 +1,5 @@
# This is debian:bookworm-slim (i.e. Debian 12, testing), which has crun v1.5.
-FROM debian:bookworm-slim@sha256:e1a80fdca0e09f557a2bef15c7f601c49915e8977ca16eead985e541afeb5770
+FROM debian:bookworm-slim@sha256:d8f9d38c21495b04d1cca99805fbb383856e19794265684019bf193c3b7d67f9
ARG TARGETOS
ARG TARGETARCH
From afd6ebdca58360622ee497778970b5120f6f53b6 Mon Sep 17 00:00:00 2001
From: Philippe Scorsolini
Date: Wed, 21 Jun 2023 17:20:01 +0200
Subject: [PATCH 03/18] fix(crank): copy to tar file one chunk at a time
Co-authored-by: Philippe Scorsolini
Co-authored-by: AdamKorcz
Signed-off-by: Philippe Scorsolini
(cherry picked from commit 1d7537693084b6f4b88ebee098636900ff16cb0d)
---
internal/xpkg/build.go | 23 ++++++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
diff --git a/internal/xpkg/build.go b/internal/xpkg/build.go
index fe318a405..1e941c2d6 100644
--- a/internal/xpkg/build.go
+++ b/internal/xpkg/build.go
@@ -109,7 +109,8 @@ func Build(ctx context.Context, b parser.Backend, p parser.Parser, l parser.Lint
if err := tw.WriteHeader(hdr); err != nil {
return nil, errors.Wrap(err, errTarFromStream)
}
- if _, err = io.Copy(tw, buf); err != nil {
+ // copy chunks of 1MB to avoid loading the entire package into memory twice
+ if _, err = copyChunks(tw, buf, 1024*1024); err != nil {
return nil, errors.Wrap(err, errTarFromStream)
}
if err := tw.Close(); err != nil {
@@ -132,3 +133,23 @@ func Build(ctx context.Context, b parser.Backend, p parser.Parser, l parser.Lint
// Append layer to to scratch image.
return mutate.AppendLayers(empty.Image, layer)
}
+
+// copyChunks pleases gosec per https://github.com/securego/gosec/pull/433.
+// Like Copy it reads from src until EOF, it does not treat an EOF from Read as
+// an error to be reported.
+//
+// NOTE(negz): This rule confused me at first because io.Copy appears to use a
+// buffer, but in fact it bypasses it if src/dst is an io.WriterTo/ReaderFrom.
+func copyChunks(dst io.Writer, src io.Reader, chunkSize int64) (int64, error) {
+ var written int64
+ for {
+ w, err := io.CopyN(dst, src, chunkSize)
+ written += w
+ if errors.Is(err, io.EOF) {
+ return written, nil
+ }
+ if err != nil {
+ return written, err
+ }
+ }
+}
From f403dca36410d31ed869aa59d2a042b06dbd8097 Mon Sep 17 00:00:00 2001
From: Philippe Scorsolini
Date: Tue, 20 Jun 2023 19:37:16 +0200
Subject: [PATCH 04/18] fix: limit xfn stdout and stderr
Signed-off-by: Philippe Scorsolini
(cherry picked from commit 43325354c08313f7556c9ed0f7ab4974cbd2513d)
---
cmd/xfn/spark/spark.go | 38 +++++++++++++++++++++++++++++++--
internal/xfn/container_linux.go | 7 ++++--
2 files changed, 41 insertions(+), 4 deletions(-)
diff --git a/cmd/xfn/spark/spark.go b/cmd/xfn/spark/spark.go
index 3f315aff1..e9690cb9b 100644
--- a/cmd/xfn/spark/spark.go
+++ b/cmd/xfn/spark/spark.go
@@ -69,6 +69,9 @@ const ociRuntimeRoot = "runtime"
// the RunFunctionRequest.
const defaultTimeout = 25 * time.Second
+// The maximum size of stdout/stderr to avoid OOM
+const maxStdioBytes = 100 << 20 // 100 MB
+
// Command runs a containerized Composition Function.
type Command struct {
CacheDir string `short:"c" help:"Directory used for caching function images and containers." default:"/xfn"`
@@ -157,16 +160,47 @@ func (c *Command) Run() error { //nolint:gocyclo // TODO(negz): Refactor some of
cmd := exec.CommandContext(ctx, c.Runtime, "--root="+root, "run", "--bundle="+b.Path(), runID)
cmd.Stdin = bytes.NewReader(req.GetInput())
- out, err := cmd.Output()
+ stdoutPipe, err := cmd.StdoutPipe()
+ if err != nil {
+ _ = b.Cleanup()
+ return errors.Wrap(err, errRuntime)
+ }
+ stderrPipe, err := cmd.StderrPipe()
+ if err != nil {
+ _ = b.Cleanup()
+ return errors.Wrap(err, errRuntime)
+ }
+
+ if err := cmd.Start(); err != nil {
+ _ = b.Cleanup()
+ return errors.Wrap(err, errRuntime)
+ }
+
+ stdout, err := io.ReadAll(io.LimitReader(stdoutPipe, maxStdioBytes))
+ if err != nil {
+ _ = b.Cleanup()
+ return errors.Wrap(err, errRuntime)
+ }
+ stderr, err := io.ReadAll(io.LimitReader(stderrPipe, maxStdioBytes))
if err != nil {
_ = b.Cleanup()
return errors.Wrap(err, errRuntime)
}
+
+ if err := cmd.Wait(); err != nil {
+ var exitErr *exec.ExitError
+ if errors.As(err, &exitErr) {
+ exitErr.Stderr = stderr
+ }
+ _ = b.Cleanup()
+ return errors.Wrap(err, errRuntime)
+ }
+
if err := b.Cleanup(); err != nil {
return errors.Wrap(err, errCleanupBundle)
}
- rsp := &v1alpha1.RunFunctionResponse{Output: out}
+ rsp := &v1alpha1.RunFunctionResponse{Output: stdout}
pb, err = proto.Marshal(rsp)
if err != nil {
return errors.Wrap(err, errMarshalResponse)
diff --git a/internal/xfn/container_linux.go b/internal/xfn/container_linux.go
index 5a82240ac..180d2e581 100644
--- a/internal/xfn/container_linux.go
+++ b/internal/xfn/container_linux.go
@@ -58,6 +58,7 @@ const (
const (
UserNamespaceUIDs = 65536
UserNamespaceGIDs = 65536
+ MaxStdioBytes = 100 << 20 // 100 MB
)
// The subcommand of xfn to invoke - i.e. "xfn spark