Merge pull request #236 from vito/tls

automatic TLS
vito · Aug 29, 2022 · e3ac8c5 · e3ac8c5
2 parents b280ab3 + 8416f8f
commit e3ac8c5
Show file tree

Hide file tree

Showing 35 changed files with 2,268 additions and 1,067 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -7,16 +7,16 @@ whether that's writing code or just providing good signals to the project.
 ## Contributing feedback & discussions
 
 Languages evolve as more people speak them. Bass is very young, so feedback is
-incredibly valuable. Using it for my own projects has led to dramatic changes,
-and it'd be best to get the bulk of these out of the way early on.
+incredibly valuable. Using it for my own projects has led to dramatic changes
+to Bass, and it'd be best to get the bulk of these out of the way early on.
 
 The best place to leave feedback is in [Discussions], but feel free to just hop
 in [Discord] too.
 
 It's hard to use a language without having something to say, so if you don't
 have a project to apply Bass to feel free to critique Bass's own Bass code:
 
-* [project.bass](project.bass) contains the bulk of the project code.
+* [bass/bass.bass](bass/bass.bass) contains the bulk of the project code.
 * [bass/build](bass/build) builds Bass binaries.
 * [bass/docs](bass/docs) builds Bass's docs.
 * [bass/test](bass/test) runs Bass's test suite.
@@ -74,9 +74,7 @@ this is a debt that I won't let grow out of control.
 To run the tests:
 
 ```sh
-# for lsp, which have a lsp config submodule for lsp config
-git submodule update --init --recursive
-go test ./...
+./hack/test
 ```
 
 The tests assume Buildkit is running somewhere, and they discover it the same

diff --git a/Makefile b/Makefile
@@ -8,7 +8,7 @@ shims=$(foreach arch,$(arches),pkg/runtimes/bin/exe.$(arch))
 
 all: cmd/bass/bass
 
-pkg/runtimes/bin/exe.%: pkg/runtimes/shim/main.go
+pkg/runtimes/bin/exe.%: pkg/runtimes/shim/*.go
 	env GOOS=linux GOARCH=$* CGO_ENABLED=0 go build -ldflags "-s -w" -o $@ ./pkg/runtimes/shim
 
 cmd/bass/bass: shims

diff --git a/bass/bass.bass b/bass/bass.bass
@@ -129,15 +129,22 @@
     (map (fn [image] (check-dist dist image))
          smoke-tests))
 
-  (defn test-buildkit [os arch]
-    (-> ($ buildkitd --addr "tcp://0.0.0.0:6107")
-        (with-image (buildkit:image os arch buildkit:test-config))
-        (with-mount (cache-dir "bass buildkitd") /var/lib/buildkit/)
-        (with-port :grpc 6107)
-        insecure!))
+  (defn test-buildkit [os arch cert]
+    (let [resolv (str "nameserver 10.73.0.1\n"
+                      "nameserver 1.1.1.1\n"
+                      "search dns.bass\n")]
+      (from (buildkit:image os arch buildkit:test-config)
+        ($ cp $cert /etc/ssl/certs/bass.crt)
+        ($ chmod "0400" /etc/ssl/certs/bass.crt)
+        (-> ($ buildkitd --addr "tcp://0.0.0.0:6107")
+            (with-mount (cache-dir "bass buildkitd") /var/lib/buildkit/)
+            (with-mount (mkfile ./resolv.conf resolv) /etc/resolv.conf)
+            (with-port :grpc 6107)
+            insecure!))))
 
   (defn with-bass-and-buildkitd [thunk src]
-    (let [buildkit-host (addr (test-buildkit "linux" "amd64") :grpc "tcp://$host:$port")
+    (let [buildkit (test-buildkit "linux" "amd64" src/pkg/runtimes/testdata/tls/bass.crt)
+          buildkit-host (addr buildkit :grpc "tcp://$host:$port")
           built-bass (dist src "dev" "linux" "amd64")]
       (-> thunk
           (with-env {:BUILDKIT_HOST buildkit-host})

diff --git a/bass/buildkit.bass b/bass/buildkit.bass
@@ -1,9 +1,7 @@
-; bumped by hack/bump-buildkit
-(def *buildkit-version* "v0.10.3")
+(def *buildkit-repository* "moby/buildkit")
 
-; TODO: move back to upstream once PRs are merged
-(def *buildkit-repository* "vito/buildkit")
-(def *buildkit-variant* "-conflist-hostname")
+; bumped by hack/bump-buildkit
+(def *buildkit-tag* "master") ; TODO: go back to a pinned version
 
 (def *cni-version* "v1.1.1")
 
@@ -13,17 +11,17 @@
   (defn image [os arch config-dir]
     (from (resolve {:platform {:os os} ; TODO: :arch arch}
                     :repository *buildkit-repository*
-                    :tag (str *buildkit-version* *buildkit-variant*)})
+                    :tag *buildkit-tag*})
       ($ apk add --no-cache dumb-init iptables ip6tables dnsmasq) ; TODO: nix?
       ($ mkdir -p /opt/cni/bin/ /etc/buildkit/)
       ($ cp $dnsname /opt/cni/bin/dnsname)
       ($ tar -zxf (cni os arch) -C /opt/cni/bin/)
       ($ sh -c "cp $0/* /etc/buildkit/" $config-dir)))
 
-  (defn buildkitd-toml [cni-path dns]
+  (defn buildkitd-toml [cni-path]
     (str
       "# support insecure! thunks\n"
-      "insecure-entitlements = [ \"security.insecure\" ]\n"
+      "insecure-entitlements = [\"security.insecure\"]\n"
       "\n"
       "# configure bridge networking\n"
       "[worker.oci]\n"
@@ -32,10 +30,7 @@
       "\n"
       "[worker.containerd]\n"
       "networkMode = \"cni\"\n"
-      "cniConfigPath = \"" (/etc/buildkit/ cni-path) "\"\n"
-      "\n"
-      "[dns]\n"
-      "nameservers = [\"" dns "\"]\n"))
+      "cniConfigPath = \"" (/etc/buildkit/ cni-path) "\"\n"))
 
   (def dnsname
     (subpath
@@ -76,9 +71,14 @@
   (def bass-config
     (mkfs
       ./bass.conflist (json (cni-config "bass" "10.64.0.0/16"))
-      ./buildkitd.toml (buildkitd-toml ./bass.conflist "10.64.0.1")))
+      ./buildkitd.toml (buildkitd-toml ./bass.conflist)))
 
   (def test-config
     (mkfs
       ./test.conflist (json (cni-config "bass" "10.73.0.0/16"))
-      ./buildkitd.toml (buildkitd-toml ./test.conflist "10.73.0.1"))))
+      ./buildkitd.toml (str (buildkitd-toml ./test.conflist)
+                            "\n"
+                            ; override the default nameserver
+                            "[dns]\n"
+                            "nameservers = [\"10.73.0.1\", \"1.1.1.1\"]\n")))
+  )
diff --git a/demos/registry-mirror.bass b/demos/registry-mirror.bass
@@ -0,0 +1,48 @@
+(use (.strings))
+
+(def config
+  {:version "0.1"
+   :http {:addr "0.0.0.0:5000"
+          :tls {:certificate "/registry.crt"
+                :key "/registry.key"}}
+   :storage {:filesystem {:rootdirectory "/var/lib/registry"}}
+   :proxy {:remoteurl "https://registry-1.docker.io"}})
+
+(def mirror
+  (-> ($ registry serve (mkfile ./config.yml (json config)))
+      (with-image (linux/registry))
+      (with-mount (cache-dir "registry-mirror") /var/lib/registry/)
+      (with-tls /registry.crt /registry.key)
+      (with-port :http 5000)))
+
+(defn resolver [thunk platform names]
+  (fn optional
+    (case optional
+      []
+      (let [expand (case names [lib] ["library" lib] _ names)
+            repository (str "$host:$port/" (strings:join "/" expand))]
+        (resolve {:platform platform
+                  :repository (addr thunk :http repository)
+                  :tag "latest"}))
+
+      [dep]
+      (resolver thunk platform (conj names (path-name dep)))
+
+      _
+      (error "invalid image path segment" :segment optional))))
+
+(def xunil
+  (resolver mirror {:os "linux"} []))
+
+(defn main []
+  ; buildkit -> thunk TLS
+  (-> ($ echo "I'm mirrored!")
+      (with-image (xunil/alpine))
+      run)
+
+  ; thunk -> thunk TLS
+  (-> ($ wget -O- (addr mirror :http "https://$host:$port/v2/library/alpine/manifests/latest"))
+      (with-image (xunil/alpine))
+      (read :json)
+      next
+      dump))
diff --git a/go.mod b/go.mod
@@ -9,8 +9,9 @@ require (
 	github.com/alecthomas/chroma v0.9.2
 	github.com/c-bata/go-prompt v0.2.6
 	github.com/cenkalti/backoff/v4 v4.1.2
-	github.com/containerd/containerd v1.6.3-0.20220401172941-5ff8fce1fcc6
-	github.com/docker/distribution v2.8.0+incompatible
+	github.com/containerd/containerd v1.6.6
+	github.com/docker/distribution v2.8.1+incompatible
+	github.com/docker/docker v20.10.7+incompatible
 	github.com/gertd/go-pluralize v0.1.7
 	github.com/gofrs/flock v0.8.1
 	github.com/google/go-cmp v0.5.7
@@ -19,8 +20,8 @@ require (
 	github.com/mattn/go-colorable v0.1.12
 	github.com/mattn/go-isatty v0.0.14
 	github.com/mattn/go-unicodeclass v0.0.1
-	github.com/moby/buildkit v0.10.3
-	github.com/moby/sys/mountinfo v0.6.0
+	github.com/moby/buildkit v0.10.1-0.20220826012947-46c8b9ee45d0
+	github.com/moby/sys/mountinfo v0.6.2
 	github.com/morikuni/aec v1.0.0
 	github.com/neovim/go-client v1.2.2-0.20220118223211-7c85d516f28c
 	github.com/opencontainers/go-digest v1.0.0
@@ -33,6 +34,7 @@ require (
 	github.com/sourcegraph/jsonrpc2 v0.1.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spy16/slurp v0.2.3
+	github.com/square/certstrap v1.3.0
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
 	github.com/vito/booklit v0.12.2-0.20220630023834-33787382be72
 	github.com/vito/go-interact v1.0.1
@@ -48,28 +50,28 @@ require (
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
 	golang.org/x/term v0.0.0-20220411215600-e5f449aeb171
-	google.golang.org/grpc v1.45.0
+	google.golang.org/grpc v1.47.0
 	google.golang.org/protobuf v1.27.1
 )
 
 require (
 	cloud.google.com/go v0.81.0 // indirect
+	filippo.io/edwards25519 v1.0.0-rc.1 // indirect
 	github.com/AdamKorcz/go-fuzz-headers v0.0.0-20210312213058-32f4d319f0d2 // indirect
-	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/apex/log v1.4.0 // indirect
 	github.com/charmbracelet/bubbles v0.13.0 // indirect
 	github.com/charmbracelet/bubbletea v0.22.0 // indirect
 	github.com/charmbracelet/lipgloss v0.5.0 // indirect
 	github.com/containerd/console v1.0.3 // indirect
-	github.com/containerd/continuity v0.2.3-0.20220330195504-d132b287edc8 // indirect
+	github.com/containerd/continuity v0.3.0 // indirect
 	github.com/containerd/go-runc v1.0.0 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/creack/pty v1.1.11 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
 	github.com/dlclark/regexp2 v1.4.0 // indirect
-	github.com/docker/cli v20.10.13+incompatible // indirect
-	github.com/docker/docker v20.10.7+incompatible // indirect
+	github.com/docker/cli v20.10.17+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/fogleman/ease v0.0.0-20170301025033-8da417bf1776 // indirect
 	github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect
@@ -85,7 +87,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/jessevdk/go-flags v1.4.0 // indirect
-	github.com/klauspost/compress v1.15.1 // indirect
+	github.com/klauspost/compress v1.15.7 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/klauspost/pgzip v1.2.4 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
@@ -98,38 +100,39 @@ require (
 	github.com/muesli/cancelreader v0.2.1 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 // indirect
-	github.com/opencontainers/runc v1.1.1 // indirect
+	github.com/opencontainers/runc v1.1.3 // indirect
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/term v1.2.0-beta.2 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rootless-containers/proto v0.1.0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/tonistiigi/fsutil v0.0.0-20220115021204-b19f7f9cb274 // indirect
+	github.com/tonistiigi/fsutil v0.0.0-20220510150904-0dbf3a8a7d58 // indirect
 	github.com/tonistiigi/vt100 v0.0.0-20210615222946-8066bb97264f // indirect
 	github.com/vbatts/go-mtree v0.5.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.29.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.1 // indirect
 	go.opentelemetry.io/otel/sdk v1.4.1 // indirect
 	go.opentelemetry.io/otel/trace v1.4.1 // indirect
 	go.opentelemetry.io/proto/otlp v0.12.0 // indirect
+	go.step.sm/crypto v0.16.2 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/goleak v1.1.12 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
+	google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )
 
 // BEGIN SYNC buildkit
-replace github.com/docker/docker => github.com/docker/docker v20.10.3-0.20220224222438-c78f6963a1c0+incompatible
+replace github.com/docker/docker => github.com/docker/docker v20.10.3-0.20220414164044-61404de7df1a+incompatible
 
 // END SYNC