From 3f06a608c36c05ada6fdcceb58a30eb59e2c1446 Mon Sep 17 00:00:00 2001 From: 117503445 Date: Wed, 7 Aug 2024 23:40:41 +0800 Subject: [PATCH 1/3] feat: support client verify for derp --- hscontrol/app.go | 2 ++ hscontrol/handlers.go | 71 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/hscontrol/app.go b/hscontrol/app.go index b66e939b4d..69b0bf40e4 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -445,6 +445,8 @@ func (h *Headscale) createRouter(grpcMux *grpcRuntime.ServeMux) *mux.Router { router.HandleFunc("/swagger/v1/openapiv2.json", headscale.SwaggerAPIv1). Methods(http.MethodGet) + router.HandleFunc("/verify", h.VerifyHandler).Methods(http.MethodPost) + if h.cfg.DERP.ServerEnabled { router.HandleFunc("/derp", h.DERPServer.DERPHandler) router.HandleFunc("/derp/probe", derpServer.DERPProbeHandler) diff --git a/hscontrol/handlers.go b/hscontrol/handlers.go index 6efe1984b0..0e185dce09 100644 --- a/hscontrol/handlers.go +++ b/hscontrol/handlers.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "html/template" + "io" "net/http" "strconv" "time" @@ -53,6 +54,76 @@ func parseCabailityVersion(req *http.Request) (tailcfg.CapabilityVersion, error) return tailcfg.CapabilityVersion(clientCapabilityVersion), nil } +// see https://github.com/tailscale/tailscale/blob/964282d34f06ecc06ce644769c66b0b31d118340/derp/derp_server.go#L1159, Derp use verifyClientsURL to verify whether a client is allowed to connect to the DERP server. +func (h *Headscale) VerifyHandler( + writer http.ResponseWriter, + req *http.Request, +) { + if req.Method != http.MethodPost { + http.Error(writer, "Wrong method", http.StatusMethodNotAllowed) + return + } + log.Debug(). + Str("handler", "/verify"). + Msg("verify client") + + body, err := io.ReadAll(req.Body) + if err != nil { + log.Error(). + Str("handler", "/verify"). + Err(err). + Msg("Cannot read request body") + http.Error(writer, "Internal error", http.StatusInternalServerError) + return + } + + var derpAdmitClientRequest tailcfg.DERPAdmitClientRequest + if err := json.Unmarshal(body, &derpAdmitClientRequest); err != nil { + log.Error(). + Caller(). + Err(err). + Msg("Cannot parse derpAdmitClientRequest") + http.Error(writer, "Internal error", http.StatusInternalServerError) + return + } + + nodes, err := h.db.ListNodes() + if err != nil { + log.Error(). + Caller(). + Err(err). + Msg("Cannot list nodes") + http.Error(writer, "Internal error", http.StatusInternalServerError) + } + + for _, node := range nodes { + log.Debug().Str("node", node.NodeKey.String()).Msg("Node") + } + + allow := false + // Check if the node is in the list of nodes + for _, node := range nodes { + if node.NodeKey == derpAdmitClientRequest.NodePublic { + allow = true + break + } + } + + resp := tailcfg.DERPAdmitClientResponse{ + Allow: allow, + } + + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusOK) + err = json.NewEncoder(writer).Encode(resp) + if err != nil { + log.Error(). + Caller(). + Err(err). + Msg("Failed to write response") + } +} + // KeyHandler provides the Headscale pub key // Listens in /key. func (h *Headscale) KeyHandler( From a8f091fadc9651d2dd7727caea7c3dbd91a7ba2a Mon Sep 17 00:00:00 2001 From: 117503445 Date: Wed, 7 Aug 2024 23:41:45 +0800 Subject: [PATCH 2/3] docs: fix doc for integration test --- integration/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/README.md b/integration/README.md index e5676a44c6..56247c52f8 100644 --- a/integration/README.md +++ b/integration/README.md @@ -11,10 +11,10 @@ Tests are located in files ending with `_test.go` and the framework are located ## Running integration tests locally -The easiest way to run tests locally is to use `[act](INSERT LINK)`, a local GitHub Actions runner: +The easiest way to run tests locally is to use [act](https://github.com/nektos/act), a local GitHub Actions runner: ``` -act pull_request -W .github/workflows/test-integration-v2-TestPingAllByIP.yaml +act pull_request -W .github/workflows/test-integration.yaml ``` Alternatively, the `docker run` command in each GitHub workflow file can be used. From ffd0a012605e081ae7b75254851ef4161da1c44d Mon Sep 17 00:00:00 2001 From: ArcticLampyrid Date: Wed, 7 Aug 2024 23:37:14 +0800 Subject: [PATCH 3/3] tests: add integration test for DERP verify endpoint --- .github/workflows/test-integration.yaml | 1 + Dockerfile.derper | 19 ++ integration/derp_verify_endpoint_test.go | 111 ++++++++ integration/dsic/dsic.go | 316 +++++++++++++++++++++++ integration/embedded_derp_test.go | 2 +- integration/hsic/hsic.go | 153 +++++------ integration/integrationutil/util.go | 90 +++++++ integration/scenario.go | 28 +- integration/tailscale.go | 1 + integration/tsic/tsic.go | 56 ++-- 10 files changed, 661 insertions(+), 116 deletions(-) create mode 100644 Dockerfile.derper create mode 100644 integration/derp_verify_endpoint_test.go create mode 100644 integration/dsic/dsic.go diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index bf55e2de30..a0887fa88c 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -37,6 +37,7 @@ jobs: - TestNodeRenameCommand - TestNodeMoveCommand - TestPolicyCommand + - TestDERPVerifyEndpoint - TestDERPServerScenario - TestPingAllByIP - TestPingAllByIPPublicDERP diff --git a/Dockerfile.derper b/Dockerfile.derper new file mode 100644 index 0000000000..a2566e5a43 --- /dev/null +++ b/Dockerfile.derper @@ -0,0 +1,19 @@ +# For testing purposes only + +FROM golang:1.22-alpine AS build-env + +WORKDIR /go/src + +RUN apk add --no-cache git +ARG VERSION_BRANCH=main +RUN git clone https://github.com/tailscale/tailscale.git --branch=$VERSION_BRANCH --depth=1 +WORKDIR /go/src/tailscale + +ARG TARGETARCH +RUN GOARCH=$TARGETARCH go install -v ./cmd/derper + +FROM alpine:3.18 +RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables curl + +COPY --from=build-env /go/bin/* /usr/local/bin/ +ENTRYPOINT [ "/usr/local/bin/derper" ] diff --git a/integration/derp_verify_endpoint_test.go b/integration/derp_verify_endpoint_test.go new file mode 100644 index 0000000000..0297e2806d --- /dev/null +++ b/integration/derp_verify_endpoint_test.go @@ -0,0 +1,111 @@ +package integration + +import ( + "encoding/json" + "fmt" + "net" + "strconv" + "strings" + "testing" + + "github.com/juanfont/headscale/hscontrol/util" + "github.com/juanfont/headscale/integration/dsic" + "github.com/juanfont/headscale/integration/hsic" + "github.com/juanfont/headscale/integration/integrationutil" + "github.com/juanfont/headscale/integration/tsic" +) + +func TestDERPVerifyEndpoint(t *testing.T) { + IntegrationSkip(t) + + // Generate random hostname for the headscale instance + hash, err := util.GenerateRandomStringDNSSafe(6) + assertNoErr(t, err) + testName := "derpverify" + hostname := fmt.Sprintf("hs-%s-%s", testName, hash) + + headscalePort := 8080 + + // Create cert for headscale + certHeadscale, keyHeadscale, err := integrationutil.CreateCertificate(hostname) + assertNoErr(t, err) + + scenario, err := NewScenario(dockertestMaxWait()) + assertNoErr(t, err) + defer scenario.Shutdown() + + spec := map[string]int{ + "user1": 10, + } + + derper, err := scenario.CreateDERPServer("head", + dsic.WithCACert(certHeadscale), + dsic.WithVerifyClientURL(fmt.Sprintf("https://%s/verify", net.JoinHostPort(hostname, strconv.Itoa(headscalePort)))), + ) + assertNoErr(t, err) + + derpConfig := "regions:\n" + derpConfig += " 900:\n" + derpConfig += " regionid: 900\n" + derpConfig += " regioncode: test-derpverify\n" + derpConfig += " regionname: TestDerpVerify\n" + derpConfig += " nodes:\n" + derpConfig += " - name: TestDerpVerify\n" + derpConfig += " regionid: 900\n" + derpConfig += " hostname: " + derper.GetHostname() + "\n" + derpConfig += " stunport: " + derper.GetSTUNPort() + "\n" + derpConfig += " stunonly: false\n" + derpConfig += " derpport: " + derper.GetDERPPort() + "\n" + + headscale, err := scenario.Headscale( + hsic.WithHostname(hostname), + hsic.WithPort(headscalePort), + hsic.WithCustomTLS(certHeadscale, keyHeadscale), + hsic.WithHostnameAsServerURL(), + hsic.WithCustomDERPServerOnly([]byte(derpConfig)), + ) + assertNoErrHeadscaleEnv(t, err) + + for userName, clientCount := range spec { + err = scenario.CreateUser(userName) + if err != nil { + t.Fatalf("failed to create user %s: %s", userName, err) + } + + err = scenario.CreateTailscaleNodesInUser(userName, "all", clientCount, tsic.WithCACert(derper.GetCert())) + if err != nil { + t.Fatalf("failed to create tailscale nodes in user %s: %s", userName, err) + } + + key, err := scenario.CreatePreAuthKey(userName, true, true) + if err != nil { + t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err) + } + + err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey()) + if err != nil { + t.Fatalf("failed to run tailscale up for user %s: %s", userName, err) + } + } + + allClients, err := scenario.ListTailscaleClients() + assertNoErrListClients(t, err) + + for _, client := range allClients { + report, err := client.DebugDERPRegion("test-derpverify") + assertNoErr(t, err) + successful := false + for _, line := range report.Info { + if strings.Contains(line, "Successfully established a DERP connection with node") { + successful = true + + break + } + } + if !successful { + stJSON, err := json.Marshal(report) + assertNoErr(t, err) + t.Errorf("Client %s could not establish a DERP connection: %s", client.Hostname(), string(stJSON)) + } + } +} diff --git a/integration/dsic/dsic.go b/integration/dsic/dsic.go new file mode 100644 index 0000000000..78339009d0 --- /dev/null +++ b/integration/dsic/dsic.go @@ -0,0 +1,316 @@ +package dsic + +import ( + "crypto/tls" + "errors" + "fmt" + "log" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/juanfont/headscale/hscontrol/util" + "github.com/juanfont/headscale/integration/dockertestutil" + "github.com/juanfont/headscale/integration/integrationutil" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" +) + +const ( + dsicHashLength = 6 + dockerContextPath = "../." + caCertRoot = "/usr/local/share/ca-certificates" + DERPerCertRoot = "/usr/local/share/derper-certs" + dockerExecuteTimeout = 60 * time.Second +) + +var errDERPerStatusCodeNotOk = errors.New("DERPer status code not OK") + +// DERPServerInContainer represents DERP Server in Container (DSIC). +type DERPServerInContainer struct { + version string + hostname string + + pool *dockertest.Pool + container *dockertest.Resource + network *dockertest.Network + + stunPort int + derpPort int + caCerts [][]byte + tlsCert []byte + tlsKey []byte + withExtraHosts []string + withVerifyClientURL string + workdir string +} + +// Option represent optional settings that can be given to a +// DERPer instance. +type Option = func(c *DERPServerInContainer) + +// WithCACert adds it to the trusted surtificate of the Tailscale container. +func WithCACert(cert []byte) Option { + return func(dsic *DERPServerInContainer) { + dsic.caCerts = append(dsic.caCerts, cert) + } +} + +// WithOrCreateNetwork sets the Docker container network to use with +// the DERPer instance, if the parameter is nil, a new network, +// isolating the DERPer, will be created. If a network is +// passed, the DERPer instance will join the given network. +func WithOrCreateNetwork(network *dockertest.Network) Option { + return func(tsic *DERPServerInContainer) { + if network != nil { + tsic.network = network + + return + } + + network, err := dockertestutil.GetFirstOrCreateNetwork( + tsic.pool, + fmt.Sprintf("%s-network", tsic.hostname), + ) + if err != nil { + log.Fatalf("failed to create network: %s", err) + } + + tsic.network = network + } +} + +// WithDockerWorkdir allows the docker working directory to be set. +func WithDockerWorkdir(dir string) Option { + return func(tsic *DERPServerInContainer) { + tsic.workdir = dir + } +} + +// WithVerifyClientURL sets the URL to verify the client. +func WithVerifyClientURL(url string) Option { + return func(tsic *DERPServerInContainer) { + tsic.withVerifyClientURL = url + } +} + +// WithExtraHosts adds extra hosts to the container. +func WithExtraHosts(hosts []string) Option { + return func(tsic *DERPServerInContainer) { + tsic.withExtraHosts = hosts + } +} + +// New returns a new TailscaleInContainer instance. +func New( + pool *dockertest.Pool, + version string, + network *dockertest.Network, + opts ...Option, +) (*DERPServerInContainer, error) { + hash, err := util.GenerateRandomStringDNSSafe(dsicHashLength) + if err != nil { + return nil, err + } + + hostname := fmt.Sprintf("derp-%s-%s", strings.ReplaceAll(version, ".", "-"), hash) + tlsCert, tlsKey, err := integrationutil.CreateCertificate(hostname) + if err != nil { + return nil, fmt.Errorf("failed to create certificates for headscale test: %w", err) + } + dsic := &DERPServerInContainer{ + version: version, + hostname: hostname, + pool: pool, + network: network, + tlsCert: tlsCert, + tlsKey: tlsKey, + stunPort: 3478, //nolint + derpPort: 443, //nolint + } + + for _, opt := range opts { + opt(dsic) + } + + cmdArgs := "--hostname=" + hostname + cmdArgs += " --certmode=manual" + cmdArgs += " --certdir=" + DERPerCertRoot + cmdArgs += " --a=:" + strconv.Itoa(dsic.derpPort) + cmdArgs += " --stun=true" + cmdArgs += " --stun-port=" + strconv.Itoa(dsic.stunPort) + if dsic.withVerifyClientURL != "" { + cmdArgs += " --verify-client-url=" + dsic.withVerifyClientURL + } + + runOptions := &dockertest.RunOptions{ + Name: hostname, + Networks: []*dockertest.Network{dsic.network}, + ExtraHosts: dsic.withExtraHosts, + // we currently need to give us some time to inject the certificate further down. + Entrypoint: []string{"/bin/sh", "-c", "/bin/sleep 3 ; update-ca-certificates ; derper " + cmdArgs}, + ExposedPorts: []string{ + "80/tcp", + fmt.Sprintf("%d/tcp", dsic.derpPort), + fmt.Sprintf("%d/udp", dsic.stunPort), + }, + } + + if dsic.workdir != "" { + runOptions.WorkingDir = dsic.workdir + } + + // dockertest isnt very good at handling containers that has already + // been created, this is an attempt to make sure this container isnt + // present. + err = pool.RemoveContainerByName(hostname) + if err != nil { + return nil, err + } + + var container *dockertest.Resource + buildOptions := &dockertest.BuildOptions{ + Dockerfile: "Dockerfile.derper", + ContextDir: dockerContextPath, + BuildArgs: []docker.BuildArg{}, + } + switch version { + case "head": + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "main", + }) + default: + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "v" + version, + }) + } + container, err = pool.BuildAndRunWithBuildOptions( + buildOptions, + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + if err != nil { + return nil, fmt.Errorf( + "%s could not start tailscale DERPer container (version: %s): %w", + hostname, + version, + err, + ) + } + log.Printf("Created %s container\n", hostname) + + dsic.container = container + + for i, cert := range dsic.caCerts { + err = dsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert) + if err != nil { + return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) + } + } + if len(dsic.tlsCert) != 0 { + err = dsic.WriteFile(fmt.Sprintf("%s/%s.crt", DERPerCertRoot, dsic.hostname), dsic.tlsCert) + if err != nil { + return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) + } + } + if len(dsic.tlsKey) != 0 { + err = dsic.WriteFile(fmt.Sprintf("%s/%s.key", DERPerCertRoot, dsic.hostname), dsic.tlsKey) + if err != nil { + return nil, fmt.Errorf("failed to write TLS key to container: %w", err) + } + } + return dsic, nil +} + +// Shutdown stops and cleans up the DERPer container. +func (t *DERPServerInContainer) Shutdown() error { + err := t.SaveLog("/tmp/control") + if err != nil { + log.Printf( + "Failed to save log from %s: %s", + t.hostname, + fmt.Errorf("failed to save log: %w", err), + ) + } + return t.pool.Purge(t.container) +} + +// GetCert returns the TLS certificate of the DERPer instance. +func (t *DERPServerInContainer) GetCert() []byte { + return t.tlsCert +} + +// Hostname returns the hostname of the DERPer instance. +func (t *DERPServerInContainer) Hostname() string { + return t.hostname +} + +// Version returns the running DERPer version of the instance. +func (t *DERPServerInContainer) Version() string { + return t.version +} + +// ID returns the Docker container ID of the DERPServerInContainer +// instance. +func (t *DERPServerInContainer) ID() string { + return t.container.Container.ID +} + +func (t *DERPServerInContainer) GetHostname() string { + return t.hostname +} + +// GetSTUNPort returns the STUN port of the DERPer instance. +func (t *DERPServerInContainer) GetSTUNPort() string { + return strconv.Itoa(t.stunPort) +} + +// GetDERPPort returns the DERP port of the DERPer instance. +func (t *DERPServerInContainer) GetDERPPort() string { + return strconv.Itoa(t.derpPort) +} + +// WaitForRunning blocks until the DERPer instance is ready to be used. +func (t *DERPServerInContainer) WaitForRunning() error { + url := "https://" + net.JoinHostPort(t.GetHostname(), t.GetDERPPort()) + "/" + log.Printf("waiting for DERPer to be ready at %s", url) + + insecureTransport := http.DefaultTransport.(*http.Transport).Clone() //nolint + insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint + client := &http.Client{Transport: insecureTransport} + + return t.pool.Retry(func() error { + resp, err := client.Get(url) //nolint + if err != nil { + return fmt.Errorf("headscale is not ready: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return errDERPerStatusCodeNotOk + } + + return nil + }) +} + +// ConnectToNetwork connects the DERPer instance to a network. +func (t *DERPServerInContainer) ConnectToNetwork(network *dockertest.Network) error { + return t.container.ConnectToNetwork(network) +} + +// WriteFile save file inside the container. +func (t *DERPServerInContainer) WriteFile(path string, data []byte) error { + return integrationutil.WriteFileToContainer(t.pool, t.container, path, data) +} + +// SaveLog saves the current stdout log of the container to a path +// on the host system. +func (t *DERPServerInContainer) SaveLog(path string) error { + return dockertestutil.SaveLog(t.pool, t.container, path) +} diff --git a/integration/embedded_derp_test.go b/integration/embedded_derp_test.go index 39a9accaf6..dfc872b85f 100644 --- a/integration/embedded_derp_test.go +++ b/integration/embedded_derp_test.go @@ -163,7 +163,7 @@ func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser( cert := hsServer.GetCert() opts = append(opts, - tsic.WithHeadscaleTLS(cert), + tsic.WithCACert(cert), ) user.createWaitGroup.Go(func() error { diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index 3794e085e3..114952717e 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -1,19 +1,12 @@ package hsic import ( - "bytes" - "crypto/rand" - "crypto/rsa" "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" "encoding/json" - "encoding/pem" "errors" "fmt" "io" "log" - "math/big" "net" "net/http" "net/url" @@ -37,6 +30,7 @@ import ( const ( hsicHashLength = 6 dockerContextPath = "../." + caCertRoot = "/usr/local/share/ca-certificates" aclPolicyPath = "/etc/headscale/acl.hujson" tlsCertPath = "/etc/headscale/tls.cert" tlsKeyPath = "/etc/headscale/tls.key" @@ -64,6 +58,7 @@ type HeadscaleInContainer struct { // optional config port int extraPorts []string + caCerts [][]byte hostPortBindings map[string][]string aclPolicy *policy.ACLPolicy env map[string]string @@ -88,18 +83,29 @@ func WithACLPolicy(acl *policy.ACLPolicy) Option { } } +// WithCACert adds it to the trusted surtificate of the container. +func WithCACert(cert []byte) Option { + return func(hsic *HeadscaleInContainer) { + hsic.caCerts = append(hsic.caCerts, cert) + } +} + // WithTLS creates certificates and enables HTTPS. func WithTLS() Option { return func(hsic *HeadscaleInContainer) { - cert, key, err := createCertificate(hsic.hostname) + cert, key, err := integrationutil.CreateCertificate(hsic.hostname) if err != nil { log.Fatalf("failed to create certificates for headscale test: %s", err) } - // TODO(kradalby): Move somewhere appropriate - hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath - hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath + hsic.tlsCert = cert + hsic.tlsKey = key + } +} +// WithCustomTLS uses the given certificates for the Headscale instance. +func WithCustomTLS(cert, key []byte) Option { + return func(hsic *HeadscaleInContainer) { hsic.tlsCert = cert hsic.tlsKey = key } @@ -146,6 +152,13 @@ func WithTestName(testName string) Option { } } +// WithHostname sets the hostname of the Headscale instance. +func WithHostname(hostname string) Option { + return func(hsic *HeadscaleInContainer) { + hsic.hostname = hostname + } +} + // WithHostnameAsServerURL sets the Headscale ServerURL based on // the Hostname. func WithHostnameAsServerURL() Option { @@ -203,6 +216,27 @@ func WithEmbeddedDERPServerOnly() Option { } } +// WithCustomDERPServerOnly configures Headscale use a custom +// DERP server only. +func WithCustomDERPServerOnly(contents []byte) Option { + return func(hsic *HeadscaleInContainer) { + hsic.env["HEADSCALE_DERP_PATHS"] = "/etc/headscale/derp.yml" + hsic.filesInContainer = append(hsic.filesInContainer, + fileInContainer{ + path: "/etc/headscale/derp.yml", + contents: contents, + }) + + // Disable global DERP server and embedded DERP server + hsic.env["HEADSCALE_DERP_URLS"] = "" + hsic.env["HEADSCALE_DERP_SERVER_ENABLED"] = "false" + + // Envknob for enabling DERP debug logs + hsic.env["DERP_DEBUG_LOGS"] = "true" + hsic.env["DERP_PROBER_DEBUG_LOGS"] = "true" + } +} + // WithTuning allows changing the tuning settings easily. func WithTuning(batchTimeout time.Duration, mapSessionChanSize int) Option { return func(hsic *HeadscaleInContainer) { @@ -294,6 +328,10 @@ func New( "HEADSCALE_DEBUG_HIGH_CARDINALITY_METRICS=1", "HEADSCALE_DEBUG_DUMP_CONFIG=1", } + if hsic.hasTLS() { + hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath + hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath + } for key, value := range hsic.env { env = append(env, fmt.Sprintf("%s=%s", key, value)) } @@ -307,7 +345,7 @@ func New( // Cmd: []string{"headscale", "serve"}, // TODO(kradalby): Get rid of this hack, we currently need to give us some // to inject the headscale configuration further down. - Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve ; /bin/sleep 30"}, + Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; update-ca-certificates ; headscale serve ; /bin/sleep 30"}, Env: env, } @@ -345,6 +383,14 @@ func New( hsic.container = container + // Write the CA certificates to the container + for i, cert := range hsic.caCerts { + err = hsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert) + if err != nil { + return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) + } + } + err = hsic.WriteFile("/etc/headscale/config.yaml", []byte(MinimumConfigYAML())) if err != nil { return nil, fmt.Errorf("failed to write headscale config to container: %w", err) @@ -737,86 +783,3 @@ func (t *HeadscaleInContainer) SendInterrupt() error { return nil } - -// nolint -func createCertificate(hostname string) ([]byte, []byte, error) { - // From: - // https://shaneutt.com/blog/golang-ca-and-signed-cert-go/ - - ca := &x509.Certificate{ - SerialNumber: big.NewInt(2019), - Subject: pkix.Name{ - Organization: []string{"Headscale testing INC"}, - Country: []string{"NL"}, - Locality: []string{"Leiden"}, - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(60 * time.Hour), - IsCA: true, - ExtKeyUsage: []x509.ExtKeyUsage{ - x509.ExtKeyUsageClientAuth, - x509.ExtKeyUsageServerAuth, - }, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - BasicConstraintsValid: true, - } - - caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, nil, err - } - - cert := &x509.Certificate{ - SerialNumber: big.NewInt(1658), - Subject: pkix.Name{ - CommonName: hostname, - Organization: []string{"Headscale testing INC"}, - Country: []string{"NL"}, - Locality: []string{"Leiden"}, - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(60 * time.Minute), - SubjectKeyId: []byte{1, 2, 3, 4, 6}, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageDigitalSignature, - DNSNames: []string{hostname}, - } - - certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, nil, err - } - - certBytes, err := x509.CreateCertificate( - rand.Reader, - cert, - ca, - &certPrivKey.PublicKey, - caPrivKey, - ) - if err != nil { - return nil, nil, err - } - - certPEM := new(bytes.Buffer) - - err = pem.Encode(certPEM, &pem.Block{ - Type: "CERTIFICATE", - Bytes: certBytes, - }) - if err != nil { - return nil, nil, err - } - - certPrivKeyPEM := new(bytes.Buffer) - - err = pem.Encode(certPrivKeyPEM, &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), - }) - if err != nil { - return nil, nil, err - } - - return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil -} diff --git a/integration/integrationutil/util.go b/integration/integrationutil/util.go index 59eeeb17b4..7b9b63b593 100644 --- a/integration/integrationutil/util.go +++ b/integration/integrationutil/util.go @@ -3,9 +3,16 @@ package integrationutil import ( "archive/tar" "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "fmt" "io" + "math/big" "path/filepath" + "time" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/ory/dockertest/v3" @@ -93,3 +100,86 @@ func FetchPathFromContainer( return buf.Bytes(), nil } + +// nolint +func CreateCertificate(hostname string) ([]byte, []byte, error) { + // From: + // https://shaneutt.com/blog/golang-ca-and-signed-cert-go/ + + ca := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"Headscale testing INC"}, + Country: []string{"NL"}, + Locality: []string{"Leiden"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(60 * time.Hour), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + x509.ExtKeyUsageServerAuth, + }, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1658), + Subject: pkix.Name{ + CommonName: hostname, + Organization: []string{"Headscale testing INC"}, + Country: []string{"NL"}, + Locality: []string{"Leiden"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(60 * time.Minute), + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + DNSNames: []string{hostname}, + } + + certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + + certBytes, err := x509.CreateCertificate( + rand.Reader, + cert, + ca, + &certPrivKey.PublicKey, + caPrivKey, + ) + if err != nil { + return nil, nil, err + } + + certPEM := new(bytes.Buffer) + + err = pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + if err != nil { + return nil, nil, err + } + + certPrivKeyPEM := new(bytes.Buffer) + + err = pem.Encode(certPrivKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }) + if err != nil { + return nil, nil, err + } + + return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil +} diff --git a/integration/scenario.go b/integration/scenario.go index bd004247dc..1edd322c1c 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -13,6 +13,7 @@ import ( v1 "github.com/juanfont/headscale/gen/go/headscale/v1" "github.com/juanfont/headscale/hscontrol/util" "github.com/juanfont/headscale/integration/dockertestutil" + "github.com/juanfont/headscale/integration/dsic" "github.com/juanfont/headscale/integration/hsic" "github.com/juanfont/headscale/integration/tsic" "github.com/ory/dockertest/v3" @@ -134,6 +135,7 @@ type Scenario struct { // TODO(kradalby): support multiple headcales for later, currently only // use one. controlServers *xsync.MapOf[string, ControlServer] + derpServers []*dsic.DERPServerInContainer users map[string]*User @@ -212,6 +214,13 @@ func (s *Scenario) Shutdown() { } } + for _, derp := range s.derpServers { + err := derp.Shutdown() + if err != nil { + log.Printf("failed to tear down derp server: %s", err) + } + } + if err := s.pool.RemoveNetwork(s.network); err != nil { log.Printf("failed to remove network: %s", err) } @@ -328,7 +337,7 @@ func (s *Scenario) CreateTailscaleNodesInUser( hostname := headscale.GetHostname() opts = append(opts, - tsic.WithHeadscaleTLS(cert), + tsic.WithCACert(cert), tsic.WithHeadscaleName(hostname), ) @@ -631,3 +640,20 @@ func (s *Scenario) WaitForTailscaleLogout() error { return nil } + +// CreateDERPServer creates a new DERP server in a container. +func (s *Scenario) CreateDERPServer(version string, opts ...dsic.Option) (*dsic.DERPServerInContainer, error) { + derp, err := dsic.New(s.pool, version, s.network, opts...) + if err != nil { + return nil, fmt.Errorf("failed to create DERP server: %w", err) + } + + err = derp.WaitForRunning() + if err != nil { + return nil, fmt.Errorf("failed to reach DERP server: %w", err) + } + + s.derpServers = append(s.derpServers, derp) + + return derp, nil +} diff --git a/integration/tailscale.go b/integration/tailscale.go index 2ea3faa92a..02fbc41a90 100644 --- a/integration/tailscale.go +++ b/integration/tailscale.go @@ -29,6 +29,7 @@ type TailscaleClient interface { FQDN() (string, error) Status(...bool) (*ipnstate.Status, error) Netmap() (*netmap.NetworkMap, error) + DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error) Netcheck() (*netcheck.Report, error) WaitForNeedsLogin() error WaitForRunning() error diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 0e3c91f8d1..9107df3d07 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -30,7 +30,7 @@ const ( defaultPingTimeout = 300 * time.Millisecond defaultPingCount = 10 dockerContextPath = "../." - headscaleCertPath = "/usr/local/share/ca-certificates/headscale.crt" + caCertRoot = "/usr/local/share/ca-certificates" dockerExecuteTimeout = 60 * time.Second ) @@ -63,7 +63,7 @@ type TailscaleInContainer struct { fqdn string // optional config - headscaleCert []byte + caCerts [][]byte headscaleHostname string withSSH bool withTags []string @@ -77,11 +77,10 @@ type TailscaleInContainer struct { // Tailscale instance. type Option = func(c *TailscaleInContainer) -// WithHeadscaleTLS takes the certificate of the Headscale instance -// and adds it to the trusted surtificate of the Tailscale container. -func WithHeadscaleTLS(cert []byte) Option { +// WithCACert adds it to the trusted surtificate of the Tailscale container. +func WithCACert(cert []byte) Option { return func(tsic *TailscaleInContainer) { - tsic.headscaleCert = cert + tsic.caCerts = append(tsic.caCerts, cert) } } @@ -110,7 +109,7 @@ func WithOrCreateNetwork(network *dockertest.Network) Option { } // WithHeadscaleName set the name of the headscale instance, -// mostly useful in combination with TLS and WithHeadscaleTLS. +// mostly useful in combination with TLS and WithCACert. func WithHeadscaleName(hsName string) Option { return func(tsic *TailscaleInContainer) { tsic.headscaleHostname = hsName @@ -206,12 +205,8 @@ func New( ExtraHosts: tsic.withExtraHosts, } - if tsic.headscaleHostname != "" { - tailscaleOptions.ExtraHosts = []string{ - "host.docker.internal:host-gateway", - fmt.Sprintf("%s:host-gateway", tsic.headscaleHostname), - } - } + tailscaleOptions.ExtraHosts = append(tailscaleOptions.ExtraHosts, + "host.docker.internal:host-gateway") if tsic.workdir != "" { tailscaleOptions.WorkingDir = tsic.workdir @@ -275,8 +270,8 @@ func New( tsic.container = container - if tsic.hasTLS() { - err = tsic.WriteFile(headscaleCertPath, tsic.headscaleCert) + for i, cert := range tsic.caCerts { + err = tsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert) if err != nil { return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) } @@ -285,10 +280,6 @@ func New( return tsic, nil } -func (t *TailscaleInContainer) hasTLS() bool { - return len(t.headscaleCert) != 0 -} - // Shutdown stops and cleans up the Tailscale container. func (t *TailscaleInContainer) Shutdown() error { err := t.SaveLog("/tmp/control") @@ -654,6 +645,33 @@ func (t *TailscaleInContainer) watchIPN(ctx context.Context) (*ipn.Notify, error } } +func (t *TailscaleInContainer) DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error) { + if !util.TailscaleVersionNewerOrEqual("1.34", t.version) { + panic(fmt.Sprintf("tsic.DebugDERPRegion() called with unsupported version: %s", t.version)) + } + + command := []string{ + "tailscale", + "debug", + "derp", + region, + } + + result, stderr, err := t.Execute(command) + if err != nil { + fmt.Printf("stderr: %s\n", stderr) + return nil, fmt.Errorf("failed to execute tailscale debug derp command: %w", err) + } + + var st ipnstate.DebugDERPRegionReport + err = json.Unmarshal([]byte(result), &st) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal tailscale derp region report: %w", err) + } + + return &st, err +} + // Netcheck returns the current Netcheck Report (netcheck.Report) of the Tailscale instance. func (t *TailscaleInContainer) Netcheck() (*netcheck.Report, error) { command := []string{