From edf9e250017708e218895c4524a4477ec7a6dcba Mon Sep 17 00:00:00 2001 From: ArcticLampyrid Date: Fri, 22 Nov 2024 20:23:05 +0800 Subject: [PATCH] feat: support client verify for derp (add integration tests) (#2046) * feat: support client verify for derp * docs: fix doc for integration test * tests: add integration test for DERP verify endpoint * tests: use `tailcfg.DERPMap` instead of `[]byte` * refactor: introduce func `ContainsNodeKey` * tests(dsic): use string builder for cmd args * ci: fix tests order * tests: fix derper failure * chore: cleanup * tests(verify-client): perfer to use `CreateHeadscaleEnv` * refactor(verify-client): simplify error handling * tests: fix `TestDERPVerifyEndpoint` * refactor: make `doVerify` a seperated func --------- Co-authored-by: 117503445 --- .github/workflows/test-integration.yaml | 1 + Dockerfile.derper | 19 ++ hscontrol/app.go | 2 + hscontrol/handlers.go | 60 +++++ hscontrol/types/node.go | 10 + integration/README.md | 4 +- integration/derp_verify_endpoint_test.go | 96 +++++++ integration/dsic/dsic.go | 321 +++++++++++++++++++++++ integration/embedded_derp_test.go | 2 +- integration/hsic/hsic.go | 162 +++++------- integration/integrationutil/util.go | 90 +++++++ integration/scenario.go | 28 +- integration/tailscale.go | 1 + integration/tsic/tsic.go | 57 ++-- 14 files changed, 735 insertions(+), 118 deletions(-) create mode 100644 Dockerfile.derper create mode 100644 integration/derp_verify_endpoint_test.go create mode 100644 integration/dsic/dsic.go diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index 65324f77..7e730aa8 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -38,6 +38,7 @@ jobs: - TestNodeMoveCommand - TestPolicyCommand - TestPolicyBrokenConfigCommand + - TestDERPVerifyEndpoint - TestResolveMagicDNS - TestValidateResolvConf - TestDERPServerScenario diff --git a/Dockerfile.derper b/Dockerfile.derper new file mode 100644 index 00000000..62adc7cf --- /dev/null +++ b/Dockerfile.derper @@ -0,0 +1,19 @@ +# For testing purposes only + +FROM golang:alpine AS build-env + +WORKDIR /go/src + +RUN apk add --no-cache git +ARG VERSION_BRANCH=main +RUN git clone https://github.com/tailscale/tailscale.git --branch=$VERSION_BRANCH --depth=1 +WORKDIR /go/src/tailscale + +ARG TARGETARCH +RUN GOARCH=$TARGETARCH go install -v ./cmd/derper + +FROM alpine:3.18 +RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables curl + +COPY --from=build-env /go/bin/* /usr/local/bin/ +ENTRYPOINT [ "/usr/local/bin/derper" ] diff --git a/hscontrol/app.go b/hscontrol/app.go index 737e8098..da20b1ae 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -457,6 +457,8 @@ func (h *Headscale) createRouter(grpcMux *grpcRuntime.ServeMux) *mux.Router { router.HandleFunc("/swagger/v1/openapiv2.json", headscale.SwaggerAPIv1). Methods(http.MethodGet) + router.HandleFunc("/verify", h.VerifyHandler).Methods(http.MethodPost) + if h.cfg.DERP.ServerEnabled { router.HandleFunc("/derp", h.DERPServer.DERPHandler) router.HandleFunc("/derp/probe", derpServer.DERPProbeHandler) diff --git a/hscontrol/handlers.go b/hscontrol/handlers.go index 72ec4e42..3858df93 100644 --- a/hscontrol/handlers.go +++ b/hscontrol/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "strconv" "strings" @@ -56,6 +57,65 @@ func parseCabailityVersion(req *http.Request) (tailcfg.CapabilityVersion, error) return tailcfg.CapabilityVersion(clientCapabilityVersion), nil } +func (h *Headscale) handleVerifyRequest( + req *http.Request, +) (bool, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return false, fmt.Errorf("cannot read request body: %w", err) + } + + var derpAdmitClientRequest tailcfg.DERPAdmitClientRequest + if err := json.Unmarshal(body, &derpAdmitClientRequest); err != nil { + return false, fmt.Errorf("cannot parse derpAdmitClientRequest: %w", err) + } + + nodes, err := h.db.ListNodes() + if err != nil { + return false, fmt.Errorf("cannot list nodes: %w", err) + } + + return nodes.ContainsNodeKey(derpAdmitClientRequest.NodePublic), nil +} + +// see https://github.com/tailscale/tailscale/blob/964282d34f06ecc06ce644769c66b0b31d118340/derp/derp_server.go#L1159, Derp use verifyClientsURL to verify whether a client is allowed to connect to the DERP server. +func (h *Headscale) VerifyHandler( + writer http.ResponseWriter, + req *http.Request, +) { + if req.Method != http.MethodPost { + http.Error(writer, "Wrong method", http.StatusMethodNotAllowed) + + return + } + log.Debug(). + Str("handler", "/verify"). + Msg("verify client") + + allow, err := h.handleVerifyRequest(req) + if err != nil { + log.Error(). + Caller(). + Err(err). + Msg("Failed to verify client") + http.Error(writer, "Internal error", http.StatusInternalServerError) + } + + resp := tailcfg.DERPAdmitClientResponse{ + Allow: allow, + } + + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusOK) + err = json.NewEncoder(writer).Encode(resp) + if err != nil { + log.Error(). + Caller(). + Err(err). + Msg("Failed to write response") + } +} + // KeyHandler provides the Headscale pub key // Listens in /key. func (h *Headscale) KeyHandler( diff --git a/hscontrol/types/node.go b/hscontrol/types/node.go index 9d632bd8..36a65062 100644 --- a/hscontrol/types/node.go +++ b/hscontrol/types/node.go @@ -223,6 +223,16 @@ func (nodes Nodes) FilterByIP(ip netip.Addr) Nodes { return found } +func (nodes Nodes) ContainsNodeKey(nodeKey key.NodePublic) bool { + for _, node := range nodes { + if node.NodeKey == nodeKey { + return true + } + } + + return false +} + func (node *Node) Proto() *v1.Node { nodeProto := &v1.Node{ Id: uint64(node.ID), diff --git a/integration/README.md b/integration/README.md index e5676a44..56247c52 100644 --- a/integration/README.md +++ b/integration/README.md @@ -11,10 +11,10 @@ Tests are located in files ending with `_test.go` and the framework are located ## Running integration tests locally -The easiest way to run tests locally is to use `[act](INSERT LINK)`, a local GitHub Actions runner: +The easiest way to run tests locally is to use [act](https://github.com/nektos/act), a local GitHub Actions runner: ``` -act pull_request -W .github/workflows/test-integration-v2-TestPingAllByIP.yaml +act pull_request -W .github/workflows/test-integration.yaml ``` Alternatively, the `docker run` command in each GitHub workflow file can be used. diff --git a/integration/derp_verify_endpoint_test.go b/integration/derp_verify_endpoint_test.go new file mode 100644 index 00000000..adad5b6a --- /dev/null +++ b/integration/derp_verify_endpoint_test.go @@ -0,0 +1,96 @@ +package integration + +import ( + "encoding/json" + "fmt" + "net" + "strconv" + "strings" + "testing" + + "github.com/juanfont/headscale/hscontrol/util" + "github.com/juanfont/headscale/integration/dsic" + "github.com/juanfont/headscale/integration/hsic" + "github.com/juanfont/headscale/integration/integrationutil" + "github.com/juanfont/headscale/integration/tsic" + "tailscale.com/tailcfg" +) + +func TestDERPVerifyEndpoint(t *testing.T) { + IntegrationSkip(t) + + // Generate random hostname for the headscale instance + hash, err := util.GenerateRandomStringDNSSafe(6) + assertNoErr(t, err) + testName := "derpverify" + hostname := fmt.Sprintf("hs-%s-%s", testName, hash) + + headscalePort := 8080 + + // Create cert for headscale + certHeadscale, keyHeadscale, err := integrationutil.CreateCertificate(hostname) + assertNoErr(t, err) + + scenario, err := NewScenario(dockertestMaxWait()) + assertNoErr(t, err) + defer scenario.ShutdownAssertNoPanics(t) + + spec := map[string]int{ + "user1": len(MustTestVersions), + } + + derper, err := scenario.CreateDERPServer("head", + dsic.WithCACert(certHeadscale), + dsic.WithVerifyClientURL(fmt.Sprintf("https://%s/verify", net.JoinHostPort(hostname, strconv.Itoa(headscalePort)))), + ) + assertNoErr(t, err) + + derpMap := tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 900: { + RegionID: 900, + RegionCode: "test-derpverify", + RegionName: "TestDerpVerify", + Nodes: []*tailcfg.DERPNode{ + { + Name: "TestDerpVerify", + RegionID: 900, + HostName: derper.GetHostname(), + STUNPort: derper.GetSTUNPort(), + STUNOnly: false, + DERPPort: derper.GetDERPPort(), + }, + }, + }, + }, + } + + err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{tsic.WithCACert(derper.GetCert())}, + hsic.WithHostname(hostname), + hsic.WithPort(headscalePort), + hsic.WithCustomTLS(certHeadscale, keyHeadscale), + hsic.WithHostnameAsServerURL(), + hsic.WithDERPConfig(derpMap)) + assertNoErrHeadscaleEnv(t, err) + + allClients, err := scenario.ListTailscaleClients() + assertNoErrListClients(t, err) + + for _, client := range allClients { + report, err := client.DebugDERPRegion("test-derpverify") + assertNoErr(t, err) + successful := false + for _, line := range report.Info { + if strings.Contains(line, "Successfully established a DERP connection with node") { + successful = true + + break + } + } + if !successful { + stJSON, err := json.Marshal(report) + assertNoErr(t, err) + t.Errorf("Client %s could not establish a DERP connection: %s", client.Hostname(), string(stJSON)) + } + } +} diff --git a/integration/dsic/dsic.go b/integration/dsic/dsic.go new file mode 100644 index 00000000..f8bb85a9 --- /dev/null +++ b/integration/dsic/dsic.go @@ -0,0 +1,321 @@ +package dsic + +import ( + "crypto/tls" + "errors" + "fmt" + "log" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/juanfont/headscale/hscontrol/util" + "github.com/juanfont/headscale/integration/dockertestutil" + "github.com/juanfont/headscale/integration/integrationutil" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" +) + +const ( + dsicHashLength = 6 + dockerContextPath = "../." + caCertRoot = "/usr/local/share/ca-certificates" + DERPerCertRoot = "/usr/local/share/derper-certs" + dockerExecuteTimeout = 60 * time.Second +) + +var errDERPerStatusCodeNotOk = errors.New("DERPer status code not OK") + +// DERPServerInContainer represents DERP Server in Container (DSIC). +type DERPServerInContainer struct { + version string + hostname string + + pool *dockertest.Pool + container *dockertest.Resource + network *dockertest.Network + + stunPort int + derpPort int + caCerts [][]byte + tlsCert []byte + tlsKey []byte + withExtraHosts []string + withVerifyClientURL string + workdir string +} + +// Option represent optional settings that can be given to a +// DERPer instance. +type Option = func(c *DERPServerInContainer) + +// WithCACert adds it to the trusted surtificate of the Tailscale container. +func WithCACert(cert []byte) Option { + return func(dsic *DERPServerInContainer) { + dsic.caCerts = append(dsic.caCerts, cert) + } +} + +// WithOrCreateNetwork sets the Docker container network to use with +// the DERPer instance, if the parameter is nil, a new network, +// isolating the DERPer, will be created. If a network is +// passed, the DERPer instance will join the given network. +func WithOrCreateNetwork(network *dockertest.Network) Option { + return func(tsic *DERPServerInContainer) { + if network != nil { + tsic.network = network + + return + } + + network, err := dockertestutil.GetFirstOrCreateNetwork( + tsic.pool, + tsic.hostname+"-network", + ) + if err != nil { + log.Fatalf("failed to create network: %s", err) + } + + tsic.network = network + } +} + +// WithDockerWorkdir allows the docker working directory to be set. +func WithDockerWorkdir(dir string) Option { + return func(tsic *DERPServerInContainer) { + tsic.workdir = dir + } +} + +// WithVerifyClientURL sets the URL to verify the client. +func WithVerifyClientURL(url string) Option { + return func(tsic *DERPServerInContainer) { + tsic.withVerifyClientURL = url + } +} + +// WithExtraHosts adds extra hosts to the container. +func WithExtraHosts(hosts []string) Option { + return func(tsic *DERPServerInContainer) { + tsic.withExtraHosts = hosts + } +} + +// New returns a new TailscaleInContainer instance. +func New( + pool *dockertest.Pool, + version string, + network *dockertest.Network, + opts ...Option, +) (*DERPServerInContainer, error) { + hash, err := util.GenerateRandomStringDNSSafe(dsicHashLength) + if err != nil { + return nil, err + } + + hostname := fmt.Sprintf("derp-%s-%s", strings.ReplaceAll(version, ".", "-"), hash) + tlsCert, tlsKey, err := integrationutil.CreateCertificate(hostname) + if err != nil { + return nil, fmt.Errorf("failed to create certificates for headscale test: %w", err) + } + dsic := &DERPServerInContainer{ + version: version, + hostname: hostname, + pool: pool, + network: network, + tlsCert: tlsCert, + tlsKey: tlsKey, + stunPort: 3478, //nolint + derpPort: 443, //nolint + } + + for _, opt := range opts { + opt(dsic) + } + + var cmdArgs strings.Builder + fmt.Fprintf(&cmdArgs, "--hostname=%s", hostname) + fmt.Fprintf(&cmdArgs, " --certmode=manual") + fmt.Fprintf(&cmdArgs, " --certdir=%s", DERPerCertRoot) + fmt.Fprintf(&cmdArgs, " --a=:%d", dsic.derpPort) + fmt.Fprintf(&cmdArgs, " --stun=true") + fmt.Fprintf(&cmdArgs, " --stun-port=%d", dsic.stunPort) + if dsic.withVerifyClientURL != "" { + fmt.Fprintf(&cmdArgs, " --verify-client-url=%s", dsic.withVerifyClientURL) + } + + runOptions := &dockertest.RunOptions{ + Name: hostname, + Networks: []*dockertest.Network{dsic.network}, + ExtraHosts: dsic.withExtraHosts, + // we currently need to give us some time to inject the certificate further down. + Entrypoint: []string{"/bin/sh", "-c", "/bin/sleep 3 ; update-ca-certificates ; derper " + cmdArgs.String()}, + ExposedPorts: []string{ + "80/tcp", + fmt.Sprintf("%d/tcp", dsic.derpPort), + fmt.Sprintf("%d/udp", dsic.stunPort), + }, + } + + if dsic.workdir != "" { + runOptions.WorkingDir = dsic.workdir + } + + // dockertest isnt very good at handling containers that has already + // been created, this is an attempt to make sure this container isnt + // present. + err = pool.RemoveContainerByName(hostname) + if err != nil { + return nil, err + } + + var container *dockertest.Resource + buildOptions := &dockertest.BuildOptions{ + Dockerfile: "Dockerfile.derper", + ContextDir: dockerContextPath, + BuildArgs: []docker.BuildArg{}, + } + switch version { + case "head": + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "main", + }) + default: + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{ + Name: "VERSION_BRANCH", + Value: "v" + version, + }) + } + container, err = pool.BuildAndRunWithBuildOptions( + buildOptions, + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + if err != nil { + return nil, fmt.Errorf( + "%s could not start tailscale DERPer container (version: %s): %w", + hostname, + version, + err, + ) + } + log.Printf("Created %s container\n", hostname) + + dsic.container = container + + for i, cert := range dsic.caCerts { + err = dsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert) + if err != nil { + return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) + } + } + if len(dsic.tlsCert) != 0 { + err = dsic.WriteFile(fmt.Sprintf("%s/%s.crt", DERPerCertRoot, dsic.hostname), dsic.tlsCert) + if err != nil { + return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) + } + } + if len(dsic.tlsKey) != 0 { + err = dsic.WriteFile(fmt.Sprintf("%s/%s.key", DERPerCertRoot, dsic.hostname), dsic.tlsKey) + if err != nil { + return nil, fmt.Errorf("failed to write TLS key to container: %w", err) + } + } + + return dsic, nil +} + +// Shutdown stops and cleans up the DERPer container. +func (t *DERPServerInContainer) Shutdown() error { + err := t.SaveLog("/tmp/control") + if err != nil { + log.Printf( + "Failed to save log from %s: %s", + t.hostname, + fmt.Errorf("failed to save log: %w", err), + ) + } + + return t.pool.Purge(t.container) +} + +// GetCert returns the TLS certificate of the DERPer instance. +func (t *DERPServerInContainer) GetCert() []byte { + return t.tlsCert +} + +// Hostname returns the hostname of the DERPer instance. +func (t *DERPServerInContainer) Hostname() string { + return t.hostname +} + +// Version returns the running DERPer version of the instance. +func (t *DERPServerInContainer) Version() string { + return t.version +} + +// ID returns the Docker container ID of the DERPServerInContainer +// instance. +func (t *DERPServerInContainer) ID() string { + return t.container.Container.ID +} + +func (t *DERPServerInContainer) GetHostname() string { + return t.hostname +} + +// GetSTUNPort returns the STUN port of the DERPer instance. +func (t *DERPServerInContainer) GetSTUNPort() int { + return t.stunPort +} + +// GetDERPPort returns the DERP port of the DERPer instance. +func (t *DERPServerInContainer) GetDERPPort() int { + return t.derpPort +} + +// WaitForRunning blocks until the DERPer instance is ready to be used. +func (t *DERPServerInContainer) WaitForRunning() error { + url := "https://" + net.JoinHostPort(t.GetHostname(), strconv.Itoa(t.GetDERPPort())) + "/" + log.Printf("waiting for DERPer to be ready at %s", url) + + insecureTransport := http.DefaultTransport.(*http.Transport).Clone() //nolint + insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint + client := &http.Client{Transport: insecureTransport} + + return t.pool.Retry(func() error { + resp, err := client.Get(url) //nolint + if err != nil { + return fmt.Errorf("headscale is not ready: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return errDERPerStatusCodeNotOk + } + + return nil + }) +} + +// ConnectToNetwork connects the DERPer instance to a network. +func (t *DERPServerInContainer) ConnectToNetwork(network *dockertest.Network) error { + return t.container.ConnectToNetwork(network) +} + +// WriteFile save file inside the container. +func (t *DERPServerInContainer) WriteFile(path string, data []byte) error { + return integrationutil.WriteFileToContainer(t.pool, t.container, path, data) +} + +// SaveLog saves the current stdout log of the container to a path +// on the host system. +func (t *DERPServerInContainer) SaveLog(path string) error { + _, _, err := dockertestutil.SaveLog(t.pool, t.container, path) + + return err +} diff --git a/integration/embedded_derp_test.go b/integration/embedded_derp_test.go index 046f3890..d5fdb161 100644 --- a/integration/embedded_derp_test.go +++ b/integration/embedded_derp_test.go @@ -310,7 +310,7 @@ func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser( cert := hsServer.GetCert() opts = append(opts, - tsic.WithHeadscaleTLS(cert), + tsic.WithCACert(cert), ) user.createWaitGroup.Go(func() error { diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index c2ae3336..8c379dc8 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -1,19 +1,12 @@ package hsic import ( - "bytes" - "crypto/rand" - "crypto/rsa" "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" "encoding/json" - "encoding/pem" "errors" "fmt" "io" "log" - "math/big" "net" "net/http" "net/url" @@ -32,11 +25,14 @@ import ( "github.com/juanfont/headscale/integration/integrationutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" + "gopkg.in/yaml.v3" + "tailscale.com/tailcfg" ) const ( hsicHashLength = 6 dockerContextPath = "../." + caCertRoot = "/usr/local/share/ca-certificates" aclPolicyPath = "/etc/headscale/acl.hujson" tlsCertPath = "/etc/headscale/tls.cert" tlsKeyPath = "/etc/headscale/tls.key" @@ -64,6 +60,7 @@ type HeadscaleInContainer struct { // optional config port int extraPorts []string + caCerts [][]byte hostPortBindings map[string][]string aclPolicy *policy.ACLPolicy env map[string]string @@ -88,18 +85,29 @@ func WithACLPolicy(acl *policy.ACLPolicy) Option { } } +// WithCACert adds it to the trusted surtificate of the container. +func WithCACert(cert []byte) Option { + return func(hsic *HeadscaleInContainer) { + hsic.caCerts = append(hsic.caCerts, cert) + } +} + // WithTLS creates certificates and enables HTTPS. func WithTLS() Option { return func(hsic *HeadscaleInContainer) { - cert, key, err := createCertificate(hsic.hostname) + cert, key, err := integrationutil.CreateCertificate(hsic.hostname) if err != nil { log.Fatalf("failed to create certificates for headscale test: %s", err) } - // TODO(kradalby): Move somewhere appropriate - hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath - hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath + hsic.tlsCert = cert + hsic.tlsKey = key + } +} +// WithCustomTLS uses the given certificates for the Headscale instance. +func WithCustomTLS(cert, key []byte) Option { + return func(hsic *HeadscaleInContainer) { hsic.tlsCert = cert hsic.tlsKey = key } @@ -146,6 +154,13 @@ func WithTestName(testName string) Option { } } +// WithHostname sets the hostname of the Headscale instance. +func WithHostname(hostname string) Option { + return func(hsic *HeadscaleInContainer) { + hsic.hostname = hostname + } +} + // WithHostnameAsServerURL sets the Headscale ServerURL based on // the Hostname. func WithHostnameAsServerURL() Option { @@ -203,6 +218,34 @@ func WithEmbeddedDERPServerOnly() Option { } } +// WithDERPConfig configures Headscale use a custom +// DERP server only. +func WithDERPConfig(derpMap tailcfg.DERPMap) Option { + return func(hsic *HeadscaleInContainer) { + contents, err := yaml.Marshal(derpMap) + if err != nil { + log.Fatalf("failed to marshal DERP map: %s", err) + + return + } + + hsic.env["HEADSCALE_DERP_PATHS"] = "/etc/headscale/derp.yml" + hsic.filesInContainer = append(hsic.filesInContainer, + fileInContainer{ + path: "/etc/headscale/derp.yml", + contents: contents, + }) + + // Disable global DERP server and embedded DERP server + hsic.env["HEADSCALE_DERP_URLS"] = "" + hsic.env["HEADSCALE_DERP_SERVER_ENABLED"] = "false" + + // Envknob for enabling DERP debug logs + hsic.env["DERP_DEBUG_LOGS"] = "true" + hsic.env["DERP_PROBER_DEBUG_LOGS"] = "true" + } +} + // WithTuning allows changing the tuning settings easily. func WithTuning(batchTimeout time.Duration, mapSessionChanSize int) Option { return func(hsic *HeadscaleInContainer) { @@ -300,6 +343,10 @@ func New( "HEADSCALE_DEBUG_HIGH_CARDINALITY_METRICS=1", "HEADSCALE_DEBUG_DUMP_CONFIG=1", } + if hsic.hasTLS() { + hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath + hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath + } for key, value := range hsic.env { env = append(env, fmt.Sprintf("%s=%s", key, value)) } @@ -313,7 +360,7 @@ func New( // Cmd: []string{"headscale", "serve"}, // TODO(kradalby): Get rid of this hack, we currently need to give us some // to inject the headscale configuration further down. - Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve ; /bin/sleep 30"}, + Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; update-ca-certificates ; headscale serve ; /bin/sleep 30"}, Env: env, } @@ -351,6 +398,14 @@ func New( hsic.container = container + // Write the CA certificates to the container + for i, cert := range hsic.caCerts { + err = hsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert) + if err != nil { + return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) + } + } + err = hsic.WriteFile("/etc/headscale/config.yaml", []byte(MinimumConfigYAML())) if err != nil { return nil, fmt.Errorf("failed to write headscale config to container: %w", err) @@ -749,86 +804,3 @@ func (t *HeadscaleInContainer) SendInterrupt() error { return nil } - -// nolint -func createCertificate(hostname string) ([]byte, []byte, error) { - // From: - // https://shaneutt.com/blog/golang-ca-and-signed-cert-go/ - - ca := &x509.Certificate{ - SerialNumber: big.NewInt(2019), - Subject: pkix.Name{ - Organization: []string{"Headscale testing INC"}, - Country: []string{"NL"}, - Locality: []string{"Leiden"}, - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(60 * time.Hour), - IsCA: true, - ExtKeyUsage: []x509.ExtKeyUsage{ - x509.ExtKeyUsageClientAuth, - x509.ExtKeyUsageServerAuth, - }, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - BasicConstraintsValid: true, - } - - caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, nil, err - } - - cert := &x509.Certificate{ - SerialNumber: big.NewInt(1658), - Subject: pkix.Name{ - CommonName: hostname, - Organization: []string{"Headscale testing INC"}, - Country: []string{"NL"}, - Locality: []string{"Leiden"}, - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(60 * time.Minute), - SubjectKeyId: []byte{1, 2, 3, 4, 6}, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageDigitalSignature, - DNSNames: []string{hostname}, - } - - certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, nil, err - } - - certBytes, err := x509.CreateCertificate( - rand.Reader, - cert, - ca, - &certPrivKey.PublicKey, - caPrivKey, - ) - if err != nil { - return nil, nil, err - } - - certPEM := new(bytes.Buffer) - - err = pem.Encode(certPEM, &pem.Block{ - Type: "CERTIFICATE", - Bytes: certBytes, - }) - if err != nil { - return nil, nil, err - } - - certPrivKeyPEM := new(bytes.Buffer) - - err = pem.Encode(certPrivKeyPEM, &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), - }) - if err != nil { - return nil, nil, err - } - - return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil -} diff --git a/integration/integrationutil/util.go b/integration/integrationutil/util.go index 59eeeb17..7b9b63b5 100644 --- a/integration/integrationutil/util.go +++ b/integration/integrationutil/util.go @@ -3,9 +3,16 @@ package integrationutil import ( "archive/tar" "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "fmt" "io" + "math/big" "path/filepath" + "time" "github.com/juanfont/headscale/integration/dockertestutil" "github.com/ory/dockertest/v3" @@ -93,3 +100,86 @@ func FetchPathFromContainer( return buf.Bytes(), nil } + +// nolint +func CreateCertificate(hostname string) ([]byte, []byte, error) { + // From: + // https://shaneutt.com/blog/golang-ca-and-signed-cert-go/ + + ca := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"Headscale testing INC"}, + Country: []string{"NL"}, + Locality: []string{"Leiden"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(60 * time.Hour), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + x509.ExtKeyUsageServerAuth, + }, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1658), + Subject: pkix.Name{ + CommonName: hostname, + Organization: []string{"Headscale testing INC"}, + Country: []string{"NL"}, + Locality: []string{"Leiden"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(60 * time.Minute), + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + DNSNames: []string{hostname}, + } + + certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + + certBytes, err := x509.CreateCertificate( + rand.Reader, + cert, + ca, + &certPrivKey.PublicKey, + caPrivKey, + ) + if err != nil { + return nil, nil, err + } + + certPEM := new(bytes.Buffer) + + err = pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + if err != nil { + return nil, nil, err + } + + certPrivKeyPEM := new(bytes.Buffer) + + err = pem.Encode(certPrivKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }) + if err != nil { + return nil, nil, err + } + + return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil +} diff --git a/integration/scenario.go b/integration/scenario.go index b45c5fe7..31686fac 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -14,6 +14,7 @@ import ( v1 "github.com/juanfont/headscale/gen/go/headscale/v1" "github.com/juanfont/headscale/hscontrol/util" "github.com/juanfont/headscale/integration/dockertestutil" + "github.com/juanfont/headscale/integration/dsic" "github.com/juanfont/headscale/integration/hsic" "github.com/juanfont/headscale/integration/tsic" "github.com/ory/dockertest/v3" @@ -140,6 +141,7 @@ type Scenario struct { // TODO(kradalby): support multiple headcales for later, currently only // use one. controlServers *xsync.MapOf[string, ControlServer] + derpServers []*dsic.DERPServerInContainer users map[string]*User @@ -224,6 +226,13 @@ func (s *Scenario) ShutdownAssertNoPanics(t *testing.T) { } } + for _, derp := range s.derpServers { + err := derp.Shutdown() + if err != nil { + log.Printf("failed to tear down derp server: %s", err) + } + } + if err := s.pool.RemoveNetwork(s.network); err != nil { log.Printf("failed to remove network: %s", err) } @@ -352,7 +361,7 @@ func (s *Scenario) CreateTailscaleNodesInUser( hostname := headscale.GetHostname() opts = append(opts, - tsic.WithHeadscaleTLS(cert), + tsic.WithCACert(cert), tsic.WithHeadscaleName(hostname), ) @@ -651,3 +660,20 @@ func (s *Scenario) WaitForTailscaleLogout() error { return nil } + +// CreateDERPServer creates a new DERP server in a container. +func (s *Scenario) CreateDERPServer(version string, opts ...dsic.Option) (*dsic.DERPServerInContainer, error) { + derp, err := dsic.New(s.pool, version, s.network, opts...) + if err != nil { + return nil, fmt.Errorf("failed to create DERP server: %w", err) + } + + err = derp.WaitForRunning() + if err != nil { + return nil, fmt.Errorf("failed to reach DERP server: %w", err) + } + + s.derpServers = append(s.derpServers, derp) + + return derp, nil +} diff --git a/integration/tailscale.go b/integration/tailscale.go index f858d2c2..66cc1ca3 100644 --- a/integration/tailscale.go +++ b/integration/tailscale.go @@ -30,6 +30,7 @@ type TailscaleClient interface { FQDN() (string, error) Status(...bool) (*ipnstate.Status, error) Netmap() (*netmap.NetworkMap, error) + DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error) Netcheck() (*netcheck.Report, error) WaitForNeedsLogin() error WaitForRunning() error diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index b0bd7a60..023cc430 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -33,7 +33,7 @@ const ( defaultPingTimeout = 300 * time.Millisecond defaultPingCount = 10 dockerContextPath = "../." - headscaleCertPath = "/usr/local/share/ca-certificates/headscale.crt" + caCertRoot = "/usr/local/share/ca-certificates" dockerExecuteTimeout = 60 * time.Second ) @@ -71,7 +71,7 @@ type TailscaleInContainer struct { fqdn string // optional config - headscaleCert []byte + caCerts [][]byte headscaleHostname string withWebsocketDERP bool withSSH bool @@ -93,11 +93,10 @@ type TailscaleInContainerBuildConfig struct { // Tailscale instance. type Option = func(c *TailscaleInContainer) -// WithHeadscaleTLS takes the certificate of the Headscale instance -// and adds it to the trusted surtificate of the Tailscale container. -func WithHeadscaleTLS(cert []byte) Option { +// WithCACert adds it to the trusted surtificate of the Tailscale container. +func WithCACert(cert []byte) Option { return func(tsic *TailscaleInContainer) { - tsic.headscaleCert = cert + tsic.caCerts = append(tsic.caCerts, cert) } } @@ -126,7 +125,7 @@ func WithOrCreateNetwork(network *dockertest.Network) Option { } // WithHeadscaleName set the name of the headscale instance, -// mostly useful in combination with TLS and WithHeadscaleTLS. +// mostly useful in combination with TLS and WithCACert. func WithHeadscaleName(hsName string) Option { return func(tsic *TailscaleInContainer) { tsic.headscaleHostname = hsName @@ -260,12 +259,8 @@ func New( ) } - if tsic.headscaleHostname != "" { - tailscaleOptions.ExtraHosts = []string{ - "host.docker.internal:host-gateway", - fmt.Sprintf("%s:host-gateway", tsic.headscaleHostname), - } - } + tailscaleOptions.ExtraHosts = append(tailscaleOptions.ExtraHosts, + "host.docker.internal:host-gateway") if tsic.workdir != "" { tailscaleOptions.WorkingDir = tsic.workdir @@ -351,8 +346,8 @@ func New( tsic.container = container - if tsic.hasTLS() { - err = tsic.WriteFile(headscaleCertPath, tsic.headscaleCert) + for i, cert := range tsic.caCerts { + err = tsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert) if err != nil { return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) } @@ -361,10 +356,6 @@ func New( return tsic, nil } -func (t *TailscaleInContainer) hasTLS() bool { - return len(t.headscaleCert) != 0 -} - // Shutdown stops and cleans up the Tailscale container. func (t *TailscaleInContainer) Shutdown() error { err := t.SaveLog("/tmp/control") @@ -739,6 +730,34 @@ func (t *TailscaleInContainer) watchIPN(ctx context.Context) (*ipn.Notify, error } } +func (t *TailscaleInContainer) DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error) { + if !util.TailscaleVersionNewerOrEqual("1.34", t.version) { + panic("tsic.DebugDERPRegion() called with unsupported version: " + t.version) + } + + command := []string{ + "tailscale", + "debug", + "derp", + region, + } + + result, stderr, err := t.Execute(command) + if err != nil { + fmt.Printf("stderr: %s\n", stderr) // nolint + + return nil, fmt.Errorf("failed to execute tailscale debug derp command: %w", err) + } + + var report ipnstate.DebugDERPRegionReport + err = json.Unmarshal([]byte(result), &report) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal tailscale derp region report: %w", err) + } + + return &report, err +} + // Netcheck returns the current Netcheck Report (netcheck.Report) of the Tailscale instance. func (t *TailscaleInContainer) Netcheck() (*netcheck.Report, error) { command := []string{