tailscale/release/dist/dist.go

// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause

// Package dist is a release artifact builder library.
package dist

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	"tailscale.com/util/multierr"
	"tailscale.com/version/mkversion"
)

// A Target is something that can be build in a Build.
type Target interface {
	String() string
	Build(build *Build) ([]string, error)
}

// Signer is pluggable signer for a Target.
type Signer func(io.Reader) ([]byte, error)

// SignFile signs the file at filePath with s and writes the signature to
// sigPath.
func (s Signer) SignFile(filePath, sigPath string) error {
	f, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer f.Close()
	sig, err := s(f)
	if err != nil {
		return err
	}
	return os.WriteFile(sigPath, sig, 0644)
}

// A Build is a build context for Targets.
type Build struct {
	// Repo is a path to the root Go module for the build.
	Repo string
	// Out is where build artifacts are written.
	Out string
	// Verbose is whether to print all command output, rather than just failed
	// commands.
	Verbose bool
	// WebClientSource is a path to the source for the web client.
	// If non-empty, web client assets will be built.
	WebClientSource string

	// Tmp is a temporary directory that gets deleted when the Builder is closed.
	Tmp string
	// Go is the path to the Go binary to use for building.
	Go string
	// Yarn is the path to the yarn binary to use for building the web client assets.
	Yarn string
	// Version is the version info of the build.
	Version mkversion.VersionInfo
	// Time is the timestamp of the build.
	Time time.Time

	// once is a cache of function invocations that should run once per process
	// (for example building a helper docker container)
	once once

	extraMu sync.Mutex
	extra   map[any]any

	goBuilds Memoize[string]
	// When running `dist build all` on a cold Go build cache, the fanout of
	// gooses and goarches results in a very large number of compile processes,
	// which bogs down the build machine.
	//
	// This throttles the number of concurrent `go build` invocations to the
	// number of CPU cores, which empirically keeps the builder responsive
	// without impacting overall build time.
	goBuildLimit chan struct{}

	onCloseFuncs []func() error // funcs to be called when Builder is closed
}

// NewBuild creates a new Build rooted at repo, and writing artifacts to out.
func NewBuild(repo, out string) (*Build, error) {
	if err := os.MkdirAll(out, 0750); err != nil {
		return nil, fmt.Errorf("creating out dir: %w", err)
	}
	tmp, err := os.MkdirTemp("", "dist-*")
	if err != nil {
		return nil, fmt.Errorf("creating tempdir: %w", err)
	}
	repo, err = findModRoot(repo)
	if err != nil {
		return nil, fmt.Errorf("finding module root: %w", err)
	}
	goTool, err := findTool(repo, "go")
	if err != nil {
		return nil, fmt.Errorf("finding go binary: %w", err)
	}
	yarnTool, err := findTool(repo, "yarn")
	if err != nil {
		return nil, fmt.Errorf("finding yarn binary: %w", err)
	}
	b := &Build{
		Repo:         repo,
		Tmp:          tmp,
		Out:          out,
		Go:           goTool,
		Yarn:         yarnTool,
		Version:      mkversion.Info(),
		Time:         time.Now().UTC(),
		extra:        map[any]any{},
		goBuildLimit: make(chan struct{}, runtime.NumCPU()),
	}

	return b, nil
}

func (b *Build) AddOnCloseFunc(f func() error) {
	b.onCloseFuncs = append(b.onCloseFuncs, f)
}

// Close ends the build, cleans up temporary files,
// and runs any onCloseFuncs.
func (b *Build) Close() error {
	var errs []error
	errs = append(errs, os.RemoveAll(b.Tmp))
	for _, f := range b.onCloseFuncs {
		errs = append(errs, f())
	}
	return errors.Join(errs...)
}

// Build builds all targets concurrently.
func (b *Build) Build(targets []Target) (files []string, err error) {
	if len(targets) == 0 {
		return nil, errors.New("no targets specified")
	}
	log.Printf("Building %d targets: %v", len(targets), targets)
	var (
		wg         sync.WaitGroup
		errs       = make([]error, len(targets))
		buildFiles = make([][]string, len(targets))
	)
	for i, t := range targets {
		wg.Add(1)
		go func(i int, t Target) {
			var err error
			defer func() {
				if err != nil {
					err = fmt.Errorf("%s: %w", t, err)
				}
				errs[i] = err
				wg.Done()
			}()
			fs, err := t.Build(b)
			buildFiles[i] = fs
		}(i, t)
	}
	wg.Wait()

	for _, fs := range buildFiles {
		files = append(files, fs...)
	}
	sort.Strings(files)

	return files, multierr.New(errs...)
}

// Once runs fn if Once hasn't been called with name before.
func (b *Build) Once(name string, fn func() error) error {
	return b.once.Do(name, fn)
}

// Extra returns a value from the build's extra state, creating it if necessary.
func (b *Build) Extra(key any, constructor func() any) any {
	b.extraMu.Lock()
	defer b.extraMu.Unlock()
	ret, ok := b.extra[key]
	if !ok {
		ret = constructor()
		b.extra[key] = ret
	}
	return ret
}

// GoPkg returns the path on disk of pkg.
// The module of pkg must be imported in b.Repo's go.mod.
func (b *Build) GoPkg(pkg string) (string, error) {
	out, err := b.Command(b.Repo, b.Go, "list", "-f", "{{.Dir}}", pkg).CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("finding package %q: %w", pkg, err)
	}
	return strings.TrimSpace(out), nil
}

// TmpDir creates and returns a new empty temporary directory.
// The caller does not need to clean up the directory after use, it will get
// deleted by b.Close().
func (b *Build) TmpDir() string {
	// Because we're creating all temp dirs in our parent temp dir, the only
	// failures that can happen at this point are sequence breaks (e.g. if b.Tmp
	// is deleted while stuff is still running). So, panic on error to slightly
	// simplify callsites.
	ret, err := os.MkdirTemp(b.Tmp, "")
	if err != nil {
		panic(fmt.Sprintf("creating temp dir: %v", err))
	}
	return ret
}

// BuildWebClientAssets builds the JS and CSS assets used by the web client.
// If b.WebClientSource is non-empty, assets are built in a "build" sub-directory of that path.
// Otherwise, no assets are built.
func (b *Build) BuildWebClientAssets() error {
	// Nothing in the web client assets is platform-specific,
	// so we only need to build it once.
	return b.Once("build-web-client-assets", func() error {
		if b.WebClientSource == "" {
			return nil
		}
		dir := b.WebClientSource
		if err := b.Command(dir, b.Yarn, "install").Run(); err != nil {
			return err
		}
		if err := b.Command(dir, b.Yarn, "build").Run(); err != nil {
			return err
		}
		return nil
	})
}

// BuildGoBinary builds the Go binary at path and returns the path to the
// binary. Builds are cached by path and env, so each build only happens once
// per process execution.
func (b *Build) BuildGoBinary(path string, env map[string]string) (string, error) {
	return b.BuildGoBinaryWithTags(path, env, nil)
}

// BuildGoBinaryWithTags builds the Go binary at path and returns the
// path to the binary. Builds are cached by path, env and tags, so
// each build only happens once per process execution.
//
// The passed in tags override gocross's automatic selection of build
// tags, so you will have to figure out and specify all the tags
// relevant to your build.
func (b *Build) BuildGoBinaryWithTags(path string, env map[string]string, tags []string) (string, error) {
	err := b.Once("init-go", func() error {
		log.Printf("Initializing Go toolchain")
		// If the build is using a tool/go, it may need to download a toolchain
		// and do other initialization. Running `go version` once takes care of
		// all of that and avoids that initialization happening concurrently
		// later on in builds.
		_, err := b.Command(b.Repo, b.Go, "version").CombinedOutput()
		return err
	})
	if err != nil {
		return "", err
	}

	buildKey := []any{"go-build", path, env, tags}
	return b.goBuilds.Do(buildKey, func() (string, error) {
		b.goBuildLimit <- struct{}{}
		defer func() { <-b.goBuildLimit }()

		var envStrs []string
		for k, v := range env {
			envStrs = append(envStrs, k+"="+v)
		}
		sort.Strings(envStrs)
		buildDir := b.TmpDir()
		outPath := buildDir
		if env["GOOS"] == "windowsdll" {
			// DLL builds fail unless we use a fully-qualified path to the output binary.
			outPath = filepath.Join(buildDir, filepath.Base(path)+".dll")
		}
		args := []string{"build", "-v", "-o", outPath}
		if len(tags) > 0 {
			tagsStr := strings.Join(tags, ",")
			log.Printf("Building %s (with env %s, tags %s)", path, strings.Join(envStrs, " "), tagsStr)
			args = append(args, "-tags="+tagsStr)
		} else {
			log.Printf("Building %s (with env %s)", path, strings.Join(envStrs, " "))
		}
		args = append(args, path)
		cmd := b.Command(b.Repo, b.Go, args...)
		for k, v := range env {
			cmd.Cmd.Env = append(cmd.Cmd.Env, k+"="+v)
		}
		if err := cmd.Run(); err != nil {
			return "", err
		}
		out := filepath.Join(buildDir, filepath.Base(path))
		if env["GOOS"] == "windows" || env["GOOS"] == "windowsgui" {
			out += ".exe"
		} else if env["GOOS"] == "windowsdll" {
			out += ".dll"
		}
		return out, nil
	})
}

// Command prepares an exec.Cmd to run [cmd, args...] in dir.
func (b *Build) Command(dir, cmd string, args ...string) *Command {
	ret := &Command{
		Cmd: exec.Command(cmd, args...),
	}
	if b.Verbose {
		ret.Cmd.Stdout = os.Stdout
		ret.Cmd.Stderr = os.Stderr
	} else {
		ret.Cmd.Stdout = &ret.Output
		ret.Cmd.Stderr = &ret.Output
	}
	// dist always wants to use gocross if any Go is involved.
	ret.Cmd.Env = append(os.Environ(), "TS_USE_GOCROSS=1")
	ret.Cmd.Dir = dir
	return ret
}

// Command runs an exec.Cmd and returns its exit status. If the command fails,
// its output is printed to os.Stdout, otherwise it's suppressed.
type Command struct {
	Cmd    *exec.Cmd
	Output bytes.Buffer
}

// Run is like c.Cmd.Run, but if the command fails, its output is printed to
// os.Stdout before returning the error.
func (c *Command) Run() error {
	err := c.Cmd.Run()
	if err != nil {
		// Command failed, dump its output.
		os.Stdout.Write(c.Output.Bytes())
	}
	return err
}

// CombinedOutput is like c.Cmd.CombinedOutput, but returns the output as a
// string instead of a byte slice.
func (c *Command) CombinedOutput() (string, error) {
	c.Cmd.Stdout = nil
	c.Cmd.Stderr = nil
	bs, err := c.Cmd.CombinedOutput()
	return string(bs), err
}

func findModRoot(path string) (string, error) {
	for {
		modpath := filepath.Join(path, "go.mod")
		if _, err := os.Stat(modpath); err == nil {
			return path, nil
		} else if !errors.Is(err, os.ErrNotExist) {
			return "", err
		}
		path = filepath.Dir(path)
		if path == "/" {
			return "", fmt.Errorf("no go.mod found in %q or any parent directory", path)
		}
	}
}

// findTool returns the path to the specified named tool.
// It first looks in the "tool" directory in the provided path,
// then in the $PATH environment variable.
func findTool(path, name string) (string, error) {
	tool := filepath.Join(path, "tool", name)
	if _, err := os.Stat(tool); err == nil {
		return tool, nil
	}
	tool, err := exec.LookPath(name)
	if err != nil {
		return "", err
	}
	return tool, nil
}

// FilterTargets returns the subset of targets that match any of the filters.
// If filters is empty, returns all targets.
func FilterTargets(targets []Target, filters []string) ([]Target, error) {
	var filts []*regexp.Regexp
	for _, f := range filters {
		if f == "all" {
			return targets, nil
		}
		filt, err := regexp.Compile(f)
		if err != nil {
			return nil, fmt.Errorf("invalid filter %q: %w", f, err)
		}
		filts = append(filts, filt)
	}
	var ret []Target
	for _, t := range targets {
		for _, filt := range filts {
			if filt.MatchString(t.String()) {
				ret = append(ret, t)
				break
			}
		}
	}
	return ret, nil
}