release: open-source release build logic for unix packages

Updates tailscale/corp#9221 Signed-off-by: David Anderson <danderson@tailscale.com>
2025-08-21 10:27:30 +00:00 · 2023-02-24 13:22:21 -08:00
parent 44e027abca
commit fc4b25d9fd
12 changed files with 1071 additions and 0 deletions
--- a/release/dist/dist.go
+++ b/release/dist/dist.go
@@ -0,0 +1,268 @@
+// Package dist is a release artifact builder library.
+package dist
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"sort"
+	"strings"
+	"sync"
+
+	"tailscale.com/util/multierr"
+	"tailscale.com/version/mkversion"
+)
+
+// A Target is something that can be build in a Build.
+type Target interface {
+	String() string
+	Build(build *Build) ([]string, error)
+}
+
+// A Build is a build context for Targets.
+type Build struct {
+	// Repo is a path to the root Go module for the build.
+	Repo string
+	// Tmp is a temporary directory that gets deleted when the Builder is closed.
+	Tmp string
+	// Out is where build artifacts are written.
+	Out string
+	// Go is the path to the Go binary to use for building.
+	Go string
+	// Version is the version info of the build.
+	Version mkversion.VersionInfo
+
+	// once is a cache of function invocations that should run once per process
+	// (for example building a helper docker container)
+	once once
+
+	extraMu sync.Mutex
+	extra   map[any]any
+
+	goBuilds Memoize[string]
+	// When running `dist build all` on a cold Go build cache, the fanout of
+	// gooses and goarches results in a very large number of compile processes,
+	// which bogs down the build machine.
+	//
+	// This throttles the number of concurrent `go build` invocations to the
+	// number of CPU cores, which empirically keeps the builder responsive
+	// without impacting overall build time.
+	goBuildLimit chan struct{}
+}
+
+// NewBuild creates a new Build rooted at repo, and writing artifacts to out.
+func NewBuild(repo, out string) (*Build, error) {
+	if err := os.MkdirAll(out, 0750); err != nil {
+		return nil, fmt.Errorf("creating out dir: %w", err)
+	}
+	tmp, err := os.MkdirTemp("", "dist-*")
+	if err != nil {
+		return nil, fmt.Errorf("creating tempdir: %w", err)
+	}
+	repo, err = findModRoot(repo)
+	if err != nil {
+		return nil, fmt.Errorf("finding module root: %w", err)
+	}
+	goTool, err := findGo(repo)
+	if err != nil {
+		return nil, fmt.Errorf("finding go binary: %w", err)
+	}
+	b := &Build{
+		Repo:         repo,
+		Tmp:          tmp,
+		Out:          out,
+		Go:           goTool,
+		Version:      mkversion.Info(),
+		extra:        map[any]any{},
+		goBuildLimit: make(chan struct{}, runtime.NumCPU()),
+	}
+
+	return b, nil
+}
+
+// Close ends the build and cleans up temporary files.
+func (b *Build) Close() error {
+	return os.RemoveAll(b.Tmp)
+}
+
+// Build builds all targets concurrently.
+func (b *Build) Build(targets []Target) (files []string, err error) {
+	if len(targets) == 0 {
+		return nil, errors.New("no targets specified")
+	}
+	log.Printf("Building %d targets: %v", len(targets), targets)
+	var (
+		wg         sync.WaitGroup
+		errs       = make([]error, len(targets))
+		buildFiles = make([][]string, len(targets))
+	)
+	for i, t := range targets {
+		wg.Add(1)
+		go func(i int, t Target) {
+			var err error
+			defer func() {
+				errs[i] = err
+				wg.Done()
+			}()
+			fs, err := t.Build(b)
+			buildFiles[i] = fs
+		}(i, t)
+	}
+	wg.Wait()
+
+	for _, fs := range buildFiles {
+		files = append(files, fs...)
+	}
+	sort.Strings(files)
+
+	return files, multierr.New(errs...)
+}
+
+// Once runs fn if Once hasn't been called with name before.
+func (b *Build) Once(name string, fn func() error) error {
+	return b.once.Do(name, fn)
+}
+
+// Extra returns a value from the build's extra state, creating it if necessary.
+func (b *Build) Extra(key any, constructor func() any) any {
+	b.extraMu.Lock()
+	defer b.extraMu.Unlock()
+	ret, ok := b.extra[key]
+	if !ok {
+		ret = constructor()
+		b.extra[key] = ret
+	}
+	return ret
+}
+
+// GoPkg returns the path on disk of pkg.
+// The module of pkg must be imported in b.Repo's go.mod.
+func (b *Build) GoPkg(pkg string) (string, error) {
+	bs, err := exec.Command(b.Go, "list", "-f", "{{.Dir}}", pkg).Output()
+	if err != nil {
+		return "", fmt.Errorf("finding package %q: %w", pkg, err)
+	}
+	return strings.TrimSpace(string(bs)), nil
+}
+
+// TmpDir creates and returns a new empty temporary directory.
+// The caller does not need to clean up the directory after use, it will get
+// deleted by b.Close().
+func (b *Build) TmpDir() string {
+	// Because we're creating all temp dirs in our parent temp dir, the only
+	// failures that can happen at this point are sequence breaks (e.g. if b.Tmp
+	// is deleted while stuff is still running). So, panic on error to slightly
+	// simplify callsites.
+	ret, err := os.MkdirTemp(b.Tmp, "")
+	if err != nil {
+		panic(fmt.Sprintf("creating temp dir: %v", err))
+	}
+	return ret
+}
+
+// BuildGoBinary builds the Go binary at path and returns the path to the
+// binary. Builds are cached by path and env, so each build only happens once
+// per process execution.
+func (b *Build) BuildGoBinary(path string, env map[string]string) (string, error) {
+	err := b.Once("init-go", func() error {
+		log.Printf("Initializing Go toolchain")
+		// If the build is using a tool/go, it may need to download a toolchain
+		// and do other initialization. Running `go version` once takes care of
+		// all of that and avoids that initialization happening concurrently
+		// later on in builds.
+		_, err := exec.Command(b.Go, "version").Output()
+		return err
+	})
+	if err != nil {
+		return "", err
+	}
+
+	buildKey := []any{"go-build", path, env}
+	return b.goBuilds.Do(buildKey, func() (string, error) {
+		b.goBuildLimit <- struct{}{}
+		defer func() { <-b.goBuildLimit }()
+
+		var envStrs []string
+		for k, v := range env {
+			envStrs = append(envStrs, k+"="+v)
+		}
+		sort.Strings(envStrs)
+		log.Printf("Building %s (with env %s)", path, strings.Join(envStrs, " "))
+		buildDir := b.TmpDir()
+		cmd := exec.Command(b.Go, "build", "-o", buildDir, path)
+		cmd.Dir = b.Repo
+		cmd.Env = os.Environ()
+		for k, v := range env {
+			cmd.Env = append(cmd.Env, k+"="+v)
+		}
+		cmd.Env = append(cmd.Env, "TS_USE_GOCROSS=1")
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			return "", err
+		}
+		out := filepath.Join(buildDir, filepath.Base(path))
+		if env["GOOS"] == "windows" || env["GOOS"] == "windowsgui" {
+			out += ".exe"
+		}
+		return out, nil
+	})
+}
+
+func findModRoot(path string) (string, error) {
+	for {
+		modpath := filepath.Join(path, "go.mod")
+		if _, err := os.Stat(modpath); err == nil {
+			return path, nil
+		} else if !errors.Is(err, os.ErrNotExist) {
+			return "", err
+		}
+		path = filepath.Dir(path)
+		if path == "/" {
+			return "", fmt.Errorf("no go.mod found in %q or any parent directory", path)
+		}
+	}
+}
+
+func findGo(path string) (string, error) {
+	toolGo := filepath.Join(path, "tool/go")
+	if _, err := os.Stat(toolGo); err == nil {
+		return toolGo, nil
+	}
+	toolGo, err := exec.LookPath("go")
+	if err != nil {
+		return "", err
+	}
+	return toolGo, nil
+}
+
+// FilterTargets returns the subset of targets that match any of the filters.
+// If filters is empty, returns all targets.
+func FilterTargets(targets []Target, filters []string) ([]Target, error) {
+	var filts []*regexp.Regexp
+	for _, f := range filters {
+		if f == "all" {
+			return targets, nil
+		}
+		filt, err := regexp.Compile(f)
+		if err != nil {
+			return nil, fmt.Errorf("invalid filter %q: %w", f, err)
+		}
+		filts = append(filts, filt)
+	}
+	var ret []Target
+	for _, t := range targets {
+		for _, filt := range filts {
+			if filt.MatchString(t.String()) {
+				ret = append(ret, t)
+				break
+			}
+		}
+	}
+	return ret, nil
+}