mirror of
https://github.com/tailscale/tailscale.git
synced 2024-12-04 23:45:34 +00:00
0a84215036
Creates new QNAP builder target, which builds go binaries then uses docker to build into QNAP packages. Much of the docker/script code here is pulled over from https://github.com/tailscale/tailscale-qpkg, with adaptation into our builder structures. The qnap/Tailscale folder contains static resources needed to build Tailscale qpkg packages, and is an exact copy of the existing folder in the tailscale-qpkg repo. Builds can be run with: ``` sudo ./tool/go run ./cmd/dist build qnap ``` Updates tailscale/tailscale-qpkg#135 Signed-off-by: Sonia Appasamy <sonia@tailscale.com>
415 lines
12 KiB
Go
415 lines
12 KiB
Go
// Copyright (c) Tailscale Inc & AUTHORS
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
// Package dist is a release artifact builder library.
|
|
package dist
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"runtime"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"tailscale.com/util/multierr"
|
|
"tailscale.com/version/mkversion"
|
|
)
|
|
|
|
// A Target is something that can be build in a Build.
|
|
type Target interface {
|
|
String() string
|
|
Build(build *Build) ([]string, error)
|
|
}
|
|
|
|
// Signer is pluggable signer for a Target.
|
|
type Signer func(io.Reader) ([]byte, error)
|
|
|
|
// SignFile signs the file at filePath with s and writes the signature to
|
|
// sigPath.
|
|
func (s Signer) SignFile(filePath, sigPath string) error {
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
sig, err := s(f)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(sigPath, sig, 0644)
|
|
}
|
|
|
|
// A Build is a build context for Targets.
|
|
type Build struct {
|
|
// Repo is a path to the root Go module for the build.
|
|
Repo string
|
|
// Out is where build artifacts are written.
|
|
Out string
|
|
// Verbose is whether to print all command output, rather than just failed
|
|
// commands.
|
|
Verbose bool
|
|
// WebClientSource is a path to the source for the web client.
|
|
// If non-empty, web client assets will be built.
|
|
WebClientSource string
|
|
|
|
// Tmp is a temporary directory that gets deleted when the Builder is closed.
|
|
Tmp string
|
|
// Go is the path to the Go binary to use for building.
|
|
Go string
|
|
// Yarn is the path to the yarn binary to use for building the web client assets.
|
|
Yarn string
|
|
// Version is the version info of the build.
|
|
Version mkversion.VersionInfo
|
|
// Time is the timestamp of the build.
|
|
Time time.Time
|
|
|
|
// once is a cache of function invocations that should run once per process
|
|
// (for example building a helper docker container)
|
|
once once
|
|
|
|
extraMu sync.Mutex
|
|
extra map[any]any
|
|
|
|
goBuilds Memoize[string]
|
|
// When running `dist build all` on a cold Go build cache, the fanout of
|
|
// gooses and goarches results in a very large number of compile processes,
|
|
// which bogs down the build machine.
|
|
//
|
|
// This throttles the number of concurrent `go build` invocations to the
|
|
// number of CPU cores, which empirically keeps the builder responsive
|
|
// without impacting overall build time.
|
|
goBuildLimit chan struct{}
|
|
|
|
onCloseFuncs []func() error // funcs to be called when Builder is closed
|
|
}
|
|
|
|
// NewBuild creates a new Build rooted at repo, and writing artifacts to out.
|
|
func NewBuild(repo, out string) (*Build, error) {
|
|
if err := os.MkdirAll(out, 0750); err != nil {
|
|
return nil, fmt.Errorf("creating out dir: %w", err)
|
|
}
|
|
tmp, err := os.MkdirTemp("", "dist-*")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("creating tempdir: %w", err)
|
|
}
|
|
repo, err = findModRoot(repo)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("finding module root: %w", err)
|
|
}
|
|
goTool, err := findTool(repo, "go")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("finding go binary: %w", err)
|
|
}
|
|
yarnTool, err := findTool(repo, "yarn")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("finding yarn binary: %w", err)
|
|
}
|
|
b := &Build{
|
|
Repo: repo,
|
|
Tmp: tmp,
|
|
Out: out,
|
|
Go: goTool,
|
|
Yarn: yarnTool,
|
|
Version: mkversion.Info(),
|
|
Time: time.Now().UTC(),
|
|
extra: map[any]any{},
|
|
goBuildLimit: make(chan struct{}, runtime.NumCPU()),
|
|
}
|
|
|
|
return b, nil
|
|
}
|
|
|
|
func (b *Build) AddOnCloseFunc(f func() error) {
|
|
b.onCloseFuncs = append(b.onCloseFuncs, f)
|
|
}
|
|
|
|
// Close ends the build, cleans up temporary files,
|
|
// and runs any onCloseFuncs.
|
|
func (b *Build) Close() error {
|
|
var errs []error
|
|
errs = append(errs, os.RemoveAll(b.Tmp))
|
|
for _, f := range b.onCloseFuncs {
|
|
errs = append(errs, f())
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
// Build builds all targets concurrently.
|
|
func (b *Build) Build(targets []Target) (files []string, err error) {
|
|
if len(targets) == 0 {
|
|
return nil, errors.New("no targets specified")
|
|
}
|
|
log.Printf("Building %d targets: %v", len(targets), targets)
|
|
var (
|
|
wg sync.WaitGroup
|
|
errs = make([]error, len(targets))
|
|
buildFiles = make([][]string, len(targets))
|
|
)
|
|
for i, t := range targets {
|
|
wg.Add(1)
|
|
go func(i int, t Target) {
|
|
var err error
|
|
defer func() {
|
|
if err != nil {
|
|
err = fmt.Errorf("%s: %w", t, err)
|
|
}
|
|
errs[i] = err
|
|
wg.Done()
|
|
}()
|
|
fs, err := t.Build(b)
|
|
buildFiles[i] = fs
|
|
}(i, t)
|
|
}
|
|
wg.Wait()
|
|
|
|
for _, fs := range buildFiles {
|
|
files = append(files, fs...)
|
|
}
|
|
sort.Strings(files)
|
|
|
|
return files, multierr.New(errs...)
|
|
}
|
|
|
|
// Once runs fn if Once hasn't been called with name before.
|
|
func (b *Build) Once(name string, fn func() error) error {
|
|
return b.once.Do(name, fn)
|
|
}
|
|
|
|
// Extra returns a value from the build's extra state, creating it if necessary.
|
|
func (b *Build) Extra(key any, constructor func() any) any {
|
|
b.extraMu.Lock()
|
|
defer b.extraMu.Unlock()
|
|
ret, ok := b.extra[key]
|
|
if !ok {
|
|
ret = constructor()
|
|
b.extra[key] = ret
|
|
}
|
|
return ret
|
|
}
|
|
|
|
// GoPkg returns the path on disk of pkg.
|
|
// The module of pkg must be imported in b.Repo's go.mod.
|
|
func (b *Build) GoPkg(pkg string) (string, error) {
|
|
out, err := b.Command(b.Repo, b.Go, "list", "-f", "{{.Dir}}", pkg).CombinedOutput()
|
|
if err != nil {
|
|
return "", fmt.Errorf("finding package %q: %w", pkg, err)
|
|
}
|
|
return strings.TrimSpace(out), nil
|
|
}
|
|
|
|
// TmpDir creates and returns a new empty temporary directory.
|
|
// The caller does not need to clean up the directory after use, it will get
|
|
// deleted by b.Close().
|
|
func (b *Build) TmpDir() string {
|
|
// Because we're creating all temp dirs in our parent temp dir, the only
|
|
// failures that can happen at this point are sequence breaks (e.g. if b.Tmp
|
|
// is deleted while stuff is still running). So, panic on error to slightly
|
|
// simplify callsites.
|
|
ret, err := os.MkdirTemp(b.Tmp, "")
|
|
if err != nil {
|
|
panic(fmt.Sprintf("creating temp dir: %v", err))
|
|
}
|
|
return ret
|
|
}
|
|
|
|
// BuildWebClientAssets builds the JS and CSS assets used by the web client.
|
|
// If b.WebClientSource is non-empty, assets are built in a "build" sub-directory of that path.
|
|
// Otherwise, no assets are built.
|
|
func (b *Build) BuildWebClientAssets() error {
|
|
// Nothing in the web client assets is platform-specific,
|
|
// so we only need to build it once.
|
|
return b.Once("build-web-client-assets", func() error {
|
|
if b.WebClientSource == "" {
|
|
return nil
|
|
}
|
|
dir := b.WebClientSource
|
|
if err := b.Command(dir, b.Yarn, "install").Run(); err != nil {
|
|
return err
|
|
}
|
|
if err := b.Command(dir, b.Yarn, "build").Run(); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// BuildGoBinary builds the Go binary at path and returns the path to the
|
|
// binary. Builds are cached by path and env, so each build only happens once
|
|
// per process execution.
|
|
func (b *Build) BuildGoBinary(path string, env map[string]string) (string, error) {
|
|
return b.BuildGoBinaryWithTags(path, env, nil)
|
|
}
|
|
|
|
// BuildGoBinaryWithTags builds the Go binary at path and returns the
|
|
// path to the binary. Builds are cached by path, env and tags, so
|
|
// each build only happens once per process execution.
|
|
//
|
|
// The passed in tags override gocross's automatic selection of build
|
|
// tags, so you will have to figure out and specify all the tags
|
|
// relevant to your build.
|
|
func (b *Build) BuildGoBinaryWithTags(path string, env map[string]string, tags []string) (string, error) {
|
|
err := b.Once("init-go", func() error {
|
|
log.Printf("Initializing Go toolchain")
|
|
// If the build is using a tool/go, it may need to download a toolchain
|
|
// and do other initialization. Running `go version` once takes care of
|
|
// all of that and avoids that initialization happening concurrently
|
|
// later on in builds.
|
|
_, err := b.Command(b.Repo, b.Go, "version").CombinedOutput()
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
buildKey := []any{"go-build", path, env, tags}
|
|
return b.goBuilds.Do(buildKey, func() (string, error) {
|
|
b.goBuildLimit <- struct{}{}
|
|
defer func() { <-b.goBuildLimit }()
|
|
|
|
var envStrs []string
|
|
for k, v := range env {
|
|
envStrs = append(envStrs, k+"="+v)
|
|
}
|
|
sort.Strings(envStrs)
|
|
buildDir := b.TmpDir()
|
|
outPath := buildDir
|
|
if env["GOOS"] == "windowsdll" {
|
|
// DLL builds fail unless we use a fully-qualified path to the output binary.
|
|
outPath = filepath.Join(buildDir, filepath.Base(path)+".dll")
|
|
}
|
|
args := []string{"build", "-v", "-o", outPath}
|
|
if len(tags) > 0 {
|
|
tagsStr := strings.Join(tags, ",")
|
|
log.Printf("Building %s (with env %s, tags %s)", path, strings.Join(envStrs, " "), tagsStr)
|
|
args = append(args, "-tags="+tagsStr)
|
|
} else {
|
|
log.Printf("Building %s (with env %s)", path, strings.Join(envStrs, " "))
|
|
}
|
|
args = append(args, path)
|
|
cmd := b.Command(b.Repo, b.Go, args...)
|
|
for k, v := range env {
|
|
cmd.Cmd.Env = append(cmd.Cmd.Env, k+"="+v)
|
|
}
|
|
if err := cmd.Run(); err != nil {
|
|
return "", err
|
|
}
|
|
out := filepath.Join(buildDir, filepath.Base(path))
|
|
if env["GOOS"] == "windows" || env["GOOS"] == "windowsgui" {
|
|
out += ".exe"
|
|
} else if env["GOOS"] == "windowsdll" {
|
|
out += ".dll"
|
|
}
|
|
return out, nil
|
|
})
|
|
}
|
|
|
|
// Command prepares an exec.Cmd to run [cmd, args...] in dir.
|
|
func (b *Build) Command(dir, cmd string, args ...string) *Command {
|
|
ret := &Command{
|
|
Cmd: exec.Command(cmd, args...),
|
|
}
|
|
if b.Verbose {
|
|
ret.Cmd.Stdout = os.Stdout
|
|
ret.Cmd.Stderr = os.Stderr
|
|
} else {
|
|
ret.Cmd.Stdout = &ret.Output
|
|
ret.Cmd.Stderr = &ret.Output
|
|
}
|
|
// dist always wants to use gocross if any Go is involved.
|
|
ret.Cmd.Env = append(os.Environ(), "TS_USE_GOCROSS=1")
|
|
ret.Cmd.Dir = dir
|
|
return ret
|
|
}
|
|
|
|
// Command runs an exec.Cmd and returns its exit status. If the command fails,
|
|
// its output is printed to os.Stdout, otherwise it's suppressed.
|
|
type Command struct {
|
|
Cmd *exec.Cmd
|
|
Output bytes.Buffer
|
|
}
|
|
|
|
// Run is like c.Cmd.Run, but if the command fails, its output is printed to
|
|
// os.Stdout before returning the error.
|
|
func (c *Command) Run() error {
|
|
err := c.Cmd.Run()
|
|
if err != nil {
|
|
// Command failed, dump its output.
|
|
os.Stdout.Write(c.Output.Bytes())
|
|
}
|
|
return err
|
|
}
|
|
|
|
// CombinedOutput is like c.Cmd.CombinedOutput, but returns the output as a
|
|
// string instead of a byte slice.
|
|
func (c *Command) CombinedOutput() (string, error) {
|
|
c.Cmd.Stdout = nil
|
|
c.Cmd.Stderr = nil
|
|
bs, err := c.Cmd.CombinedOutput()
|
|
return string(bs), err
|
|
}
|
|
|
|
func findModRoot(path string) (string, error) {
|
|
for {
|
|
modpath := filepath.Join(path, "go.mod")
|
|
if _, err := os.Stat(modpath); err == nil {
|
|
return path, nil
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
return "", err
|
|
}
|
|
path = filepath.Dir(path)
|
|
if path == "/" {
|
|
return "", fmt.Errorf("no go.mod found in %q or any parent directory", path)
|
|
}
|
|
}
|
|
}
|
|
|
|
// findTool returns the path to the specified named tool.
|
|
// It first looks in the "tool" directory in the provided path,
|
|
// then in the $PATH environment variable.
|
|
func findTool(path, name string) (string, error) {
|
|
tool := filepath.Join(path, "tool", name)
|
|
if _, err := os.Stat(tool); err == nil {
|
|
return tool, nil
|
|
}
|
|
tool, err := exec.LookPath(name)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return tool, nil
|
|
}
|
|
|
|
// FilterTargets returns the subset of targets that match any of the filters.
|
|
// If filters is empty, returns all targets.
|
|
func FilterTargets(targets []Target, filters []string) ([]Target, error) {
|
|
var filts []*regexp.Regexp
|
|
for _, f := range filters {
|
|
if f == "all" {
|
|
return targets, nil
|
|
}
|
|
filt, err := regexp.Compile(f)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid filter %q: %w", f, err)
|
|
}
|
|
filts = append(filts, filt)
|
|
}
|
|
var ret []Target
|
|
for _, t := range targets {
|
|
for _, filt := range filts {
|
|
if filt.MatchString(t.String()) {
|
|
ret = append(ret, t)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return ret, nil
|
|
}
|