From 0cce456ee5bf45b555521109ff525ef81adb9650 Mon Sep 17 00:00:00 2001 From: Sonia Appasamy Date: Tue, 23 Apr 2024 19:11:31 -0400 Subject: [PATCH] release/dist/qnap: use tmp file directory for qpkg building This change allows for the release/dist/qnap package to be used outside of the tailscale repo (notably, will be used from corp), by using an embedded file system for build files which gets temporarily written to a new folder during qnap build runs. Without this change, when used from corp, the release/dist/qnap folder will fail to be found within the corp repo, causing various steps of the build to fail. The file renames in this change are to combine the build files into a /files folder, separated into /scripts and /Tailscale. Updates tailscale/tailscale-qpkg#135 Signed-off-by: Sonia Appasamy --- .../qnap/{ => files}/Tailscale/build_sign.csv | 0 .../{ => files}/Tailscale/config/.gitkeep | 0 .../qnap/{ => files}/Tailscale/icons/.gitkeep | 0 .../{ => files}/Tailscale/icons/Tailscale.gif | Bin .../Tailscale/icons/Tailscale_80.gif | Bin .../Tailscale/icons/Tailscale_gray.gif | Bin .../{ => files}/Tailscale/package_routines | 0 .../qnap/{ => files}/Tailscale/qpkg.cfg.in | 0 .../{ => files}/Tailscale/shared/Tailscale.sh | 0 .../{ => files}/Tailscale/shared/ui/.htaccess | 0 .../{ => files}/Tailscale/shared/ui/index.cgi | 0 .../qnap/{ => files/scripts}/Dockerfile.qpkg | 0 .../qnap/{ => files/scripts}/build-qpkg.sh | 0 release/dist/qnap/pkgs.go | 147 ++++++++++++------ 14 files changed, 98 insertions(+), 49 deletions(-) rename release/dist/qnap/{ => files}/Tailscale/build_sign.csv (100%) rename release/dist/qnap/{ => files}/Tailscale/config/.gitkeep (100%) rename release/dist/qnap/{ => files}/Tailscale/icons/.gitkeep (100%) rename release/dist/qnap/{ => files}/Tailscale/icons/Tailscale.gif (100%) rename release/dist/qnap/{ => files}/Tailscale/icons/Tailscale_80.gif (100%) rename release/dist/qnap/{ => files}/Tailscale/icons/Tailscale_gray.gif (100%) rename release/dist/qnap/{ => files}/Tailscale/package_routines (100%) rename release/dist/qnap/{ => files}/Tailscale/qpkg.cfg.in (100%) rename release/dist/qnap/{ => files}/Tailscale/shared/Tailscale.sh (100%) rename release/dist/qnap/{ => files}/Tailscale/shared/ui/.htaccess (100%) rename release/dist/qnap/{ => files}/Tailscale/shared/ui/index.cgi (100%) rename release/dist/qnap/{ => files/scripts}/Dockerfile.qpkg (100%) rename release/dist/qnap/{ => files/scripts}/build-qpkg.sh (100%) diff --git a/release/dist/qnap/Tailscale/build_sign.csv b/release/dist/qnap/files/Tailscale/build_sign.csv similarity index 100% rename from release/dist/qnap/Tailscale/build_sign.csv rename to release/dist/qnap/files/Tailscale/build_sign.csv diff --git a/release/dist/qnap/Tailscale/config/.gitkeep b/release/dist/qnap/files/Tailscale/config/.gitkeep similarity index 100% rename from release/dist/qnap/Tailscale/config/.gitkeep rename to release/dist/qnap/files/Tailscale/config/.gitkeep diff --git a/release/dist/qnap/Tailscale/icons/.gitkeep b/release/dist/qnap/files/Tailscale/icons/.gitkeep similarity index 100% rename from release/dist/qnap/Tailscale/icons/.gitkeep rename to release/dist/qnap/files/Tailscale/icons/.gitkeep diff --git a/release/dist/qnap/Tailscale/icons/Tailscale.gif b/release/dist/qnap/files/Tailscale/icons/Tailscale.gif similarity index 100% rename from release/dist/qnap/Tailscale/icons/Tailscale.gif rename to release/dist/qnap/files/Tailscale/icons/Tailscale.gif diff --git a/release/dist/qnap/Tailscale/icons/Tailscale_80.gif b/release/dist/qnap/files/Tailscale/icons/Tailscale_80.gif similarity index 100% rename from release/dist/qnap/Tailscale/icons/Tailscale_80.gif rename to release/dist/qnap/files/Tailscale/icons/Tailscale_80.gif diff --git a/release/dist/qnap/Tailscale/icons/Tailscale_gray.gif b/release/dist/qnap/files/Tailscale/icons/Tailscale_gray.gif similarity index 100% rename from release/dist/qnap/Tailscale/icons/Tailscale_gray.gif rename to release/dist/qnap/files/Tailscale/icons/Tailscale_gray.gif diff --git a/release/dist/qnap/Tailscale/package_routines b/release/dist/qnap/files/Tailscale/package_routines similarity index 100% rename from release/dist/qnap/Tailscale/package_routines rename to release/dist/qnap/files/Tailscale/package_routines diff --git a/release/dist/qnap/Tailscale/qpkg.cfg.in b/release/dist/qnap/files/Tailscale/qpkg.cfg.in similarity index 100% rename from release/dist/qnap/Tailscale/qpkg.cfg.in rename to release/dist/qnap/files/Tailscale/qpkg.cfg.in diff --git a/release/dist/qnap/Tailscale/shared/Tailscale.sh b/release/dist/qnap/files/Tailscale/shared/Tailscale.sh similarity index 100% rename from release/dist/qnap/Tailscale/shared/Tailscale.sh rename to release/dist/qnap/files/Tailscale/shared/Tailscale.sh diff --git a/release/dist/qnap/Tailscale/shared/ui/.htaccess b/release/dist/qnap/files/Tailscale/shared/ui/.htaccess similarity index 100% rename from release/dist/qnap/Tailscale/shared/ui/.htaccess rename to release/dist/qnap/files/Tailscale/shared/ui/.htaccess diff --git a/release/dist/qnap/Tailscale/shared/ui/index.cgi b/release/dist/qnap/files/Tailscale/shared/ui/index.cgi similarity index 100% rename from release/dist/qnap/Tailscale/shared/ui/index.cgi rename to release/dist/qnap/files/Tailscale/shared/ui/index.cgi diff --git a/release/dist/qnap/Dockerfile.qpkg b/release/dist/qnap/files/scripts/Dockerfile.qpkg similarity index 100% rename from release/dist/qnap/Dockerfile.qpkg rename to release/dist/qnap/files/scripts/Dockerfile.qpkg diff --git a/release/dist/qnap/build-qpkg.sh b/release/dist/qnap/files/scripts/build-qpkg.sh similarity index 100% rename from release/dist/qnap/build-qpkg.sh rename to release/dist/qnap/files/scripts/build-qpkg.sh diff --git a/release/dist/qnap/pkgs.go b/release/dist/qnap/pkgs.go index 5ee29e988..e42afc30a 100644 --- a/release/dist/qnap/pkgs.go +++ b/release/dist/qnap/pkgs.go @@ -7,8 +7,9 @@ package qnap import ( - "errors" + "embed" "fmt" + "io/fs" "log" "os" "os/exec" @@ -39,13 +40,7 @@ func (t *target) Build(b *dist.Build) ([]string, error) { return nil, fmt.Errorf("docker not found, cannot build: %w", err) } - if t.signer != nil { - if err := t.setUpSignatureFiles(b); err != nil { - return nil, err - } - } - - qnapBuilds := getQnapBuilds(b) + qnapBuilds := getQnapBuilds(b, t.signer) inner, err := qnapBuilds.buildInnerPackage(b, t.goenv) if err != nil { return nil, err @@ -77,8 +72,8 @@ func (t *target) buildQPKG(b *dist.Build, qnapBuilds *qnapBuilds, inner *innerPk "-v", fmt.Sprintf("%s:/tailscale", inner.tailscalePath), "-v", fmt.Sprintf("%s:/tailscaled", inner.tailscaledPath), // Tailscale folder has QNAP package setup files needed for building. - "-v", fmt.Sprintf("%s:/Tailscale", filepath.Join(b.Repo, "release/dist/qnap/Tailscale")), - "-v", fmt.Sprintf("%s:/build-qpkg.sh", filepath.Join(b.Repo, "release/dist/qnap/build-qpkg.sh")), + "-v", fmt.Sprintf("%s:/Tailscale", filepath.Join(qnapBuilds.tmpDir, "files/Tailscale")), + "-v", fmt.Sprintf("%s:/build-qpkg.sh", filepath.Join(qnapBuilds.tmpDir, "files/scripts/build-qpkg.sh")), "-v", fmt.Sprintf("%s:/out", b.Out), "build.tailscale.io/qdk:latest", "/build-qpkg.sh", @@ -101,40 +96,6 @@ func (t *target) buildQPKG(b *dist.Build, qnapBuilds *qnapBuilds, inner *innerPk return []string{filePath, filePath + ".md5"}, nil } -func (t *target) setUpSignatureFiles(b *dist.Build) error { - return b.Once(fmt.Sprintf("qnap-signature-%s-%s", t.signer.privateKeyPath, t.signer.certificatePath), func() error { - log.Print("Setting up qnap signature files") - - key, err := os.ReadFile(t.signer.privateKeyPath) - if err != nil { - return err - } - cert, err := os.ReadFile(t.signer.certificatePath) - if err != nil { - return err - } - - // QNAP's qbuild command expects key and cert files to be in the root - // of the project directory (in our case release/dist/qnap/Tailscale). - // So here, we copy the key and cert over to the project folder for the - // duration of qnap package building and then delete them on close. - - keyPath := filepath.Join(b.Repo, "release/dist/qnap/Tailscale/private_key") - if err := os.WriteFile(keyPath, key, 0400); err != nil { - return err - } - certPath := filepath.Join(b.Repo, "release/dist/qnap/Tailscale/certificate") - if err := os.WriteFile(certPath, cert, 0400); err != nil { - return err - } - - b.AddOnCloseFunc(func() error { - return errors.Join(os.Remove(keyPath), os.Remove(certPath)) - }) - return nil - }) -} - type qnapBuildsMemoizeKey struct{} type innerPkg struct { @@ -148,11 +109,99 @@ type qnapBuilds struct { // It is used to avoid repeated compilations for the same architecture. innerPkgs dist.Memoize[*innerPkg] dockerImageMu sync.Mutex + // tmpDir is a temp directory used for building qpkgs. + // It gets cleaned up when the dist.Build is closed. + tmpDir string } // getQnapBuilds returns the qnapBuilds for b, creating one if needed. -func getQnapBuilds(b *dist.Build) *qnapBuilds { - return b.Extra(qnapBuildsMemoizeKey{}, func() any { return new(qnapBuilds) }).(*qnapBuilds) +func getQnapBuilds(b *dist.Build, signer *signer) *qnapBuilds { + return b.Extra(qnapBuildsMemoizeKey{}, func() any { + builds, err := newQNAPBuilds(b, signer) + if err != nil { + panic(fmt.Errorf("setUpTmpDir: %v", err)) + } + return builds + }).(*qnapBuilds) +} + +//go:embed all:files +var buildFiles embed.FS + +// newQNAPBuilds creates a new qnapBuilds instance to hold context shared by +// all qnap targets, and sets up its local temp directory used for building. +// +// The qnapBuilds.tmpDir is filled with the contents of the buildFiles embedded +// FS for building. +// +// We do this to allow for this tailscale.com/release/dist/qnap package to be +// used from both the corp and OSS repos. When built from OSS source directly, +// this is a superfluous extra step, but when imported as a go module to another +// repo (such as corp), we must do this to allow for the module's build files +// to be reachable and editable from docker. +// +// This runs only once per dist.Build instance, is shared by all qnap targets, +// and gets cleaned up upon close of the dist.Build. +// +// When a signer is provided, newQNAPBuilds also sets up the qpkg signature +// files in qbuild's expected location within m.tmpDir. +func newQNAPBuilds(b *dist.Build, signer *signer) (*qnapBuilds, error) { + m := new(qnapBuilds) + + log.Print("Setting up qnap tmp build directory") + m.tmpDir = filepath.Join(b.Repo, "tmp-qnap-build") + b.AddOnCloseFunc(func() error { + return os.RemoveAll(m.tmpDir) + }) + + if err := fs.WalkDir(buildFiles, "files", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + outPath := filepath.Join(m.tmpDir, path) + if d.IsDir() { + return os.MkdirAll(outPath, 0700) + } + file, err := fs.ReadFile(buildFiles, path) + if err != nil { + return err + } + perm := fs.FileMode(0600) + if filepath.Ext(path) == ".sh" { + perm = 0700 + } + return os.WriteFile(outPath, file, perm) + }); err != nil { + return nil, err + } + + if signer != nil { + log.Print("Setting up qnap signing files") + + key, err := os.ReadFile(signer.privateKeyPath) + if err != nil { + return nil, err + } + cert, err := os.ReadFile(signer.certificatePath) + if err != nil { + return nil, err + } + + // QNAP's qbuild command expects key and cert files to be in the root + // of the project directory (in our case release/dist/qnap/Tailscale). + // So here, we copy the key and cert over to the project folder for the + // duration of qnap package building and then delete them on close. + + keyPath := filepath.Join(m.tmpDir, "files/Tailscale/private_key") + if err := os.WriteFile(keyPath, key, 0400); err != nil { + return nil, err + } + certPath := filepath.Join(m.tmpDir, "files/Tailscale/certificate") + if err := os.WriteFile(certPath, cert, 0400); err != nil { + return nil, err + } + } + return m, nil } // buildInnerPackage builds the go binaries used for qnap packages. @@ -181,7 +230,7 @@ func (m *qnapBuilds) buildInnerPackage(b *dist.Build, goenv map[string]string) ( // So here, we move the binaries into a directory within the b.Repo // path and clean it up when the builder closes. - tmpDir := filepath.Join(b.Repo, fmt.Sprintf("/tmp-qnap-%s-%s-%s", b.Version.Short, goenv["GOOS"], goenv["GOARCH"])) + tmpDir := filepath.Join(m.tmpDir, fmt.Sprintf("/binaries-%s-%s-%s", b.Version.Short, goenv["GOOS"], goenv["GOARCH"])) if err = os.MkdirAll(tmpDir, 0755); err != nil { return nil, err } @@ -216,9 +265,9 @@ func (m *qnapBuilds) makeDockerImage(b *dist.Build) error { log.Printf("Building qnapbuilder docker image") cmd := b.Command(b.Repo, "docker", "build", - "-f", filepath.Join(b.Repo, "release/dist/qnap/Dockerfile.qpkg"), + "-f", filepath.Join(m.tmpDir, "files/scripts/Dockerfile.qpkg"), "-t", "build.tailscale.io/qdk:latest", - filepath.Join(b.Repo, "release/dist/qnap/"), + filepath.Join(m.tmpDir, "files/scripts"), ) out, err := cmd.CombinedOutput() if err != nil {