ipn/ipnlocal: run "tailscale update" via systemd-run on Linux (#10229)

When we run tailscled under systemd, restarting the unit kills all child
processes, including "tailscale update". And during update, the package
manager will restart the tailscaled unit. Specifically on Debian-based
distros, interrupting `apt-get install` can get the system into a wedged
state which requires the user to manually run `dpkg --configure` to
recover.

To avoid all this, use `systemd-run` where available to run the
`tailscale update` process. This launches it in a separate temporary
unit and doesn't kill it when parent unit is restarted.

Also, detect when `apt-get install` complains about aborted update and
try to restore the system by running `dpkg --configure tailscale`. This
could help if the system unexpectedly shuts down during our auto-update.

Fixes https://github.com/tailscale/corp/issues/15771

Signed-off-by: Andrew Lytvynov <awly@tailscale.com>
This commit is contained in:
Andrew Lytvynov 2023-11-13 17:41:21 -07:00 committed by GitHub
parent c99488ea19
commit 955e2fcbfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 32 additions and 10 deletions

View File

@ -415,17 +415,25 @@ func (up *Updater) updateDebLike() error {
// we're not updating them:
"-o", "APT::Get::List-Cleanup=0",
)
cmd.Stdout = up.Stdout
cmd.Stderr = up.Stderr
if err := cmd.Run(); err != nil {
return err
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("apt-get update failed: %w; output:\n%s", err, out)
}
cmd = exec.Command("apt-get", "install", "--yes", "--allow-downgrades", "tailscale="+ver)
cmd.Stdout = up.Stdout
cmd.Stderr = up.Stderr
if err := cmd.Run(); err != nil {
return err
for i := 0; i < 2; i++ {
out, err := exec.Command("apt-get", "install", "--yes", "--allow-downgrades", "tailscale="+ver).CombinedOutput()
if err != nil {
if !bytes.Contains(out, []byte(`dpkg was interrupted`)) {
return fmt.Errorf("apt-get install failed: %w; output:\n%s", err, out)
}
up.Logf("apt-get install failed: %s; output:\n%s", err, out)
up.Logf("running dpkg --configure tailscale")
out, err = exec.Command("dpkg", "--force-confdef,downgrade", "--configure", "tailscale").CombinedOutput()
if err != nil {
return fmt.Errorf("dpkg --configure tailscale failed: %w; output:\n%s", err, out)
}
continue
}
break
}
return nil

View File

@ -280,7 +280,7 @@ func handleC2NUpdatePost(b *LocalBackend, w http.ResponseWriter, r *http.Request
return
}
cmd := exec.Command(cmdTS, "update", "--yes")
cmd := tailscaleUpdateCmd(cmdTS)
buf := new(bytes.Buffer)
cmd.Stdout = buf
cmd.Stderr = buf
@ -412,6 +412,20 @@ func findCmdTailscale() (string, error) {
return "", errors.New("tailscale executable not found in expected place")
}
func tailscaleUpdateCmd(cmdTS string) *exec.Cmd {
if runtime.GOOS != "linux" {
return exec.Command(cmdTS, "update", "--yes")
}
if _, err := exec.LookPath("systemd-run"); err != nil {
return exec.Command(cmdTS, "update", "--yes")
}
// When systemd-run is available, use it to run the update command. This
// creates a new temporary unit separate from the tailscaled unit. When
// tailscaled is restarted during the update, systemd won't kill this
// temporary update unit, which could cause unexpected breakage.
return exec.Command("systemd-run", "--wait", "--pipe", "--collect", cmdTS, "update", "--yes")
}
func regularFileExists(path string) bool {
fi, err := os.Stat(path)
return err == nil && fi.Mode().IsRegular()