ipn/ipnlocal: run "tailscale update" via systemd-run on Linux ()

When we run tailscled under systemd, restarting the unit kills all child
processes, including "tailscale update". And during update, the package
manager will restart the tailscaled unit. Specifically on Debian-based
distros, interrupting `apt-get install` can get the system into a wedged
state which requires the user to manually run `dpkg --configure` to
recover.

To avoid all this, use `systemd-run` where available to run the
`tailscale update` process. This launches it in a separate temporary
unit and doesn't kill it when parent unit is restarted.

Also, detect when `apt-get install` complains about aborted update and
try to restore the system by running `dpkg --configure tailscale`. This
could help if the system unexpectedly shuts down during our auto-update.

Fixes https://github.com/tailscale/corp/issues/15771

Signed-off-by: Andrew Lytvynov <awly@tailscale.com>
This commit is contained in:
Andrew Lytvynov 2023-11-13 17:41:21 -07:00 committed by GitHub
parent c99488ea19
commit 955e2fcbfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 32 additions and 10 deletions
clientupdate
ipn/ipnlocal

@ -415,17 +415,25 @@ func (up *Updater) updateDebLike() error {
// we're not updating them: // we're not updating them:
"-o", "APT::Get::List-Cleanup=0", "-o", "APT::Get::List-Cleanup=0",
) )
cmd.Stdout = up.Stdout if out, err := cmd.CombinedOutput(); err != nil {
cmd.Stderr = up.Stderr return fmt.Errorf("apt-get update failed: %w; output:\n%s", err, out)
if err := cmd.Run(); err != nil {
return err
} }
cmd = exec.Command("apt-get", "install", "--yes", "--allow-downgrades", "tailscale="+ver) for i := 0; i < 2; i++ {
cmd.Stdout = up.Stdout out, err := exec.Command("apt-get", "install", "--yes", "--allow-downgrades", "tailscale="+ver).CombinedOutput()
cmd.Stderr = up.Stderr if err != nil {
if err := cmd.Run(); err != nil { if !bytes.Contains(out, []byte(`dpkg was interrupted`)) {
return err return fmt.Errorf("apt-get install failed: %w; output:\n%s", err, out)
}
up.Logf("apt-get install failed: %s; output:\n%s", err, out)
up.Logf("running dpkg --configure tailscale")
out, err = exec.Command("dpkg", "--force-confdef,downgrade", "--configure", "tailscale").CombinedOutput()
if err != nil {
return fmt.Errorf("dpkg --configure tailscale failed: %w; output:\n%s", err, out)
}
continue
}
break
} }
return nil return nil

@ -280,7 +280,7 @@ func handleC2NUpdatePost(b *LocalBackend, w http.ResponseWriter, r *http.Request
return return
} }
cmd := exec.Command(cmdTS, "update", "--yes") cmd := tailscaleUpdateCmd(cmdTS)
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
cmd.Stdout = buf cmd.Stdout = buf
cmd.Stderr = buf cmd.Stderr = buf
@ -412,6 +412,20 @@ func findCmdTailscale() (string, error) {
return "", errors.New("tailscale executable not found in expected place") return "", errors.New("tailscale executable not found in expected place")
} }
func tailscaleUpdateCmd(cmdTS string) *exec.Cmd {
if runtime.GOOS != "linux" {
return exec.Command(cmdTS, "update", "--yes")
}
if _, err := exec.LookPath("systemd-run"); err != nil {
return exec.Command(cmdTS, "update", "--yes")
}
// When systemd-run is available, use it to run the update command. This
// creates a new temporary unit separate from the tailscaled unit. When
// tailscaled is restarted during the update, systemd won't kill this
// temporary update unit, which could cause unexpected breakage.
return exec.Command("systemd-run", "--wait", "--pipe", "--collect", cmdTS, "update", "--yes")
}
func regularFileExists(path string) bool { func regularFileExists(path string) bool {
fi, err := os.Stat(path) fi, err := os.Stat(path)
return err == nil && fi.Mode().IsRegular() return err == nil && fi.Mode().IsRegular()