tailscale/wgengine/router/router_linux_test.go
Brad Fitzpatrick ed8587f90d wgengine/router: take a link monitor
Prep for  which will need to make Linux's router react to changes
that the link monitor observes.

The router package already depended on the monitor package
transitively. Now it's explicit.

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
2021-07-20 13:43:40 -07:00

716 lines
20 KiB
Go

// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package router
import (
"bytes"
"errors"
"fmt"
"math/rand"
"os"
"sort"
"strings"
"sync/atomic"
"testing"
"github.com/google/go-cmp/cmp"
"golang.zx2c4.com/wireguard/tun"
"inet.af/netaddr"
"tailscale.com/types/logger"
"tailscale.com/wgengine/monitor"
)
func TestRouterStates(t *testing.T) {
basic := `
ip rule add -4 pref 5210 fwmark 0x80000 table main
ip rule add -4 pref 5230 fwmark 0x80000 table default
ip rule add -4 pref 5250 fwmark 0x80000 type unreachable
ip rule add -4 pref 5270 table 52
ip rule add -6 pref 5210 fwmark 0x80000 table main
ip rule add -6 pref 5230 fwmark 0x80000 table default
ip rule add -6 pref 5250 fwmark 0x80000 type unreachable
ip rule add -6 pref 5270 table 52
`
states := []struct {
name string
in *Config
want string
}{
{
name: "no config",
in: nil,
want: `
up` + basic,
},
{
name: "local addr only",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.103/10"),
NetfilterMode: netfilterOff,
},
want: `
up
ip addr add 100.101.102.103/10 dev tailscale0` + basic,
},
{
name: "addr and routes",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.103/10"),
Routes: mustCIDRs("100.100.100.100/32", "192.168.16.0/24"),
NetfilterMode: netfilterOff,
},
want: `
up
ip addr add 100.101.102.103/10 dev tailscale0
ip route add 100.100.100.100/32 dev tailscale0 table 52
ip route add 192.168.16.0/24 dev tailscale0 table 52` + basic,
},
{
name: "addr and routes and subnet routes",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.103/10"),
Routes: mustCIDRs("100.100.100.100/32", "192.168.16.0/24"),
SubnetRoutes: mustCIDRs("200.0.0.0/8"),
NetfilterMode: netfilterOff,
},
want: `
up
ip addr add 100.101.102.103/10 dev tailscale0
ip route add 100.100.100.100/32 dev tailscale0 table 52
ip route add 192.168.16.0/24 dev tailscale0 table 52` + basic,
},
{
name: "addr and routes and subnet routes with netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
SubnetRoutes: mustCIDRs("200.0.0.0/8"),
SNATSubnetRoutes: true,
NetfilterMode: netfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 table 52
ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
`v4/filter/FORWARD -j ts-forward
v4/filter/INPUT -j ts-input
v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
v4/filter/ts-forward -o tailscale0 -j ACCEPT
v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
v4/nat/POSTROUTING -j ts-postrouting
v4/nat/ts-postrouting -m mark --mark 0x40000 -j MASQUERADE
v6/filter/FORWARD -j ts-forward
v6/filter/INPUT -j ts-input
v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v6/filter/ts-forward -o tailscale0 -j ACCEPT
v6/nat/POSTROUTING -j ts-postrouting
v6/nat/ts-postrouting -m mark --mark 0x40000 -j MASQUERADE
`,
},
{
name: "addr and routes with netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
NetfilterMode: netfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 table 52
ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
`v4/filter/FORWARD -j ts-forward
v4/filter/INPUT -j ts-input
v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
v4/filter/ts-forward -o tailscale0 -j ACCEPT
v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
v4/nat/POSTROUTING -j ts-postrouting
v6/filter/FORWARD -j ts-forward
v6/filter/INPUT -j ts-input
v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v6/filter/ts-forward -o tailscale0 -j ACCEPT
v6/nat/POSTROUTING -j ts-postrouting
`,
},
{
name: "addr and routes and subnet routes with netfilter but no SNAT",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
SubnetRoutes: mustCIDRs("200.0.0.0/8"),
SNATSubnetRoutes: false,
NetfilterMode: netfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 table 52
ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
`v4/filter/FORWARD -j ts-forward
v4/filter/INPUT -j ts-input
v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
v4/filter/ts-forward -o tailscale0 -j ACCEPT
v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
v4/nat/POSTROUTING -j ts-postrouting
v6/filter/FORWARD -j ts-forward
v6/filter/INPUT -j ts-input
v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v6/filter/ts-forward -o tailscale0 -j ACCEPT
v6/nat/POSTROUTING -j ts-postrouting
`,
},
{
name: "addr and routes with netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
NetfilterMode: netfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 table 52
ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
`v4/filter/FORWARD -j ts-forward
v4/filter/INPUT -j ts-input
v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
v4/filter/ts-forward -o tailscale0 -j ACCEPT
v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
v4/nat/POSTROUTING -j ts-postrouting
v6/filter/FORWARD -j ts-forward
v6/filter/INPUT -j ts-input
v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v6/filter/ts-forward -o tailscale0 -j ACCEPT
v6/nat/POSTROUTING -j ts-postrouting
`,
},
{
name: "addr and routes with half netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
NetfilterMode: netfilterNoDivert,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 table 52
ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
`v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
v4/filter/ts-forward -o tailscale0 -j ACCEPT
v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v6/filter/ts-forward -o tailscale0 -j ACCEPT
`,
},
{
name: "addr and routes with netfilter2",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
NetfilterMode: netfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 table 52
ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
`v4/filter/FORWARD -j ts-forward
v4/filter/INPUT -j ts-input
v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
v4/filter/ts-forward -o tailscale0 -j ACCEPT
v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
v4/nat/POSTROUTING -j ts-postrouting
v6/filter/FORWARD -j ts-forward
v6/filter/INPUT -j ts-input
v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v6/filter/ts-forward -o tailscale0 -j ACCEPT
v6/nat/POSTROUTING -j ts-postrouting
`,
},
{
name: "addr, routes, and local routes with netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "0.0.0.0/0"),
LocalRoutes: mustCIDRs("10.0.0.0/8"),
NetfilterMode: netfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 0.0.0.0/0 dev tailscale0 table 52
ip route add 100.100.100.100/32 dev tailscale0 table 52
ip route add throw 10.0.0.0/8 table 52` + basic +
`v4/filter/FORWARD -j ts-forward
v4/filter/INPUT -j ts-input
v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
v4/filter/ts-forward -o tailscale0 -j ACCEPT
v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
v4/nat/POSTROUTING -j ts-postrouting
v6/filter/FORWARD -j ts-forward
v6/filter/INPUT -j ts-input
v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
v6/filter/ts-forward -o tailscale0 -j ACCEPT
v6/nat/POSTROUTING -j ts-postrouting
`,
},
{
name: "addr, routes, and local routes with no netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "0.0.0.0/0"),
LocalRoutes: mustCIDRs("10.0.0.0/8", "192.168.0.0/24"),
NetfilterMode: netfilterOff,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 0.0.0.0/0 dev tailscale0 table 52
ip route add 100.100.100.100/32 dev tailscale0 table 52
ip route add throw 10.0.0.0/8 table 52
ip route add throw 192.168.0.0/24 table 52` + basic,
},
}
mon, err := monitor.New(logger.Discard)
if err != nil {
t.Fatal(err)
}
mon.Start()
defer mon.Close()
fake := NewFakeOS(t)
router, err := newUserspaceRouterAdvanced(t.Logf, "tailscale0", mon, fake.netfilter4, fake.netfilter6, fake, true, true)
if err != nil {
t.Fatalf("failed to create router: %v", err)
}
if err := router.Up(); err != nil {
t.Fatalf("failed to up router: %v", err)
}
testState := func(t *testing.T, i int) {
t.Helper()
if err := router.Set(states[i].in); err != nil {
t.Fatalf("failed to set router config: %v", err)
}
got := fake.String()
want := strings.TrimSpace(states[i].want)
if diff := cmp.Diff(got, want); diff != "" {
t.Fatalf("unexpected OS state (-got+want):\n%s", diff)
}
}
for i, state := range states {
t.Run(state.name, func(t *testing.T) { testState(t, i) })
}
// Cycle through a bunch of states in pseudorandom order, to
// verify that we transition cleanly from state to state no matter
// the order.
for randRun := 0; randRun < 5*len(states); randRun++ {
i := rand.Intn(len(states))
state := states[i]
t.Run(state.name, func(t *testing.T) { testState(t, i) })
}
}
type fakeNetfilter struct {
t *testing.T
n map[string][]string
}
func newNetfilter(t *testing.T) *fakeNetfilter {
return &fakeNetfilter{
t: t,
n: map[string][]string{
"filter/INPUT": nil,
"filter/OUTPUT": nil,
"filter/FORWARD": nil,
"nat/PREROUTING": nil,
"nat/OUTPUT": nil,
"nat/POSTROUTING": nil,
},
}
}
func (n *fakeNetfilter) Insert(table, chain string, pos int, args ...string) error {
k := table + "/" + chain
if rules, ok := n.n[k]; ok {
if pos > len(rules)+1 {
n.t.Errorf("bad position %d in %s", pos, k)
return errExec
}
rules = append(rules, "")
copy(rules[pos:], rules[pos-1:])
rules[pos-1] = strings.Join(args, " ")
n.n[k] = rules
} else {
n.t.Errorf("unknown table/chain %s", k)
return errExec
}
return nil
}
func (n *fakeNetfilter) Append(table, chain string, args ...string) error {
k := table + "/" + chain
return n.Insert(table, chain, len(n.n[k])+1, args...)
}
func (n *fakeNetfilter) Exists(table, chain string, args ...string) (bool, error) {
k := table + "/" + chain
if rules, ok := n.n[k]; ok {
for _, rule := range rules {
if rule == strings.Join(args, " ") {
return true, nil
}
}
return false, nil
} else {
n.t.Errorf("unknown table/chain %s", k)
return false, errExec
}
}
func (n *fakeNetfilter) Delete(table, chain string, args ...string) error {
k := table + "/" + chain
if rules, ok := n.n[k]; ok {
for i, rule := range rules {
if rule == strings.Join(args, " ") {
rules = append(rules[:i], rules[i+1:]...)
n.n[k] = rules
return nil
}
}
n.t.Errorf("delete of unknown rule %q from %s", strings.Join(args, " "), k)
return errExec
} else {
n.t.Errorf("unknown table/chain %s", k)
return errExec
}
}
func (n *fakeNetfilter) ClearChain(table, chain string) error {
k := table + "/" + chain
if _, ok := n.n[k]; ok {
n.n[k] = nil
return nil
} else {
n.t.Logf("note: ClearChain: unknown table/chain %s", k)
return errors.New("exitcode:1")
}
}
func (n *fakeNetfilter) NewChain(table, chain string) error {
k := table + "/" + chain
if _, ok := n.n[k]; ok {
n.t.Errorf("table/chain %s already exists", k)
return errExec
}
n.n[k] = nil
return nil
}
func (n *fakeNetfilter) DeleteChain(table, chain string) error {
k := table + "/" + chain
if rules, ok := n.n[k]; ok {
if len(rules) != 0 {
n.t.Errorf("%s is not empty", k)
return errExec
}
delete(n.n, k)
return nil
} else {
n.t.Errorf("%s does not exist", k)
return errExec
}
}
// fakeOS implements commandRunner and provides v4 and v6
// netfilterRunners, but captures changes without touching the OS.
type fakeOS struct {
t *testing.T
up bool
ips []string
routes []string
rules []string
netfilter4 *fakeNetfilter
netfilter6 *fakeNetfilter
}
func NewFakeOS(t *testing.T) *fakeOS {
return &fakeOS{
t: t,
netfilter4: newNetfilter(t),
netfilter6: newNetfilter(t),
}
}
var errExec = errors.New("execution failed")
func (o *fakeOS) String() string {
var b strings.Builder
if o.up {
b.WriteString("up\n")
} else {
b.WriteString("down\n")
}
for _, ip := range o.ips {
fmt.Fprintf(&b, "ip addr add %s\n", ip)
}
for _, route := range o.routes {
fmt.Fprintf(&b, "ip route add %s\n", route)
}
for _, rule := range o.rules {
fmt.Fprintf(&b, "ip rule add %s\n", rule)
}
var chains []string
for chain := range o.netfilter4.n {
chains = append(chains, chain)
}
sort.Strings(chains)
for _, chain := range chains {
for _, rule := range o.netfilter4.n[chain] {
fmt.Fprintf(&b, "v4/%s %s\n", chain, rule)
}
}
chains = nil
for chain := range o.netfilter6.n {
chains = append(chains, chain)
}
sort.Strings(chains)
for _, chain := range chains {
for _, rule := range o.netfilter6.n[chain] {
fmt.Fprintf(&b, "v6/%s %s\n", chain, rule)
}
}
return b.String()[:len(b.String())-1]
}
func (o *fakeOS) run(args ...string) error {
unexpected := func() error {
o.t.Errorf("unexpected invocation %q", strings.Join(args, " "))
return errors.New("unrecognized invocation")
}
if args[0] != "ip" {
return unexpected()
}
if len(args) == 2 && args[1] == "rule" {
// naked invocation of `ip rule` is a feature test. Return
// successfully.
return nil
}
family := ""
rest := strings.Join(args[3:], " ")
if args[1] == "-4" || args[1] == "-6" {
family = args[1]
copy(args[1:], args[2:])
args = args[:len(args)-1]
rest = family + " " + strings.Join(args[3:], " ")
}
var l *[]string
switch args[1] {
case "link":
got := strings.Join(args[2:], " ")
switch got {
case "set dev tailscale0 up":
o.up = true
case "set dev tailscale0 down":
o.up = false
default:
return unexpected()
}
return nil
case "addr":
l = &o.ips
case "route":
l = &o.routes
case "rule":
l = &o.rules
default:
return unexpected()
}
switch args[2] {
case "add":
for _, el := range *l {
if el == rest {
o.t.Errorf("can't add %q, already present", rest)
return errors.New("already exists")
}
}
*l = append(*l, rest)
sort.Strings(*l)
case "del":
found := false
for i, el := range *l {
if el == rest {
found = true
*l = append((*l)[:i], (*l)[i+1:]...)
break
}
}
if !found {
o.t.Logf("note: can't delete %q, not present", rest)
// 'ip rule del' exits with code 2 when a row is
// missing. We don't want to consider that an error,
// for cleanup purposes.
// TODO(apenwarr): this is a hack.
// I'd like to return an exec.ExitError(2) here, but
// I can't, because the ExitCode is implemented in
// os.ProcessState, which is an opaque object I can't
// instantiate or modify. Go's 75 levels of abstraction
// between me and an 8-bit int are really paying off
// here, as you can see.
return errors.New("exitcode:2")
}
default:
return unexpected()
}
return nil
}
func (o *fakeOS) output(args ...string) ([]byte, error) {
want := "ip rule list priority 10000"
got := strings.Join(args, " ")
if got != want {
o.t.Errorf("unexpected command that wants output: %v", got)
return nil, errExec
}
var ret []string
for _, rule := range o.rules {
if strings.Contains(rule, "10000") {
ret = append(ret, rule)
}
}
return []byte(strings.Join(ret, "\n")), nil
}
var tunTestNum int64
func createTestTUN(t *testing.T) tun.Device {
const minimalMTU = 1280
tunName := fmt.Sprintf("tuntest%d", atomic.AddInt64(&tunTestNum, 1))
tun, err := tun.CreateTUN(tunName, minimalMTU)
if err != nil {
t.Fatalf("CreateTUN(%q): %v", tunName, err)
}
return tun
}
func TestDelRouteIdempotent(t *testing.T) {
if os.Getuid() != 0 {
t.Skip("test requires root")
}
tun := createTestTUN(t)
defer tun.Close()
var logOutput bytes.Buffer
logf := func(format string, args ...interface{}) {
fmt.Fprintf(&logOutput, format, args...)
if !bytes.HasSuffix(logOutput.Bytes(), []byte("\n")) {
logOutput.WriteByte('\n')
}
}
mon, err := monitor.New(logger.Discard)
if err != nil {
t.Fatal(err)
}
mon.Start()
defer mon.Close()
r, err := newUserspaceRouter(logf, tun, mon)
if err != nil {
t.Fatal(err)
}
lr := r.(*linuxRouter)
if err := lr.upInterface(); err != nil {
t.Fatal(err)
}
for _, s := range []string{
"192.0.2.0/24", // RFC 5737
"2001:DB8::/32", // RFC 3849
} {
cidr := netaddr.MustParseIPPrefix(s)
if err := lr.addRoute(cidr); err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
if err := lr.delRoute(cidr); err != nil {
t.Fatalf("delRoute(i=%d): %v", i, err)
}
}
}
wantSubs := map[string]int{
"warning: tried to delete route 192.0.2.0/24 but it was already gone; ignoring error": 1,
"warning: tried to delete route 2001:db8::/32 but it was already gone; ignoring error": 1,
}
out := logOutput.String()
for sub, want := range wantSubs {
got := strings.Count(out, sub)
if got != want {
t.Errorf("log output substring %q occurred %d time; want %d", sub, got, want)
}
}
if t.Failed() {
t.Logf("Log output:\n%s", out)
}
}