// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import (
"context"
"fmt"
"io"
"net/http"
"net/netip"
"reflect"
"strings"
"sync"
"testing"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/kubetypes"
)
func Test_updatesForSvc(t *testing.T) {
tailnetIPv4, tailnetIPv6 := netip.MustParseAddr("100.99.99.99"), netip.MustParseAddr("fd7a:115c:a1e0::701:b62a")
tailnetIPv4_1, tailnetIPv6_1 := netip.MustParseAddr("100.88.88.88"), netip.MustParseAddr("fd7a:115c:a1e0::4101:512f")
ports := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {}}
ports1 := map[egressservices.PortMap]struct{}{{Protocol: "udp", MatchPort: 4004, TargetPort: 53}: {}}
ports2 := map[egressservices.PortMap]struct{}{{Protocol: "tcp", MatchPort: 4003, TargetPort: 80}: {},
{Protocol: "tcp", MatchPort: 4005, TargetPort: 443}: {}}
fqdnSpec := egressservices.Config{
TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
Ports: ports,
}
fqdnSpec1 := egressservices.Config{
TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
Ports: ports1,
}
fqdnSpec2 := egressservices.Config{
TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
Ports: ports,
}
fqdnSpec3 := egressservices.Config{
TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
Ports: ports2,
}
r := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4}
r1 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6}
r2 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv4}
r3 := rule{tailnetPort: 53, containerPort: 4004, protocol: "udp", tailnetIP: tailnetIPv6}
r4 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv4_1}
r5 := rule{containerPort: 4003, tailnetPort: 80, protocol: "tcp", tailnetIP: tailnetIPv6_1}
r6 := rule{containerPort: 4005, tailnetPort: 443, protocol: "tcp", tailnetIP: tailnetIPv4}
tests := []struct {
name string
svcName string
tailnetTargetIPs []netip.Addr
podIP string
spec egressservices.Config
status *egressservices.Status
wantRulesToAdd []rule
wantRulesToDelete []rule
}{
{
name: "add_fqdn_svc_that_does_not_yet_exist",
svcName: "test",
tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
spec: fqdnSpec,
status: &egressservices.Status{},
wantRulesToAdd: []rule{r, r1},
wantRulesToDelete: []rule{},
},
{
name: "fqdn_svc_already_exists",
svcName: "test",
tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
spec: fqdnSpec,
status: &egressservices.Status{
Services: map[string]*egressservices.ServiceStatus{"test": {
TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
Ports: ports,
}}},
wantRulesToAdd: []rule{},
wantRulesToDelete: []rule{},
},
{
name: "fqdn_svc_already_exists_add_port_remove_port",
svcName: "test",
tailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
spec: fqdnSpec1,
status: &egressservices.Status{
Services: map[string]*egressservices.ServiceStatus{"test": {
TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
Ports: ports,
}}},
wantRulesToAdd: []rule{r2, r3},
wantRulesToDelete: []rule{r, r1},
},
{
name: "fqdn_svc_already_exists_change_fqdn_backend_ips",
svcName: "test",
tailnetTargetIPs: []netip.Addr{tailnetIPv4_1, tailnetIPv6_1},
spec: fqdnSpec,
status: &egressservices.Status{
Services: map[string]*egressservices.ServiceStatus{"test": {
TailnetTargetIPs: []netip.Addr{tailnetIPv4, tailnetIPv6},
TailnetTarget: egressservices.TailnetTarget{FQDN: "test"},
Ports: ports,
}}},
wantRulesToAdd: []rule{r4, r5},
wantRulesToDelete: []rule{r, r1},
},
{
name: "add_ip_service",
svcName: "test",
tailnetTargetIPs: []netip.Addr{tailnetIPv4},
spec: fqdnSpec2,
status: &egressservices.Status{},
wantRulesToAdd: []rule{r},
wantRulesToDelete: []rule{},
},
{
name: "add_ip_service_already_exists",
svcName: "test",
tailnetTargetIPs: []netip.Addr{tailnetIPv4},
spec: fqdnSpec2,
status: &egressservices.Status{
Services: map[string]*egressservices.ServiceStatus{"test": {
TailnetTargetIPs: []netip.Addr{tailnetIPv4},
TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
Ports: ports,
}}},
wantRulesToAdd: []rule{},
wantRulesToDelete: []rule{},
},
{
name: "ip_service_add_port",
svcName: "test",
tailnetTargetIPs: []netip.Addr{tailnetIPv4},
spec: fqdnSpec3,
status: &egressservices.Status{
Services: map[string]*egressservices.ServiceStatus{"test": {
TailnetTargetIPs: []netip.Addr{tailnetIPv4},
TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
Ports: ports,
}}},
wantRulesToAdd: []rule{r6},
wantRulesToDelete: []rule{},
},
{
name: "ip_service_delete_port",
svcName: "test",
tailnetTargetIPs: []netip.Addr{tailnetIPv4},
spec: fqdnSpec,
status: &egressservices.Status{
Services: map[string]*egressservices.ServiceStatus{"test": {
TailnetTargetIPs: []netip.Addr{tailnetIPv4},
TailnetTarget: egressservices.TailnetTarget{IP: tailnetIPv4.String()},
Ports: ports2,
}}},
wantRulesToAdd: []rule{},
wantRulesToDelete: []rule{r6},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotRulesToAdd, gotRulesToDelete, err := updatesForCfg(tt.svcName, tt.spec, tt.status, tt.tailnetTargetIPs)
if err != nil {
t.Errorf("updatesForSvc() unexpected error %v", err)
return
}
if !reflect.DeepEqual(gotRulesToAdd, tt.wantRulesToAdd) {
t.Errorf("updatesForSvc() got rulesToAdd = \n%v\n want rulesToAdd \n%v", gotRulesToAdd, tt.wantRulesToAdd)
}
if !reflect.DeepEqual(gotRulesToDelete, tt.wantRulesToDelete) {
t.Errorf("updatesForSvc() got rulesToDelete = \n%v\n want rulesToDelete \n%v", gotRulesToDelete, tt.wantRulesToDelete)
}
})
}
}
// A failure of this test will most likely look like a timeout.
func TestWaitTillSafeToShutdown(t *testing.T) {
podIP := "10.0.0.1"
anotherIP := "10.0.0.2"
tests := []struct {
name string
// services is a map of service name to the number of calls to make to the healthcheck endpoint before
// returning a response that does NOT contain this Pod's IP in headers.
services map[string]int
replicas int
healthCheckSet bool
}{
{
name: "no_configs",
},
{
name: "one_service_immediately_safe_to_shutdown",
services: map[string]int{
"svc1": 0,
},
replicas: 2,
healthCheckSet: true,
},
{
name: "multiple_services_immediately_safe_to_shutdown",
services: map[string]int{
"svc1": 0,
"svc2": 0,
"svc3": 0,
},
replicas: 2,
healthCheckSet: true,
},
{
name: "multiple_services_no_healthcheck_endpoints",
services: map[string]int{
"svc1": 0,
"svc2": 0,
"svc3": 0,
},
replicas: 2,
},
{
name: "one_service_eventually_safe_to_shutdown",
services: map[string]int{
"svc1": 3, // After 3 calls to health check endpoint, no longer returns this Pod's IP
},
replicas: 2,
healthCheckSet: true,
},
{
name: "multiple_services_eventually_safe_to_shutdown",
services: map[string]int{
"svc1": 1, // After 1 call to health check endpoint, no longer returns this Pod's IP
"svc2": 3, // After 3 calls to health check endpoint, no longer returns this Pod's IP
"svc3": 5, // After 5 calls to the health check endpoint, no longer returns this Pod's IP
},
replicas: 2,
healthCheckSet: true,
},
{
name: "multiple_services_eventually_safe_to_shutdown_with_higher_replica_count",
services: map[string]int{
"svc1": 7,
"svc2": 10,
},
replicas: 5,
healthCheckSet: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfgs := &egressservices.Configs{}
switches := make(map[string]int)
for svc, callsToSwitch := range tt.services {
endpoint := fmt.Sprintf("http://%s.local", svc)
if tt.healthCheckSet {
(*cfgs)[svc] = egressservices.Config{
HealthCheckEndpoint: endpoint,
}
}
switches[endpoint] = callsToSwitch
}
ep := &egressProxy{
podIPv4: podIP,
client: &mockHTTPClient{
podIP: podIP,
anotherIP: anotherIP,
switches: switches,
},
}
ep.waitTillSafeToShutdown(context.Background(), cfgs, tt.replicas)
})
}
}
// mockHTTPClient is a client that receives an HTTP call for an egress service endpoint and returns a response with an
// IP address in a 'Pod-IPv4' header. It can be configured to return one IP address for N calls, then switch to another
// IP address to simulate a scenario where an IP is eventually no longer a backend for an endpoint.
// TODO(irbekrm): to test this more thoroughly, we should have the client take into account the number of replicas and
// return as if traffic was round robin load balanced across different Pods.
type mockHTTPClient struct {
// podIP - initial IP address to return, that matches the current proxy's IP address.
podIP string
anotherIP string
// after how many calls to an endpoint, the client should start returning 'anotherIP' instead of 'podIP.
switches map[string]int
mu sync.Mutex // protects the following
// calls tracks the number of calls received.
calls map[string]int
}
func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
m.mu.Lock()
if m.calls == nil {
m.calls = make(map[string]int)
}
endpoint := req.URL.String()
m.calls[endpoint]++
calls := m.calls[endpoint]
m.mu.Unlock()
resp := &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader("")),
}
if calls <= m.switches[endpoint] {
resp.Header.Set(kubetypes.PodIPv4Header, m.podIP) // Pod is still routable
} else {
resp.Header.Set(kubetypes.PodIPv4Header, m.anotherIP) // Pod is no longer routable
}
return resp, nil
}