mirror of
https://github.com/tailscale/tailscale.git
synced 2024-11-29 04:55:31 +00:00
cmd/tailscale, ipn/ipnlocal: add suggest exit node CLI option (#11407)
Updates tailscale/corp#17516 Signed-off-by: Claire Wang <claire@tailscale.com>
This commit is contained in:
parent
449f46c207
commit
9171b217ba
@ -49,3 +49,11 @@ type ReloadConfigResponse struct {
|
||||
Reloaded bool // whether the config was reloaded
|
||||
Err string // any error message
|
||||
}
|
||||
|
||||
// ExitNodeSuggestionResponse is the response to a LocalAPI suggest-exit-node GET request.
|
||||
// It returns the StableNodeID, name, and location of a suggested exit node for the client making the request.
|
||||
type ExitNodeSuggestionResponse struct {
|
||||
ID tailcfg.StableNodeID
|
||||
Name string
|
||||
Location tailcfg.LocationView `json:",omitempty"`
|
||||
}
|
||||
|
@ -1514,3 +1514,12 @@ func (w *IPNBusWatcher) Next() (ipn.Notify, error) {
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// SuggestExitNode requests an exit node suggestion and returns the exit node's details.
|
||||
func (lc *LocalClient) SuggestExitNode(ctx context.Context) (apitype.ExitNodeSuggestionResponse, error) {
|
||||
body, err := lc.get200(ctx, "/localapi/v0/suggest-exit-node")
|
||||
if err != nil {
|
||||
return apitype.ExitNodeSuggestionResponse{}, err
|
||||
}
|
||||
return decodeJSON[apitype.ExitNodeSuggestionResponse](body)
|
||||
}
|
||||
|
@ -40,6 +40,12 @@ func exitNodeCmd() *ffcli.Command {
|
||||
fs.StringVar(&exitNodeArgs.filter, "filter", "", "filter exit nodes by country")
|
||||
return fs
|
||||
})(),
|
||||
},
|
||||
{
|
||||
Name: "suggest",
|
||||
ShortUsage: "tailscale exit-node suggest",
|
||||
ShortHelp: "Suggests the best available exit node",
|
||||
Exec: runExitNodeSuggest,
|
||||
}},
|
||||
(func() []*ffcli.Command {
|
||||
if !envknob.UseWIPCode() {
|
||||
@ -134,11 +140,37 @@ func runExitNodeList(ctx context.Context, args []string) error {
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "# To use an exit node, use `tailscale set --exit-node=` followed by the hostname or IP")
|
||||
|
||||
fmt.Fprintln(w, "# To use an exit node, use `tailscale set --exit-node=` followed by the hostname or IP.")
|
||||
if hasAnyExitNodeSuggestions(peers) {
|
||||
fmt.Fprintln(w, "# To have Tailscale suggest an exit node, use `tailscale exit-node suggest`.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runExitNodeSuggest returns a suggested exit node ID to connect to and shows the chosen exit node tailcfg.StableNodeID.
|
||||
// If there are no derp based exit nodes to choose from or there is a failure in finding a suggestion, the command will return an error indicating so.
|
||||
func runExitNodeSuggest(ctx context.Context, args []string) error {
|
||||
res, err := localClient.SuggestExitNode(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("suggest exit node: %w", err)
|
||||
}
|
||||
if res.ID == "" {
|
||||
fmt.Println("No exit node suggestion is available.")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("Suggested exit node: %v\nTo accept this suggestion, use `tailscale set --exit-node=%v`.\n", res.Name, res.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasAnyExitNodeSuggestions(peers []*ipnstate.PeerStatus) bool {
|
||||
for _, peer := range peers {
|
||||
if peer.HasCap(tailcfg.NodeAttrSuggestExitNode) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// peerStatus returns a string representing the current state of
|
||||
// a peer. If there is no notable state, a - is returned.
|
||||
func peerStatus(peer *ipnstate.PeerStatus) string {
|
||||
|
@ -297,7 +297,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
|
||||
tailscale.com/net/flowtrack from tailscale.com/net/packet+
|
||||
💣 tailscale.com/net/interfaces from tailscale.com/cmd/tailscaled+
|
||||
tailscale.com/net/netaddr from tailscale.com/ipn+
|
||||
tailscale.com/net/netcheck from tailscale.com/wgengine/magicsock
|
||||
tailscale.com/net/netcheck from tailscale.com/wgengine/magicsock+
|
||||
tailscale.com/net/neterror from tailscale.com/net/dns/resolver+
|
||||
tailscale.com/net/netkernelconf from tailscale.com/ipn/ipnlocal
|
||||
tailscale.com/net/netknob from tailscale.com/logpolicy+
|
||||
|
@ -14,6 +14,7 @@
|
||||
"log"
|
||||
"maps"
|
||||
"math"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
@ -59,6 +60,7 @@
|
||||
"tailscale.com/net/dnscache"
|
||||
"tailscale.com/net/dnsfallback"
|
||||
"tailscale.com/net/interfaces"
|
||||
"tailscale.com/net/netcheck"
|
||||
"tailscale.com/net/netkernelconf"
|
||||
"tailscale.com/net/netmon"
|
||||
"tailscale.com/net/netns"
|
||||
@ -6295,3 +6297,217 @@ func mayDeref[T any](p *T) (v T) {
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
var ErrNoPreferredDERP = errors.New("no preferred DERP, try again later")
|
||||
|
||||
// SuggestExitNode computes a suggestion based on the current netmap and last netcheck report. If
|
||||
// there are multiple equally good options, one is selected at random, so the result is not stable. To be
|
||||
// eligible for consideration, the peer must have NodeAttrSuggestExitNode in its CapMap.
|
||||
//
|
||||
// Currently, peers with a DERP home are preferred over those without (typically this means Mullvad).
|
||||
// Peers are selected based on having a DERP home that is the lowest latency to this device. For peers
|
||||
// without a DERP home, we look for geographic proximity to this device's DERP home.
|
||||
func (b *LocalBackend) SuggestExitNode() (response apitype.ExitNodeSuggestionResponse, err error) {
|
||||
b.mu.Lock()
|
||||
lastReport := b.MagicConn().GetLastNetcheckReport(b.ctx)
|
||||
netMap := b.netMap
|
||||
b.mu.Unlock()
|
||||
seed := time.Now().UnixNano()
|
||||
r := rand.New(rand.NewSource(seed))
|
||||
return suggestExitNode(lastReport, netMap, r)
|
||||
}
|
||||
|
||||
func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, r *rand.Rand) (res apitype.ExitNodeSuggestionResponse, err error) {
|
||||
if report.PreferredDERP == 0 {
|
||||
return res, ErrNoPreferredDERP
|
||||
}
|
||||
candidates := make([]tailcfg.NodeView, 0, len(netMap.Peers))
|
||||
for _, peer := range netMap.Peers {
|
||||
if peer.CapMap().Has(tailcfg.NodeAttrSuggestExitNode) && tsaddr.ContainsExitRoutes(peer.AllowedIPs()) {
|
||||
candidates = append(candidates, peer)
|
||||
}
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return res, nil
|
||||
}
|
||||
if len(candidates) == 1 {
|
||||
peer := candidates[0]
|
||||
if hi := peer.Hostinfo(); hi.Valid() {
|
||||
if loc := hi.Location(); loc != nil {
|
||||
res.Location = loc.View()
|
||||
}
|
||||
}
|
||||
res.ID = peer.StableID()
|
||||
res.Name = peer.Name()
|
||||
return res, nil
|
||||
}
|
||||
|
||||
candidatesByRegion := make(map[int][]tailcfg.NodeView, len(netMap.DERPMap.Regions))
|
||||
var preferredDERP *tailcfg.DERPRegion = netMap.DERPMap.Regions[report.PreferredDERP]
|
||||
var minDistance float64 = math.MaxFloat64
|
||||
type nodeDistance struct {
|
||||
nv tailcfg.NodeView
|
||||
distance float64 // in meters, approximately
|
||||
}
|
||||
distances := make([]nodeDistance, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
if !c.Valid() {
|
||||
continue
|
||||
}
|
||||
if c.DERP() != "" {
|
||||
ipp, err := netip.ParseAddrPort(c.DERP())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if ipp.Addr() != tailcfg.DerpMagicIPAddr {
|
||||
continue
|
||||
}
|
||||
regionID := int(ipp.Port())
|
||||
candidatesByRegion[regionID] = append(candidatesByRegion[regionID], c)
|
||||
continue
|
||||
}
|
||||
if len(candidatesByRegion) > 0 {
|
||||
// Since a candidate exists that does have a DERP home, skip this candidate. We never select
|
||||
// a candidate without a DERP home if there is a candidate available with a DERP home.
|
||||
continue
|
||||
}
|
||||
// This candidate does not have a DERP home.
|
||||
// Use geographic distance from our DERP home to estimate how good this candidate is.
|
||||
hi := c.Hostinfo()
|
||||
if !hi.Valid() {
|
||||
continue
|
||||
}
|
||||
loc := hi.Location()
|
||||
if loc == nil {
|
||||
continue
|
||||
}
|
||||
distance := longLatDistance(preferredDERP.Latitude, preferredDERP.Longitude, loc.Latitude, loc.Longitude)
|
||||
if distance < minDistance {
|
||||
minDistance = distance
|
||||
}
|
||||
distances = append(distances, nodeDistance{nv: c, distance: distance})
|
||||
}
|
||||
// First, try to select an exit node that has the closest DERP home, based on lastReport's DERP latency.
|
||||
// If there are no latency values, it returns an arbitrary region
|
||||
if len(candidatesByRegion) > 0 {
|
||||
minRegion := minLatencyDERPRegion(xmaps.Keys(candidatesByRegion), report)
|
||||
if minRegion == 0 {
|
||||
minRegion = randomRegion(xmaps.Keys(candidatesByRegion), r)
|
||||
}
|
||||
regionCandidates, ok := candidatesByRegion[minRegion]
|
||||
if !ok {
|
||||
return res, errors.New("no candidates in expected region: this is a bug")
|
||||
}
|
||||
chosen := randomNode(regionCandidates, r)
|
||||
res.ID = chosen.StableID()
|
||||
res.Name = chosen.Name()
|
||||
if hi := chosen.Hostinfo(); hi.Valid() {
|
||||
if loc := hi.Location(); loc != nil {
|
||||
res.Location = loc.View()
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
// None of the candidates have a DERP home, so proceed to select based on geographical distance from our preferred DERP region.
|
||||
|
||||
// allowanceMeters is the extra distance that will be permitted when considering peers. By this point, there
|
||||
// are multiple approximations taking place (DERP location standing in for this device's location, the peer's
|
||||
// location may only be city granularity, the distance algorithm assumes a spherical planet, etc.) so it is
|
||||
// reasonable to consider peers that are similar distances. Those peers are good enough to be within
|
||||
// measurement error. 100km corresponds to approximately 1ms of additional round trip light
|
||||
// propagation delay in a fiber optic cable and seems like a reasonable heuristic. It may be adjusted in
|
||||
// future.
|
||||
const allowanceMeters = 100000
|
||||
pickFrom := make([]tailcfg.NodeView, 0, len(distances))
|
||||
for _, candidate := range distances {
|
||||
if candidate.nv.Valid() && candidate.distance <= minDistance+allowanceMeters {
|
||||
pickFrom = append(pickFrom, candidate.nv)
|
||||
}
|
||||
}
|
||||
chosen := pickWeighted(pickFrom)
|
||||
if !chosen.Valid() {
|
||||
return res, errors.New("chosen candidate invalid: this is a bug")
|
||||
}
|
||||
res.ID = chosen.StableID()
|
||||
res.Name = chosen.Name()
|
||||
if hi := chosen.Hostinfo(); hi.Valid() {
|
||||
if loc := hi.Location(); loc != nil {
|
||||
res.Location = loc.View()
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// pickWeighted chooses the node with highest priority given a list of mullvad nodes.
|
||||
func pickWeighted(candidates []tailcfg.NodeView) tailcfg.NodeView {
|
||||
maxWeight := 0
|
||||
var best tailcfg.NodeView
|
||||
for _, c := range candidates {
|
||||
hi := c.Hostinfo()
|
||||
if !hi.Valid() {
|
||||
continue
|
||||
}
|
||||
loc := hi.Location()
|
||||
if loc == nil || loc.Priority <= maxWeight {
|
||||
continue
|
||||
}
|
||||
maxWeight = loc.Priority
|
||||
best = c
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// randomNode chooses a node randomly given a list of nodes and a *rand.Rand.
|
||||
func randomNode(nodes []tailcfg.NodeView, r *rand.Rand) tailcfg.NodeView {
|
||||
return nodes[r.Intn(len(nodes))]
|
||||
}
|
||||
|
||||
// randomRegion chooses a region randomly given a list of ints and a *rand.Rand
|
||||
func randomRegion(regions []int, r *rand.Rand) int {
|
||||
if testenv.InTest() {
|
||||
regions = slices.Clone(regions)
|
||||
slices.Sort(regions)
|
||||
}
|
||||
return regions[r.Intn(len(regions))]
|
||||
}
|
||||
|
||||
// minLatencyDERPRegion returns the region with the lowest latency value given the last netcheck report.
|
||||
// If there are no latency values, it returns 0.
|
||||
func minLatencyDERPRegion(regions []int, report *netcheck.Report) int {
|
||||
min := slices.MinFunc(regions, func(i, j int) int {
|
||||
const largeDuration time.Duration = math.MaxInt64
|
||||
iLatency, ok := report.RegionLatency[i]
|
||||
if !ok {
|
||||
iLatency = largeDuration
|
||||
}
|
||||
jLatency, ok := report.RegionLatency[j]
|
||||
if !ok {
|
||||
jLatency = largeDuration
|
||||
}
|
||||
if c := cmp.Compare(iLatency, jLatency); c != 0 {
|
||||
return c
|
||||
}
|
||||
return cmp.Compare(i, j)
|
||||
})
|
||||
latency, ok := report.RegionLatency[min]
|
||||
if !ok || latency == 0 {
|
||||
return 0
|
||||
} else {
|
||||
return min
|
||||
}
|
||||
}
|
||||
|
||||
// longLatDistance returns an estimated distance given the geographic coordinates of two locations, in degrees.
|
||||
// The coordinates are separated into four separate float64 values.
|
||||
// Value is returned in meters.
|
||||
func longLatDistance(fromLat, fromLong, toLat, toLong float64) float64 {
|
||||
const toRadians = math.Pi / 180
|
||||
diffLat := (fromLat - toLat) * toRadians
|
||||
diffLong := (fromLong - toLong) * toRadians
|
||||
lat1 := fromLat * toRadians
|
||||
lat2 := toLat * toRadians
|
||||
a := math.Pow(math.Sin(diffLat/2), 2) + math.Cos(lat1)*math.Cos(lat2)*math.Pow(math.Sin(diffLong/2), 2)
|
||||
const earthRadiusMeters = 6371000
|
||||
c := 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))
|
||||
return earthRadiusMeters * c
|
||||
}
|
||||
|
@ -8,6 +8,8 @@
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
@ -29,6 +31,7 @@
|
||||
"tailscale.com/ipn"
|
||||
"tailscale.com/ipn/store/mem"
|
||||
"tailscale.com/net/interfaces"
|
||||
"tailscale.com/net/netcheck"
|
||||
"tailscale.com/net/tsaddr"
|
||||
"tailscale.com/tailcfg"
|
||||
"tailscale.com/tsd"
|
||||
@ -2634,3 +2637,734 @@ func (b *LocalBackend) SetPrefsForTest(newp *ipn.Prefs) {
|
||||
defer unlock()
|
||||
b.setPrefsLockedOnEntry(newp, unlock)
|
||||
}
|
||||
|
||||
func TestSuggestExitNode(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
lastReport netcheck.Report
|
||||
netMap netmap.NetworkMap
|
||||
wantID tailcfg.StableNodeID
|
||||
wantName string
|
||||
wantLocation tailcfg.LocationView
|
||||
wantError error
|
||||
}{
|
||||
{
|
||||
name: "2 exit nodes in same region",
|
||||
lastReport: netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 10 * time.Millisecond,
|
||||
2: 20 * time.Millisecond,
|
||||
3: 30 * time.Millisecond,
|
||||
},
|
||||
PreferredDERP: 1,
|
||||
},
|
||||
netMap: netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{
|
||||
Addresses: []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.1.1/32"),
|
||||
netip.MustParsePrefix("fe70::1/128"),
|
||||
},
|
||||
}).View(),
|
||||
DERPMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {},
|
||||
2: {},
|
||||
3: {},
|
||||
},
|
||||
},
|
||||
Peers: []tailcfg.NodeView{
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
Name: "2",
|
||||
StableID: "2",
|
||||
DERP: "127.3.3.40:1",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
(&tailcfg.Node{
|
||||
ID: 3,
|
||||
Name: "3",
|
||||
StableID: "3",
|
||||
DERP: "127.3.3.40:1",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
},
|
||||
},
|
||||
wantName: "3",
|
||||
wantID: tailcfg.StableNodeID("3"),
|
||||
},
|
||||
{
|
||||
name: "2 derp based exit nodes, different regions, no latency measurements",
|
||||
lastReport: netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 0,
|
||||
2: 0,
|
||||
3: 0,
|
||||
},
|
||||
PreferredDERP: 1,
|
||||
},
|
||||
netMap: netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{
|
||||
Addresses: []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.1.1/32"),
|
||||
netip.MustParsePrefix("fe70::1/128"),
|
||||
},
|
||||
}).View(),
|
||||
DERPMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {},
|
||||
2: {},
|
||||
3: {},
|
||||
},
|
||||
},
|
||||
Peers: []tailcfg.NodeView{
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
Name: "2",
|
||||
DERP: "127.3.3.40:2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
(&tailcfg.Node{
|
||||
ID: 3,
|
||||
StableID: "3",
|
||||
Name: "3",
|
||||
DERP: "127.3.3.40:3",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
},
|
||||
},
|
||||
wantName: "3",
|
||||
wantID: tailcfg.StableNodeID("3"),
|
||||
},
|
||||
{
|
||||
name: "2 derp based exit nodes, different regions, same latency",
|
||||
lastReport: netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 10,
|
||||
2: 10,
|
||||
3: 0,
|
||||
},
|
||||
PreferredDERP: 1,
|
||||
},
|
||||
netMap: netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{
|
||||
Addresses: []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.1.1/32"),
|
||||
netip.MustParsePrefix("fe70::1/128"),
|
||||
},
|
||||
}).View(),
|
||||
DERPMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {},
|
||||
2: {},
|
||||
3: {},
|
||||
},
|
||||
},
|
||||
Peers: []tailcfg.NodeView{
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
Name: "2",
|
||||
DERP: "127.3.3.40:1",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
(&tailcfg.Node{
|
||||
ID: 3,
|
||||
StableID: "3",
|
||||
Name: "3",
|
||||
DERP: "127.3.3.40:2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
},
|
||||
},
|
||||
wantName: "2",
|
||||
wantID: tailcfg.StableNodeID("2"),
|
||||
},
|
||||
{
|
||||
name: "mullvad nodes, no derp based exit nodes",
|
||||
lastReport: netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 0,
|
||||
2: 0,
|
||||
3: 0,
|
||||
},
|
||||
PreferredDERP: 1,
|
||||
},
|
||||
netMap: netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{
|
||||
Addresses: []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.1.1/32"),
|
||||
netip.MustParsePrefix("fe70::1/128"),
|
||||
},
|
||||
}).View(),
|
||||
DERPMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {
|
||||
Latitude: 40.73061,
|
||||
Longitude: -73.935242,
|
||||
},
|
||||
2: {},
|
||||
3: {},
|
||||
},
|
||||
},
|
||||
Peers: []tailcfg.NodeView{
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Name: "Dallas",
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Latitude: 32.89748,
|
||||
Longitude: -97.040443,
|
||||
Priority: 100,
|
||||
},
|
||||
}).View(),
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
(&tailcfg.Node{
|
||||
ID: 3,
|
||||
StableID: "3",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Name: "San Jose",
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Latitude: 37.3382082,
|
||||
Longitude: -121.8863286,
|
||||
Priority: 20,
|
||||
},
|
||||
}).View(),
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
},
|
||||
},
|
||||
wantID: tailcfg.StableNodeID("2"),
|
||||
wantLocation: (&tailcfg.Location{
|
||||
Latitude: 32.89748,
|
||||
Longitude: -97.040443,
|
||||
Priority: 100,
|
||||
}).View(),
|
||||
wantName: "Dallas",
|
||||
},
|
||||
{
|
||||
name: "mullvad nodes close to each other, different priorities",
|
||||
lastReport: netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 0,
|
||||
2: 0,
|
||||
3: 0,
|
||||
},
|
||||
PreferredDERP: 1,
|
||||
},
|
||||
netMap: netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{
|
||||
Addresses: []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.1.1/32"),
|
||||
netip.MustParsePrefix("fe70::1/128"),
|
||||
},
|
||||
}).View(),
|
||||
DERPMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {
|
||||
Latitude: 40.73061,
|
||||
Longitude: -73.935242,
|
||||
},
|
||||
2: {},
|
||||
3: {},
|
||||
},
|
||||
},
|
||||
Peers: []tailcfg.NodeView{
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Name: "Dallas",
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Latitude: 32.89748,
|
||||
Longitude: -97.040443,
|
||||
Priority: 10,
|
||||
},
|
||||
}).View(),
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
(&tailcfg.Node{
|
||||
ID: 3,
|
||||
StableID: "3",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Name: "Fort Worth",
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Latitude: 37.768799,
|
||||
Longitude: -97.309341,
|
||||
Priority: 50,
|
||||
},
|
||||
}).View(),
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
},
|
||||
},
|
||||
wantID: tailcfg.StableNodeID("3"),
|
||||
wantLocation: (&tailcfg.Location{
|
||||
Latitude: 37.768799,
|
||||
Longitude: -97.309341,
|
||||
Priority: 50,
|
||||
}).View(),
|
||||
wantName: "Fort Worth",
|
||||
},
|
||||
{
|
||||
name: "mullvad nodes, no preferred derp region exit nodes",
|
||||
lastReport: netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 0,
|
||||
2: 0,
|
||||
3: 0,
|
||||
},
|
||||
PreferredDERP: 1,
|
||||
},
|
||||
netMap: netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{
|
||||
Addresses: []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.1.1/32"),
|
||||
netip.MustParsePrefix("fe70::1/128"),
|
||||
},
|
||||
}).View(),
|
||||
DERPMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {
|
||||
Latitude: 40.73061,
|
||||
Longitude: -73.935242,
|
||||
},
|
||||
2: {},
|
||||
3: {},
|
||||
},
|
||||
},
|
||||
Peers: []tailcfg.NodeView{
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Name: "Dallas",
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Latitude: 32.89748,
|
||||
Longitude: -97.040443,
|
||||
Priority: 20,
|
||||
},
|
||||
}).View(),
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
(&tailcfg.Node{
|
||||
ID: 3,
|
||||
StableID: "3",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Name: "San Jose",
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Latitude: 37.3382082,
|
||||
Longitude: -121.8863286,
|
||||
Priority: 30,
|
||||
},
|
||||
}).View(),
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
(&tailcfg.Node{
|
||||
ID: 3,
|
||||
StableID: "3",
|
||||
Name: "3",
|
||||
DERP: "127.3.3.40:2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
|
||||
tailcfg.NodeAttrSuggestExitNode: {},
|
||||
}),
|
||||
}).View(),
|
||||
},
|
||||
},
|
||||
wantID: tailcfg.StableNodeID("3"),
|
||||
wantName: "3",
|
||||
},
|
||||
{
|
||||
name: "no mullvad nodes; no derp nodes",
|
||||
lastReport: netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 0,
|
||||
2: 0,
|
||||
3: 0,
|
||||
},
|
||||
PreferredDERP: 1,
|
||||
},
|
||||
netMap: netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{
|
||||
Addresses: []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.1.1/32"),
|
||||
netip.MustParsePrefix("fe70::1/128"),
|
||||
},
|
||||
}).View(),
|
||||
DERPMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {},
|
||||
2: {},
|
||||
3: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no preferred derp region",
|
||||
lastReport: netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 0,
|
||||
2: -1,
|
||||
3: 0,
|
||||
},
|
||||
},
|
||||
netMap: netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{
|
||||
Addresses: []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.1.1/32"),
|
||||
netip.MustParsePrefix("fe70::1/128"),
|
||||
},
|
||||
}).View(),
|
||||
DERPMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {},
|
||||
2: {},
|
||||
3: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantError: ErrNoPreferredDERP,
|
||||
},
|
||||
{
|
||||
name: "derp exit node and mullvad exit node both with no suggest exit node attribute",
|
||||
lastReport: netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 0,
|
||||
2: 0,
|
||||
3: 0,
|
||||
},
|
||||
PreferredDERP: 1,
|
||||
},
|
||||
netMap: netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{
|
||||
Addresses: []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.1.1/32"),
|
||||
netip.MustParsePrefix("fe70::1/128"),
|
||||
},
|
||||
}).View(),
|
||||
DERPMap: &tailcfg.DERPMap{
|
||||
Regions: map[int]*tailcfg.DERPRegion{
|
||||
1: {},
|
||||
2: {},
|
||||
3: {},
|
||||
},
|
||||
},
|
||||
Peers: []tailcfg.NodeView{
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
Name: "2",
|
||||
DERP: "127.3.3.40:1",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
}).View(),
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Name: "Dallas",
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Latitude: 32.89748,
|
||||
Longitude: -97.040443,
|
||||
Priority: 30,
|
||||
},
|
||||
}).View(),
|
||||
}).View(),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := rand.New(rand.NewSource(100))
|
||||
got, err := suggestExitNode(&tt.lastReport, &tt.netMap, r)
|
||||
if got.Name != tt.wantName {
|
||||
t.Errorf("name=%v, want %v", got.Name, tt.wantName)
|
||||
}
|
||||
if got.ID != tt.wantID {
|
||||
t.Errorf("ID=%v, want %v", got.ID, tt.wantID)
|
||||
}
|
||||
if tt.wantError == nil && err != nil {
|
||||
t.Errorf("err=%v, want no error", err)
|
||||
}
|
||||
if tt.wantError != nil && !errors.Is(err, tt.wantError) {
|
||||
t.Errorf("err=%v, want %v", err, tt.wantError)
|
||||
}
|
||||
if !reflect.DeepEqual(got.Location, tt.wantLocation) {
|
||||
t.Errorf("location=%v, want %v", got.Location, tt.wantLocation)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestExitNodePickWeighted(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
candidates []tailcfg.NodeView
|
||||
wantValue tailcfg.NodeView
|
||||
wantValid bool
|
||||
}{
|
||||
{
|
||||
name: ">1 candidates",
|
||||
candidates: []tailcfg.NodeView{
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Priority: 20,
|
||||
},
|
||||
}).View(),
|
||||
}).View(),
|
||||
(&tailcfg.Node{
|
||||
ID: 3,
|
||||
StableID: "3",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Priority: 10,
|
||||
},
|
||||
}).View(),
|
||||
}).View(),
|
||||
},
|
||||
wantValue: (&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Priority: 20,
|
||||
},
|
||||
}).View(),
|
||||
}).View(),
|
||||
wantValid: true,
|
||||
},
|
||||
{
|
||||
name: "<1 candidates",
|
||||
candidates: []tailcfg.NodeView{},
|
||||
wantValid: false,
|
||||
},
|
||||
{
|
||||
name: "1 candidate",
|
||||
candidates: []tailcfg.NodeView{
|
||||
(&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Priority: 20,
|
||||
},
|
||||
}).View(),
|
||||
}).View(),
|
||||
},
|
||||
wantValue: (&tailcfg.Node{
|
||||
ID: 2,
|
||||
StableID: "2",
|
||||
AllowedIPs: []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0"),
|
||||
},
|
||||
Hostinfo: (&tailcfg.Hostinfo{
|
||||
Location: &tailcfg.Location{
|
||||
Priority: 20,
|
||||
},
|
||||
}).View(),
|
||||
}).View(),
|
||||
wantValid: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := pickWeighted(tt.candidates)
|
||||
if !reflect.DeepEqual(got, tt.wantValue) {
|
||||
t.Errorf("got value %v want %v", got, tt.wantValue)
|
||||
if tt.wantValid != got.Valid() {
|
||||
t.Errorf("got invalid candidate expected valid")
|
||||
}
|
||||
if tt.wantValid {
|
||||
if !reflect.DeepEqual(got, tt.wantValue) {
|
||||
t.Errorf("got value %v want %v", got, tt.wantValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestExitNodeLongLatDistance(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fromLat float64
|
||||
fromLong float64
|
||||
toLat float64
|
||||
toLong float64
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "zero values",
|
||||
fromLat: 0,
|
||||
fromLong: 0,
|
||||
toLat: 0,
|
||||
toLong: 0,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "valid values",
|
||||
fromLat: 40.73061,
|
||||
fromLong: -73.935242,
|
||||
toLat: 37.3382082,
|
||||
toLong: -121.8863286,
|
||||
want: 4117266.873301274,
|
||||
},
|
||||
{
|
||||
name: "valid values, locations in north and south of equator",
|
||||
fromLat: 40.73061,
|
||||
fromLong: -73.935242,
|
||||
toLat: -33.861481,
|
||||
toLong: 151.205475,
|
||||
want: 15994089.144368416,
|
||||
},
|
||||
}
|
||||
// The wanted values are computed using a more precise algorithm using the WGS84 model but
|
||||
// longLatDistance uses a spherical approximation for simplicity. To account for this, we allow for
|
||||
// 10km of error.
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := longLatDistance(tt.fromLat, tt.fromLong, tt.toLat, tt.toLong)
|
||||
const maxError = 10000 // 10km
|
||||
if math.Abs(got-tt.want) > maxError {
|
||||
t.Errorf("distance=%vm, want within %vm of %vm", got, maxError, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinLatencyDERPregion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
regions []int
|
||||
report *netcheck.Report
|
||||
wantRegion int
|
||||
}{
|
||||
{
|
||||
name: "regions, no latency values",
|
||||
regions: []int{1, 2, 3},
|
||||
wantRegion: 0,
|
||||
report: &netcheck.Report{},
|
||||
},
|
||||
{
|
||||
name: "regions, different latency values",
|
||||
regions: []int{1, 2, 3},
|
||||
wantRegion: 2,
|
||||
report: &netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 10 * time.Millisecond,
|
||||
2: 5 * time.Millisecond,
|
||||
3: 30 * time.Millisecond,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "regions, same values",
|
||||
regions: []int{1, 2, 3},
|
||||
wantRegion: 1,
|
||||
report: &netcheck.Report{
|
||||
RegionLatency: map[int]time.Duration{
|
||||
1: 10 * time.Millisecond,
|
||||
2: 10 * time.Millisecond,
|
||||
3: 10 * time.Millisecond,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := minLatencyDERPRegion(tt.regions, tt.report)
|
||||
if got != tt.wantRegion {
|
||||
t.Errorf("got region %v want region %v", got, tt.wantRegion)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -122,6 +122,7 @@
|
||||
"set-use-exit-node-enabled": (*Handler).serveSetUseExitNodeEnabled,
|
||||
"start": (*Handler).serveStart,
|
||||
"status": (*Handler).serveStatus,
|
||||
"suggest-exit-node": (*Handler).serveSuggestExitNode,
|
||||
"tka/affected-sigs": (*Handler).serveTKAAffectedSigs,
|
||||
"tka/cosign-recovery-aum": (*Handler).serveTKACosignRecoveryAUM,
|
||||
"tka/disable": (*Handler).serveTKADisable,
|
||||
@ -2872,3 +2873,18 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) {
|
||||
// User-visible LocalAPI endpoints.
|
||||
metricFilePutCalls = clientmetric.NewCounter("localapi_file_put")
|
||||
)
|
||||
|
||||
// serveSuggestExitNode serves a POST endpoint for returning a suggested exit node.
|
||||
func (h *Handler) serveSuggestExitNode(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "GET" {
|
||||
http.Error(w, "only GET allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
res, err := h.b.SuggestExitNode()
|
||||
if err != nil {
|
||||
writeErrorJSON(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(res)
|
||||
}
|
||||
|
@ -3063,3 +3063,17 @@ func getPeerMTUsProbedMetric(mtu tstun.WireMTU) *clientmetric.Metric {
|
||||
mm, _ := metricRecvDiscoPeerMTUProbesByMTU.LoadOrInit(key, func() *clientmetric.Metric { return clientmetric.NewCounter(key) })
|
||||
return mm
|
||||
}
|
||||
|
||||
// GetLastNetcheckReport returns the last netcheck report, running a new one if a recent one does not exist.
|
||||
func (c *Conn) GetLastNetcheckReport(ctx context.Context) *netcheck.Report {
|
||||
lastReport := c.lastNetCheckReport.Load()
|
||||
if lastReport == nil {
|
||||
nr, err := c.updateNetInfo(ctx)
|
||||
if err != nil {
|
||||
c.logf("magicsock.Conn.GetLastNetcheckReport: updateNetInfo: %v", err)
|
||||
return nil
|
||||
}
|
||||
return nr
|
||||
}
|
||||
return lastReport
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user