Start with discover routes, basic functionality

appc/appconnector.go

@@ -166,6 +166,13 @@ func (e *AppConnector) updateDomains(domains []string) {
 
 	var oldDomains map[string][]netip.Addr
 	oldDomains, e.domains = e.domains, make(map[string][]netip.Addr, len(domains))
+	var oldDiscovered map[string]*routeinfo.DatedRoutes
+	var routeInfo *routeinfo.RouteInfo
+	shouldStoreRoutes := e.ShouldStoreRoutes
+	if shouldStoreRoutes {
+		routeInfo = e.RouteInfo()
+		oldDiscovered, routeInfo.Discovered = routeInfo.Discovered, make(map[string]*routeinfo.DatedRoutes, len(domains))
+	}
 	e.wildcards = e.wildcards[:0]
 	for _, d := range domains {
 		d = strings.ToLower(d)
@@ -178,6 +185,10 @@ func (e *AppConnector) updateDomains(domains []string) {
 		}
 		e.domains[d] = oldDomains[d]
 		delete(oldDomains, d)
+		if shouldStoreRoutes {
+			routeInfo.Discovered[d] = oldDiscovered[d]
+			delete(oldDiscovered, d)
+		}
 	}
 
 	// Ensure that still-live wildcards addresses are preserved as well.
@@ -189,6 +200,27 @@ func (e *AppConnector) updateDomains(domains []string) {
 			}
 		}
 	}
+	if shouldStoreRoutes {
+		for d, dr := range oldDiscovered {
+			for _, wc := range e.wildcards {
+				if dnsname.HasSuffix(d, wc) {
+					routeInfo.Discovered[d] = dr
+					delete(oldDiscovered, d)
+					break
+				}
+			}
+		}
+	}
+
+	if shouldStoreRoutes {
+		e.UpdateRouteInfo(routeInfo)
+		// everything left in oldDiscovered a) won't be in e.domains and b) can be unadvertised if it's not in local or control
+		for domainName, domainsRoutes := range oldDiscovered {
+			toRemove := domainsRoutes.RoutesSlice()
+			e.logf("unadvertising %d routes for domain: %s", len(toRemove), domainName)
+			e.scheduleUnadvertisement(domainName, toRemove...)
+		}
+	}
 	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
 }
 
@@ -394,6 +426,11 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		}
 
 		e.logf("[v2] observed new routes for %s: %s", domain, toAdvertise)
+		if e.ShouldStoreRoutes && len(toAdvertise) != 0 {
+			routeInfo := e.RouteInfo()
+			routeInfo.AddRoutesInDiscoveredForDomain(domain, toAdvertise)
+			e.UpdateRouteInfo(routeInfo)
+		}
 		e.scheduleAdvertisement(domain, toAdvertise...)
 	}
 }
@@ -472,6 +509,27 @@ func (e *AppConnector) scheduleAdvertisement(domain string, routes ...netip.Pref
 	})
 }
 
+func (e *AppConnector) scheduleUnadvertisement(domain string, routes ...netip.Prefix) {
+	e.queue.Add(func() {
+		if err := e.routeAdvertiser.UnadvertiseRoute(routes...); err != nil {
+			e.logf("failed to unadvertise routes for %s: %v: %v", domain, routes, err)
+			return
+		}
+		e.mu.Lock()
+		defer e.mu.Unlock()
+
+		for _, route := range routes {
+			if !route.IsSingleIP() {
+				continue
+			}
+			addr := route.Addr()
+
+			//e.deleteDomainAddrLocked(domain, addr)
+			e.logf("[v2] unadvertised route for %v: %v", domain, addr)
+		}
+	})
+}
+
 // hasDomainAddrLocked returns true if the address has been observed in a
 // resolution of domain.
 func (e *AppConnector) hasDomainAddrLocked(domain string, addr netip.Addr) bool {

appc/routeinfo/routeinfo.go

@@ -40,7 +40,7 @@ func (ri *RouteInfo) Routes(local, control, discovered bool) []netip.Prefix {
 
 	if discovered {
 		for _, dr := range ri.Discovered {
-			ret = append(ret, dr.routesSlice()...)
+			ret = append(ret, dr.RoutesSlice()...)
 		}
 	}
 	return ret
@@ -53,10 +53,33 @@ type DatedRoutes struct {
 	LastCleanup time.Time
 }
 
-func (dr *DatedRoutes) routesSlice() []netip.Prefix {
+func (dr *DatedRoutes) RoutesSlice() []netip.Prefix {
 	var routes []netip.Prefix
 	for k := range dr.Routes {
 		routes = append(routes, k)
 	}
 	return routes
 }
+
+func (r *RouteInfo) AddRoutesInDiscoveredForDomain(domain string, addrs []netip.Prefix) {
+	dr, hasKey := r.Discovered[domain]
+	if !hasKey || dr == nil || dr.Routes == nil {
+		newDatedRoutes := &DatedRoutes{make(map[netip.Prefix]time.Time), time.Now()}
+		newDatedRoutes.addAddrsToDatedRoute(addrs)
+		r.Discovered[domain] = newDatedRoutes
+		return
+	}
+
+	// kevin comment: we won't see any existing routes here because know addrs are filtered.
+	currentRoutes := r.Discovered[domain]
+	currentRoutes.addAddrsToDatedRoute(addrs)
+	r.Discovered[domain] = currentRoutes
+	return
+}
+
+func (d *DatedRoutes) addAddrsToDatedRoute(addrs []netip.Prefix) {
+	time := time.Now()
+	for _, addr := range addrs {
+		d.Routes[addr] = time
+	}
+}

ipn/ipnlocal/local_test.go

@@ -2858,8 +2858,10 @@ func TestPatchPrefsHandlerWithoutPresistStore(t *testing.T) {
 }
 
 func TestFran(t *testing.T) {
-	explicitlyAdvertisedRoutes := []netip.Prefix{netip.MustParsePrefix("192.1.0.8/32"), netip.MustParsePrefix("192.1.0.9/32")}
+	epIP2 := netip.MustParsePrefix("192.1.0.9/32")
+	explicitlyAdvertisedRoutes := []netip.Prefix{netip.MustParsePrefix("192.1.0.8/32"), epIP2}
 	oneRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32"), netip.MustParsePrefix("192.0.0.16/32")}
+	//twoRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.10/32"), netip.MustParsePrefix("192.0.0.18/32"), epIP2}
 	twoRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.10/32"), netip.MustParsePrefix("192.0.0.18/32")}
 
 	for _, shouldStore := range []bool{true, false} {