appc,ipn/ipnlocal,types/appctype: implement control provided routes

Control can now send down a set of routes along with the domains, and
the routes will be advertised, with any newly overlapped routes being
removed to reduce the size of the routing table.

Fixes tailscale/corp#16833
Signed-off-by: James Tucker <james@tailscale.com>

appc/appconnector.go

@@ -28,6 +28,9 @@ type RouteAdvertiser interface {
 	// AdvertiseRoute adds a new route advertisement if the route is not already
 	// being advertised.
 	AdvertiseRoute(netip.Prefix) error
+
+	// UnadvertiseRoute removes a route advertisement.
+	UnadvertiseRoute(netip.Prefix) error
 }
 
 // AppConnector is an implementation of an AppConnector that performs
@@ -45,10 +48,14 @@ type AppConnector struct {
 
 	// mu guards the fields that follow
 	mu sync.Mutex
+
 	// domains is a map of lower case domain names with no trailing dot, to a
 	// list of resolved IP addresses.
 	domains map[string][]netip.Addr
 
+	// controlRoutes is the list of routes that were last supplied by control.
+	controlRoutes []netip.Prefix
+
 	// wildcards is the list of domain strings that match subdomains.
 	wildcards []string
 }
@@ -97,6 +104,42 @@ func (e *AppConnector) UpdateDomains(domains []string) {
 	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
 }
 
+// UpdateRoutes merges the supplied routes into the currently configured routes. The routes supplied
+// by control for UpdateRoutes are supplemental to the routes discovered by DNS resolution, but are
+// also more often whole ranges. UpdateRoutes will remove any single address routes that are now
+// covered by new ranges.
+func (e *AppConnector) UpdateRoutes(routes []netip.Prefix) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	// If there was no change since the last update, no work to do.
+	if slices.Equal(e.controlRoutes, routes) {
+		return
+	}
+
+nextRoute:
+	for _, r := range routes {
+		if err := e.routeAdvertiser.AdvertiseRoute(r); err != nil {
+			e.logf("failed to advertise route: %v: %v", r, err)
+			continue
+		}
+
+		for _, addr := range e.domains {
+			for _, a := range addr {
+				if r.Contains(a) {
+					pfx := netip.PrefixFrom(a, a.BitLen())
+					if err := e.routeAdvertiser.UnadvertiseRoute(pfx); err != nil {
+						e.logf("failed to unadvertise route: %v: %v", pfx, err)
+					}
+					continue nextRoute
+				}
+			}
+		}
+	}
+
+	e.controlRoutes = routes
+}
+
 // Domains returns the currently configured domain list.
 func (e *AppConnector) Domains() views.Slice[string] {
 	e.mu.Lock()
@@ -132,6 +175,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		return
 	}
 
+nextAnswer:
 	for {
 		h, err := p.AnswerHeader()
 		if err == dnsmessage.ErrSectionDone {
@@ -206,6 +250,16 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		if slices.Contains(addrs, addr) {
 			continue
 		}
+		for _, route := range e.controlRoutes {
+			if route.Contains(addr) {
+				// record the new address associated with the domain for faster matching in subsequent
+				// requests and for diagnostic records.
+				e.mu.Lock()
+				e.domains[domain] = append(addrs, addr)
+				e.mu.Unlock()
+				continue nextAnswer
+			}
+		}
 		if err := e.routeAdvertiser.AdvertiseRoute(netip.PrefixFrom(addr, addr.BitLen())); err != nil {
 			e.logf("failed to advertise route for %s: %v: %v", domain, addr, err)
 			continue

appc/appconnector_test.go

@@ -11,6 +11,7 @@ import (
 
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 )
 
@@ -36,6 +37,30 @@ func TestUpdateDomains(t *testing.T) {
 	}
 }
 
+func TestUpdateRoutes(t *testing.T) {
+	rc := &routeCollector{}
+	a := NewAppConnector(t.Logf, rc)
+	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
+	a.UpdateRoutes(routes)
+
+	if !slices.EqualFunc(routes, rc.routes, prefixEqual) {
+		t.Fatalf("got %v, want %v", rc.routes, routes)
+	}
+}
+
+func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
+	rc := &routeCollector{}
+	a := NewAppConnector(t.Logf, rc)
+	mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
+	rc.routes = []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
+	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
+	a.UpdateRoutes(routes)
+
+	if !slices.EqualFunc(routes, rc.routes, prefixEqual) {
+		t.Fatalf("got %v, want %v", rc.routes, routes)
+	}
+}
+
 func TestDomainRoutes(t *testing.T) {
 	rc := &routeCollector{}
 	a := NewAppConnector(t.Logf, rc)
@@ -79,7 +104,19 @@ func TestObserveDNSResponse(t *testing.T) {
 	// don't re-advertise routes that have already been advertised
 	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
 	if !slices.Equal(rc.routes, wantRoutes) {
-		t.Errorf("got %v; want %v", rc.routes, wantRoutes)
+		t.Errorf("rc.routes: got %v; want %v", rc.routes, wantRoutes)
+	}
+
+	// don't advertise addresses that are already in a control provided route
+	pfx := netip.MustParsePrefix("192.0.2.0/24")
+	a.UpdateRoutes([]netip.Prefix{pfx})
+	wantRoutes = append(wantRoutes, pfx)
+	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
+	if !slices.Equal(rc.routes, wantRoutes) {
+		t.Errorf("rc.routes: got %v; want %v", rc.routes, wantRoutes)
+	}
+	if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
+		t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
 	}
 }
 
@@ -160,3 +197,18 @@ func (rc *routeCollector) AdvertiseRoute(pfx netip.Prefix) error {
 	rc.routes = append(rc.routes, pfx)
 	return nil
 }
+
+func (rc *routeCollector) UnadvertiseRoute(pfx netip.Prefix) error {
+	routes := rc.routes
+	rc.routes = rc.routes[:0]
+	for _, r := range routes {
+		if r != pfx {
+			rc.routes = append(rc.routes, r)
+		}
+	}
+	return nil
+}
+
+func prefixEqual(a, b netip.Prefix) bool {
+	return a.Addr().Compare(b.Addr()) == 0 && a.Bits() == b.Bits()
+}

ipn/ipnlocal/local.go

@@ -3446,14 +3446,21 @@ func (b *LocalBackend) reconfigAppConnectorLocked(nm *netmap.NetworkMap, prefs i
 		})
 	}
 
-	var domains []string
+	var (
+		domains []string
+		routes  []netip.Prefix
+	)
 	for _, attr := range attrs {
 		if slices.Contains(attr.Connectors, "*") || selfHasTag(attr.Connectors) {
 			domains = append(domains, attr.Domains...)
+			routes = append(routes, attr.Routes...)
 		}
 	}
 	slices.Sort(domains)
+	slices.SortFunc(routes, func(i, j netip.Prefix) int { return i.Addr().Compare(j.Addr()) })
 	domains = slices.Compact(domains)
+	routes = slices.Compact(routes)
+	b.appConnector.UpdateRoutes(routes)
 	b.appConnector.UpdateDomains(domains)
 }
 
@@ -5805,6 +5812,30 @@ func (b *LocalBackend) AdvertiseRoute(ipp netip.Prefix) error {
 	return err
 }
 
+// UnadvertiseRoute implements the appc.RouteAdvertiser interface. It removes
+// a route advertisement if one is present in the existing routes.
+func (b *LocalBackend) UnadvertiseRoute(ipp netip.Prefix) error {
+	currentRoutes := b.Prefs().AdvertiseRoutes().AsSlice()
+	if !slices.Contains(currentRoutes, ipp) {
+		return nil
+	}
+
+	newRoutes := currentRoutes[:0]
+	for _, r := range currentRoutes {
+		if r != ipp {
+			newRoutes = append(newRoutes, r)
+		}
+	}
+
+	_, err := b.EditPrefs(&ipn.MaskedPrefs{
+		Prefs: ipn.Prefs{
+			AdvertiseRoutes: newRoutes,
+		},
+		AdvertiseRoutesSet: true,
+	})
+	return err
+}
+
 // seamlessRenewalEnabled reports whether seamless key renewals are enabled
 // (i.e. we saw our self node with the SeamlessKeyRenewal attr in a netmap).
 // This enables beta functionality of renewing node keys without breaking

ipn/ipnlocal/local_test.go

@@ -1169,6 +1169,13 @@ func TestRouteAdvertiser(t *testing.T) {
 	if routes.Len() != 1 || routes.At(0) != testPrefix {
 		t.Fatalf("got routes %v, want %v", routes, []netip.Prefix{testPrefix})
 	}
+
+	must.Do(ra.UnadvertiseRoute(testPrefix))
+
+	routes = b.Prefs().AdvertiseRoutes()
+	if routes.Len() != 0 {
+		t.Fatalf("got routes %v, want none", routes)
+	}
 }
 
 func TestRouterAdvertiserIgnoresContainedRoutes(t *testing.T) {
@@ -1352,6 +1359,17 @@ func (rc *routeCollector) AdvertiseRoute(pfx netip.Prefix) error {
 	return nil
 }
 
+func (rc *routeCollector) UnadvertiseRoute(pfx netip.Prefix) error {
+	routes := rc.routes
+	rc.routes = rc.routes[:0]
+	for _, r := range routes {
+		if r != pfx {
+			rc.routes = append(rc.routes, r)
+		}
+	}
+	return nil
+}
+
 type errorSyspolicyHandler struct {
 	t         *testing.T
 	err       error

types/appctype/appconnector.go

@@ -66,6 +66,8 @@ type AppConnectorAttr struct {
 	// Domains enumerates the domains serviced by the specified app connectors.
 	// Domains can be of the form: example.com, or *.example.com.
 	Domains []string `json:"domains,omitempty"`
+	// Routes enumerates the predetermined routes to be advertised by the specified app connectors.
+	Routes []netip.Prefix `json:"routes,omitempty"`
 	// Connectors enumerates the app connectors which service these domains.
 	// These can either be "*" to match any advertising connector, or a
 	// tag of the form tag:<tag-name>.