ipn/ipnlocal: do not stall event processing for appc route updates (#17663)

A follow-up to #17411. Put AppConnector events into a task queue, as they may
take some time to process. Ensure that the queue is stopped at shutdown so that
cleanup will remain orderly.

Because events are delivered on a separate goroutine, slow processing of an
event does not cause an immediate problem; however, a subscriber that blocks
for a long time will push back on the bus as a whole. See
https://godoc.org/tailscale.com/util/eventbus#hdr-Expected_subscriber_behavior
for more discussion.

Updates #17192
Updates #15160

Change-Id: Ib313cc68aec273daf2b1ad79538266c81ef063e3
Signed-off-by: M. J. Fromberger <fromberger@tailscale.com>
(cherry picked from commit 06b092388e)

ipn/ipnlocal/local.go

@@ -87,6 +87,7 @@ import (
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/eventbus"
+	"tailscale.com/util/execqueue"
 	"tailscale.com/util/goroutines"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/osuser"
@@ -187,6 +188,7 @@ type LocalBackend struct {
 	statsLogf logger.Logf             // for printing peers stats on change
 	sys       *tsd.System
 	eventSubs eventbus.Monitor
+	appcTask  execqueue.ExecQueue // handles updates from appc
 
 	health                   *health.Tracker     // always non-nil
 	polc                     policyclient.Client // always non-nil
@@ -642,12 +644,14 @@ func (b *LocalBackend) consumeEventbusTopics(ec *eventbus.Client) func(*eventbus
 				// We need to find a way to ensure that changes to the backend state are applied
 				// consistently in the presnce of profile changes, which currently may not happen in
 				// a single atomic step.  See: https://github.com/tailscale/tailscale/issues/17414
+				b.appcTask.Add(func() {
 					if err := b.AdvertiseRoute(ru.Advertise...); err != nil {
 						b.logf("appc: failed to advertise routes: %v: %v", ru.Advertise, err)
 					}
 					if err := b.UnadvertiseRoute(ru.Unadvertise...); err != nil {
 						b.logf("appc: failed to unadvertise routes: %v: %v", ru.Unadvertise, err)
 					}
+				})
 			case ri := <-storeRoutesSub.Events():
 				// Whether or not routes should be stored can change over time.
 				shouldStoreRoutes := b.ControlKnobs().AppCStoreRoutes.Load()
@@ -1113,6 +1117,7 @@ func (b *LocalBackend) Shutdown() {
 	//     they can deadlock with c.Shutdown().
 	//  2. LocalBackend.consumeEventbusTopics event handlers may not guard against
 	//     undesirable post/in-progress LocalBackend.Shutdown() behaviors.
+	b.appcTask.Shutdown()
 	b.eventSubs.Close()
 
 	b.em.close()