From ba8c6d0775109b967bd08a6e12502b51be0f8acc Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 24 Feb 2021 21:29:51 -0800 Subject: [PATCH] health, controlclient, ipn, magicsock: tell health package state of things Not yet checking anything. Just plumbing states into the health package. Updates #1505 Signed-off-by: Brad Fitzpatrick --- control/controlclient/auto.go | 3 + control/controlclient/direct.go | 8 ++- health/health.go | 100 +++++++++++++++++++++++++++++++- ipn/ipnlocal/local.go | 2 + wgengine/magicsock/magicsock.go | 15 +++++ 5 files changed, 126 insertions(+), 2 deletions(-) diff --git a/control/controlclient/auto.go b/control/controlclient/auto.go index b9360a287..191378568 100644 --- a/control/controlclient/auto.go +++ b/control/controlclient/auto.go @@ -520,8 +520,10 @@ func (c *Client) mapRoutine() { c.mu.Lock() c.inPollNetMap = false c.mu.Unlock() + health.SetInPollNetMap(false) err := c.direct.PollNetMap(ctx, -1, func(nm *netmap.NetworkMap) { + health.SetInPollNetMap(true) c.mu.Lock() select { @@ -554,6 +556,7 @@ func (c *Client) mapRoutine() { } }) + health.SetInPollNetMap(false) c.mu.Lock() c.synced = false c.inPollNetMap = false diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index 186fbe53e..94d2a75e8 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -535,7 +535,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm vlogf = c.logf } - request := tailcfg.MapRequest{ + request := &tailcfg.MapRequest{ Version: tailcfg.CurrentMapRequestVersion, KeepAlive: c.keepAlive, NodeKey: tailcfg.NodeKey(persist.PrivateNodeKey.Public()), @@ -604,6 +604,8 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm } defer res.Body.Close() + health.NoteMapRequestHeard(request) + if cb == nil { io.Copy(ioutil.Discard, res.Body) return nil @@ -677,6 +679,10 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm return err } + if allowStream { + health.GotStreamedMapResponse() + } + if pr := resp.PingRequest; pr != nil { go answerPing(c.logf, c.httpc, pr) } diff --git a/health/health.go b/health/health.go index a199ead62..a00116a75 100644 --- a/health/health.go +++ b/health/health.go @@ -8,12 +8,28 @@ package health import ( "sync" + "time" + + "tailscale.com/tailcfg" ) var ( - mu sync.Mutex + // mu guards everything in this var block. + mu sync.Mutex + m = map[string]error{} // error key => err (or nil for no error) watchers = map[*watchHandle]func(string, error){} // opt func to run if error state changes + + inMapPoll bool + inMapPollSince time.Time + lastMapPollEndedAt time.Time + lastStreamedMapResponse time.Time + derpHomeRegion int + derpRegionConnected = map[int]bool{} + derpRegionLastFrame = map[int]time.Time{} + lastMapRequestHeard time.Time // time we got a 200 from control for a MapRequest + ipnState string + ipnWantRunning bool ) type watchHandle byte @@ -53,6 +69,7 @@ func set(key string, err error) { if !ok && err == nil { // Initial happy path. m[key] = nil + selfCheckLocked() return } if ok && (old == nil) == (err == nil) { @@ -65,7 +82,88 @@ func set(key string, err error) { return } m[key] = err + selfCheckLocked() for _, cb := range watchers { go cb(key, err) } } + +// GotStreamedMapResponse notes that we got a tailcfg.MapResponse +// message in streaming mode, even if it's just a keep-alive message. +func GotStreamedMapResponse() { + mu.Lock() + defer mu.Unlock() + lastStreamedMapResponse = time.Now() + selfCheckLocked() +} + +// SetInPollNetMap records that we're in +func SetInPollNetMap(v bool) { + mu.Lock() + defer mu.Unlock() + if v == inMapPoll { + return + } + inMapPoll = v + if v { + inMapPollSince = time.Now() + } else { + lastMapPollEndedAt = time.Now() + } +} + +// SetMagicSockDERPHome notes what magicsock's view of its home DERP is. +func SetMagicSockDERPHome(region int) { + mu.Lock() + defer mu.Unlock() + derpHomeRegion = region + selfCheckLocked() +} + +// NoteMapRequestHeard notes whenever we successfully sent a map request +// to control for which we received a 200 response. +func NoteMapRequestHeard(mr *tailcfg.MapRequest) { + mu.Lock() + defer mu.Unlock() + // TODO: extract mr.HostInfo.NetInfo.PreferredDERP, compare + // against SetMagicSockDERPHome and + // SetDERPRegionConnectedState + + lastMapRequestHeard = time.Now() + selfCheckLocked() +} + +func SetDERPRegionConnectedState(region int, connected bool) { + mu.Lock() + defer mu.Unlock() + derpRegionConnected[region] = connected + selfCheckLocked() +} + +func NoteDERPRegionReceivedFrame(region int) { + mu.Lock() + defer mu.Unlock() + derpRegionLastFrame[region] = time.Now() + selfCheckLocked() +} + +// state is an ipn.State.String() value: "Running", "Stopped", "NeedsLogin", etc. +func SetIPNState(state string, wantRunning bool) { + mu.Lock() + defer mu.Unlock() + ipnState = state + ipnWantRunning = wantRunning + selfCheckLocked() +} + +func selfCheckLocked() { + // TODO: check states against each other. + // For staticcheck for now: + _ = inMapPollSince + _ = lastMapPollEndedAt + _ = lastStreamedMapResponse + _ = derpHomeRegion + _ = lastMapRequestHeard + _ = ipnState + _ = ipnWantRunning +} diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 6036d5000..444594271 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -18,6 +18,7 @@ import ( "golang.org/x/oauth2" "inet.af/netaddr" "tailscale.com/control/controlclient" + "tailscale.com/health" "tailscale.com/internal/deepprint" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" @@ -1514,6 +1515,7 @@ func (b *LocalBackend) enterState(newState ipn.State) { } b.logf("Switching ipn state %v -> %v (WantRunning=%v)", state, newState, prefs.WantRunning) + health.SetIPNState(newState.String(), prefs.WantRunning) if notify != nil { b.send(ipn.Notify{State: &newState}) } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 5584e3324..917e55828 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -37,6 +37,7 @@ import ( "tailscale.com/derp" "tailscale.com/derp/derphttp" "tailscale.com/disco" + "tailscale.com/health" "tailscale.com/ipn/ipnstate" "tailscale.com/logtail/backoff" "tailscale.com/net/dnscache" @@ -973,6 +974,7 @@ func (c *Conn) setNearestDERP(derpNum int) (wantDERP bool) { defer c.mu.Unlock() if !c.wantDerpLocked() { c.myDerp = 0 + health.SetMagicSockDERPHome(0) return false } if derpNum == c.myDerp { @@ -980,6 +982,7 @@ func (c *Conn) setNearestDERP(derpNum int) (wantDERP bool) { return true } c.myDerp = derpNum + health.SetMagicSockDERPHome(derpNum) if c.privateKey.IsZero() { // No private key yet, so DERP connections won't come up anyway. @@ -1438,13 +1441,18 @@ func (c *Conn) runDerpReader(ctx context.Context, derpFakeAddr netaddr.IPPort, d return n } + defer health.SetDERPRegionConnectedState(regionID, false) + // peerPresent is the set of senders we know are present on this // connection, based on messages we've received from the server. peerPresent := map[key.Public]bool{} bo := backoff.NewBackoff(fmt.Sprintf("derp-%d", regionID), c.logf, 5*time.Second) + var lastPacketTime time.Time + for { msg, connGen, err := dc.RecvDetail() if err != nil { + health.SetDERPRegionConnectedState(regionID, false) // Forget that all these peers have routes. for peer := range peerPresent { delete(peerPresent, peer) @@ -1480,8 +1488,15 @@ func (c *Conn) runDerpReader(ctx context.Context, derpFakeAddr netaddr.IPPort, d } bo.BackOff(ctx, nil) // reset + now := time.Now() + if lastPacketTime.IsZero() || now.Sub(lastPacketTime) > 5*time.Second { + health.NoteDERPRegionReceivedFrame(regionID) + lastPacketTime = now + } + switch m := msg.(type) { case derp.ServerInfoMessage: + health.SetDERPRegionConnectedState(regionID, true) c.logf("magicsock: derp-%d connected; connGen=%v", regionID, connGen) continue case derp.ReceivedPacket: