From 88c7d19d544b870bf481f9d679d46bf8c0eb2801 Mon Sep 17 00:00:00 2001 From: Tom DNetto Date: Wed, 1 Mar 2023 14:14:35 -0800 Subject: [PATCH] tka: compact TKA storage on startup Signed-off-by: Tom DNetto --- ipn/ipnlocal/local.go | 3 + ipn/ipnlocal/network-lock.go | 5 ++ tka/tailchonk.go | 104 +++++++++++++++++++++++++++++++++-- tka/tailchonk_test.go | 79 ++++++++++++++++++++++++++ tka/tka.go | 14 +++++ tka/tka_test.go | 50 +++++++++++++++++ 6 files changed, 250 insertions(+), 5 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 96b84d89f..a9d0a3550 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4740,6 +4740,9 @@ func (b *LocalBackend) initTKALocked() error { if err != nil { return fmt.Errorf("initializing tka: %v", err) } + if err := authority.Compact(storage, tkaCompactionDefaults); err != nil { + b.logf("tka compaction failed: %v", err) + } b.tka = &tkaState{ profile: cp.ID, diff --git a/ipn/ipnlocal/network-lock.go b/ipn/ipnlocal/network-lock.go index 955ad3a76..2690fff97 100644 --- a/ipn/ipnlocal/network-lock.go +++ b/ipn/ipnlocal/network-lock.go @@ -37,6 +37,11 @@ import ( var ( errMissingNetmap = errors.New("missing netmap: verify that you are logged in") errNetworkLockNotActive = errors.New("network-lock is not active") + + tkaCompactionDefaults = tka.CompactionOptions{ + MinChain: 24, // Keep at minimum 24 AUMs since head. + MinAge: 14 * 24 * time.Hour, // Keep 2 weeks of AUMs. + } ) type tkaState struct { diff --git a/tka/tailchonk.go b/tka/tailchonk.go index 1bbdb8b4b..90af2df7f 100644 --- a/tka/tailchonk.go +++ b/tka/tailchonk.go @@ -200,6 +200,11 @@ func ChonkDir(dir string) (*FS, error) { if !stat.IsDir() { return nil, fmt.Errorf("chonk directory %q is a file", dir) } + + // TODO(tom): *FS marks AUMs as deleted but does not actually + // delete them, to avoid data loss in the event of a bug. + // Implement deletion after we are fairly sure in the implementation. + return &FS{base: dir}, nil } @@ -213,8 +218,17 @@ func ChonkDir(dir string) (*FS, error) { // much smaller than JSON for AUMs. The 'keyasint' thing isn't essential // but again it saves a bunch of bytes. type fsHashInfo struct { - Children []AUMHash `cbor:"1,keyasint"` - AUM *AUM `cbor:"2,keyasint"` + Children []AUMHash `cbor:"1,keyasint"` + AUM *AUM `cbor:"2,keyasint"` + CreatedUnix int64 `cbor:"3,keyasint,omitempty"` + + // PurgedUnix is set when the AUM is deleted. The value is + // the unix epoch at the time it was deleted. + // + // While a non-zero PurgedUnix symbolizes the AUM is deleted, + // the fsHashInfo entry can continue to exist to track children + // of this AUMHash. + PurgedUnix int64 `cbor:"4,keyasint,omitempty"` } // aumDir returns the directory an AUM is stored in, and its filename @@ -238,12 +252,45 @@ func (c *FS) AUM(hash AUMHash) (AUM, error) { } return AUM{}, err } - if info.AUM == nil { + if info.AUM == nil || info.PurgedUnix > 0 { return AUM{}, os.ErrNotExist } return *info.AUM, nil } +// CommitTime returns the time at which the AUM was committed. +// +// If the AUM does not exist, then os.ErrNotExist is returned. +func (c *FS) CommitTime(h AUMHash) (time.Time, error) { + c.mu.RLock() + defer c.mu.RUnlock() + + info, err := c.get(h) + if err != nil { + if os.IsNotExist(err) { + return time.Time{}, os.ErrNotExist + } + return time.Time{}, err + } + if info.PurgedUnix > 0 { + return time.Time{}, os.ErrNotExist + } + if info.CreatedUnix > 0 { + return time.Unix(info.CreatedUnix, 0), nil + } + + // If we got this far, the AUM exists but CreatedUnix is not + // set, presumably because this AUM was committed using a version + // of tailscaled that pre-dates the introduction of CreatedUnix. + // As such, we use the file modification time as a suitable analog. + dir, base := c.aumDir(h) + s, err := os.Stat(filepath.Join(dir, base)) + if err != nil { + return time.Time{}, nil + } + return s.ModTime(), nil +} + // AUM returns any known AUMs with a specific parent hash. func (c *FS) ChildAUMs(prevAUMHash AUMHash) ([]AUM, error) { c.mu.RLock() @@ -257,6 +304,9 @@ func (c *FS) ChildAUMs(prevAUMHash AUMHash) ([]AUM, error) { } return nil, err } + // NOTE(tom): We don't check PurgedUnix here because 'purged' + // only applies to that specific AUM (i.e. info.AUM) and not to + // any information about children stored against that hash. out := make([]AUM, len(info.Children)) for i, h := range info.Children { @@ -265,7 +315,7 @@ func (c *FS) ChildAUMs(prevAUMHash AUMHash) ([]AUM, error) { // We expect any AUM recorded as a child on its parent to exist. return nil, fmt.Errorf("reading child %d of %x: %v", i, h, err) } - if c.AUM == nil { + if c.AUM == nil || c.PurgedUnix > 0 { return nil, fmt.Errorf("child %d of %x: AUM not stored", i, h) } out[i] = *c.AUM @@ -309,13 +359,27 @@ func (c *FS) Heads() ([]AUM, error) { out := make([]AUM, 0, 6) // 6 is arbitrary. err := c.scanHashes(func(info *fsHashInfo) { - if len(info.Children) == 0 && info.AUM != nil { + if len(info.Children) == 0 && info.AUM != nil && info.PurgedUnix == 0 { out = append(out, *info.AUM) } }) return out, err } +// AllAUMs returns all AUMs stored in the chonk. +func (c *FS) AllAUMs() ([]AUMHash, error) { + c.mu.RLock() + defer c.mu.RUnlock() + + out := make([]AUMHash, 0, 6) // 6 is arbitrary. + err := c.scanHashes(func(info *fsHashInfo) { + if info.AUM != nil && info.PurgedUnix == 0 { + out = append(out, info.AUM.Hash()) + } + }) + return out, err +} + func (c *FS) scanHashes(eachHashInfo func(*fsHashInfo)) error { prefixDirs, err := os.ReadDir(c.base) if err != nil { @@ -411,6 +475,7 @@ func (c *FS) CommitVerifiedAUMs(updates []AUM) error { } err := c.commit(h, func(info *fsHashInfo) { + info.PurgedUnix = 0 // just in-case it was set for some reason info.AUM = &aum }) if err != nil { @@ -421,6 +486,31 @@ func (c *FS) CommitVerifiedAUMs(updates []AUM) error { return nil } +// PurgeAUMs marks the specified AUMs for deletion from storage. +func (c *FS) PurgeAUMs(hashes []AUMHash) error { + c.mu.Lock() + defer c.mu.Unlock() + + now := time.Now() + for i, h := range hashes { + stored, err := c.get(h) + if err != nil { + return fmt.Errorf("reading %d (%x): %w", i, h, err) + } + if stored.AUM == nil || stored.PurgedUnix > 0 { + continue + } + + err = c.commit(h, func(info *fsHashInfo) { + info.PurgedUnix = now.Unix() + }) + if err != nil { + return fmt.Errorf("committing purge[%d] (%x): %w", i, h, err) + } + } + return nil +} + // commit calls the provided updater function to record changes relevant // to the given hash. The caller is expected to update the AUM and // Children fields, as relevant. @@ -430,6 +520,7 @@ func (c *FS) commit(h AUMHash, updater func(*fsHashInfo)) error { existing, err := c.get(h) switch { case os.IsNotExist(err): + toCommit.CreatedUnix = time.Now().Unix() case err != nil: return err default: @@ -754,5 +845,8 @@ func Compact(storage CompactableChonk, head AUMHash, opts CompactionOptions) (la } } + if err := storage.SetLastActiveAncestor(lastActiveAncestor); err != nil { + return AUMHash{}, err + } return lastActiveAncestor, storage.PurgeAUMs(toDelete) } diff --git a/tka/tailchonk_test.go b/tka/tailchonk_test.go index 04c364090..0ba263b7b 100644 --- a/tka/tailchonk_test.go +++ b/tka/tailchonk_test.go @@ -173,6 +173,85 @@ func TestTailchonkFS_Commit(t *testing.T) { if _, err := os.Stat(filepath.Join(chonk.base, "M7", "M7LL2NDB4NKCZIUPVS6RDM2GUOIMW6EEAFVBWMVCPUANQJPHT3SQ")); err != nil { t.Errorf("stat of AUM parent failed: %v", err) } + + info, err := chonk.get(aum.Hash()) + if err != nil { + t.Fatal(err) + } + if info.PurgedUnix > 0 { + t.Errorf("recently-created AUM PurgedUnix = %d, want 0", info.PurgedUnix) + } +} + +func TestTailchonkFS_CommitTime(t *testing.T) { + chonk := &FS{base: t.TempDir()} + parentHash := randHash(t, 1) + aum := AUM{MessageKind: AUMNoOp, PrevAUMHash: parentHash[:]} + + if err := chonk.CommitVerifiedAUMs([]AUM{aum}); err != nil { + t.Fatal(err) + } + ct, err := chonk.CommitTime(aum.Hash()) + if err != nil { + t.Fatalf("CommitTime() failed: %v", err) + } + if ct.Before(time.Now().Add(-time.Minute)) || ct.After(time.Now().Add(time.Minute)) { + t.Errorf("commit time was wrong: %v more than a minute off from now (%v)", ct, time.Now()) + } +} + +func TestTailchonkFS_PurgeAUMs(t *testing.T) { + chonk := &FS{base: t.TempDir()} + parentHash := randHash(t, 1) + aum := AUM{MessageKind: AUMNoOp, PrevAUMHash: parentHash[:]} + + if err := chonk.CommitVerifiedAUMs([]AUM{aum}); err != nil { + t.Fatal(err) + } + if err := chonk.PurgeAUMs([]AUMHash{aum.Hash()}); err != nil { + t.Fatal(err) + } + + if _, err := chonk.AUM(aum.Hash()); err != os.ErrNotExist { + t.Errorf("AUM() on purged AUM returned err = %v, want ErrNotExist", err) + } + + info, err := chonk.get(aum.Hash()) + if err != nil { + t.Fatal(err) + } + if info.PurgedUnix == 0 { + t.Errorf("recently-created AUM PurgedUnix = %d, want non-zero", info.PurgedUnix) + } +} + +func TestTailchonkFS_AllAUMs(t *testing.T) { + chonk := &FS{base: t.TempDir()} + genesis := AUM{MessageKind: AUMRemoveKey, KeyID: []byte{1, 2}} + gHash := genesis.Hash() + intermediate := AUM{PrevAUMHash: gHash[:]} + iHash := intermediate.Hash() + leaf := AUM{PrevAUMHash: iHash[:]} + + commitSet := []AUM{ + genesis, + intermediate, + leaf, + } + if err := chonk.CommitVerifiedAUMs(commitSet); err != nil { + t.Fatalf("CommitVerifiedAUMs failed: %v", err) + } + + hashes, err := chonk.AllAUMs() + if err != nil { + t.Fatal(err) + } + hashesLess := func(a, b AUMHash) bool { + return bytes.Compare(a[:], b[:]) < 0 + } + if diff := cmp.Diff([]AUMHash{genesis.Hash(), intermediate.Hash(), leaf.Hash()}, hashes, cmpopts.SortSlices(hashesLess)); diff != "" { + t.Fatalf("AllAUMs() output differs (-want, +got):\n%s", diff) + } } func TestMarkActiveChain(t *testing.T) { diff --git a/tka/tka.go b/tka/tka.go index 7f714ec4f..b88312417 100644 --- a/tka/tka.go +++ b/tka/tka.go @@ -720,3 +720,17 @@ func (a *Authority) Keys() []Key { func (a *Authority) StateIDs() (uint64, uint64) { return a.state.StateID1, a.state.StateID2 } + +// Compact deletes historical AUMs based on the given compaction options. +func (a *Authority) Compact(storage CompactableChonk, o CompactionOptions) error { + newAncestor, err := Compact(storage, a.head.Hash(), o) + if err != nil { + return err + } + ancestor, err := storage.AUM(newAncestor) + if err != nil { + return err + } + a.oldestAncestor = ancestor + return nil +} diff --git a/tka/tka_test.go b/tka/tka_test.go index 11d10bac6..562b9dc92 100644 --- a/tka/tka_test.go +++ b/tka/tka_test.go @@ -474,3 +474,53 @@ func TestInteropWithNLKey(t *testing.T) { t.Error("pub3 want untrusted, got trusted") } } + +func TestAuthorityCompact(t *testing.T) { + pub, priv := testingKey25519(t, 1) + key := Key{Kind: Key25519, Public: pub, Votes: 2} + + c := newTestchain(t, ` + G -> A -> B -> C -> D -> E + + G.template = genesis + C.template = checkpoint2 + `, + optTemplate("genesis", AUM{MessageKind: AUMCheckpoint, State: &State{ + Keys: []Key{key}, + DisablementSecrets: [][]byte{DisablementKDF([]byte{1, 2, 3})}, + }}), + optTemplate("checkpoint2", AUM{MessageKind: AUMCheckpoint, State: &State{ + Keys: []Key{key}, + DisablementSecrets: [][]byte{DisablementKDF([]byte{1, 2, 3})}, + }}), + optKey("key", key, priv), + optSignAllUsing("key")) + + storage := &FS{base: t.TempDir()} + a, err := Bootstrap(storage, c.AUMs["G"]) + if err != nil { + t.Fatalf("Bootstrap() failed: %v", err) + } + a.Inform(storage, []AUM{c.AUMs["A"], c.AUMs["B"], c.AUMs["C"], c.AUMs["D"], c.AUMs["E"]}) + + // Should compact down to C -> D -> E + if err := a.Compact(storage, CompactionOptions{MinChain: 2, MinAge: 1}); err != nil { + t.Fatal(err) + } + if a.oldestAncestor.Hash() != c.AUMHashes["C"] { + t.Errorf("ancestor = %v, want %v", a.oldestAncestor.Hash(), c.AUMHashes["C"]) + } + + // Make sure the stored authority is still openable and resolves to the same state. + stored, err := Open(storage) + if err != nil { + t.Fatalf("Failed to open stored authority: %v", err) + } + if stored.Head() != a.Head() { + t.Errorf("Stored authority head differs: head = %v, want %v", stored.Head(), a.Head()) + } + t.Logf("original ancestor = %v", c.AUMHashes["G"]) + if anc, _ := storage.LastActiveAncestor(); *anc != c.AUMHashes["C"] { + t.Errorf("ancestor = %v, want %v", anc, c.AUMHashes["C"]) + } +}