From 15b8665787d8140c7f8f19e1c55d286d3514ee68 Mon Sep 17 00:00:00 2001 From: David Crawshaw Date: Sat, 6 Aug 2022 16:11:33 -0700 Subject: [PATCH] tka: stable text representation of AUMHash This makes debugging easier, you can pass an AUMHash to a printf and get a string that is easy to debug. Also rearrange how directories/files work in the FS store: use the first two characters of the string representation as the prefix directory, and use the entire AUMHash string as the file name. This is again to aid debugging: you can `ls` a directory and line up what prints out easily with what you get from a printf in debug code. Signed-off-by: David Crawshaw --- tka/aum.go | 27 +++++++++++++++++++++++++++ tka/tailchonk.go | 42 ++++++++++++++++++------------------------ tka/tailchonk_test.go | 12 ++++++++---- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/tka/aum.go b/tka/aum.go index 7fcd04da9..f83ceeeed 100644 --- a/tka/aum.go +++ b/tka/aum.go @@ -7,6 +7,7 @@ package tka import ( "bytes" "crypto/ed25519" + "encoding/base32" "errors" "fmt" @@ -18,6 +19,32 @@ import ( // AUMHash represents the BLAKE2s digest of an Authority Update Message (AUM). type AUMHash [blake2s.Size]byte +var base32StdNoPad = base32.StdEncoding.WithPadding(base32.NoPadding) + +// String returns the AUMHash encoded as base32. +// This is suitable for use as a filename, and for storing in text-preferred media. +func (h AUMHash) String() string { + return base32StdNoPad.EncodeToString(h[:]) +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (h *AUMHash) UnmarshalText(text []byte) error { + if l := base32StdNoPad.DecodedLen(len(text)); l != len(h) { + return fmt.Errorf("tka.AUMHash.UnmarshalText: text wrong length: %d, want %d", l, len(text)) + } + if _, err := base32StdNoPad.Decode(h[:], text); err != nil { + return fmt.Errorf("tka.AUMHash.UnmarshalText: %w", err) + } + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (h AUMHash) MarshalText() ([]byte, error) { + b := make([]byte, base32StdNoPad.EncodedLen(len(h))) + base32StdNoPad.Encode(b, h[:]) + return b, nil +} + // AUMKind describes valid AUM types. type AUMKind uint8 diff --git a/tka/tailchonk.go b/tka/tailchonk.go index e62e1d520..16be78ce7 100644 --- a/tka/tailchonk.go +++ b/tka/tailchonk.go @@ -6,8 +6,6 @@ package tka import ( "bytes" - "encoding/base32" - "encoding/hex" "fmt" "io/ioutil" "os" @@ -190,7 +188,8 @@ func ChonkDir(dir string) (*FS, error) { // fsHashInfo describes how information about an AUMHash is represented // on disk. // -// The CBOR-serialization of this struct is stored to base/hex(hash[0])/base32(hash[1:]) +// The CBOR-serialization of this struct is stored to base/__/base32(hash) +// where __ are the first two characters of base32(hash). // // CBOR was chosen because we are already using it and it serializes // much smaller than JSON for AUMs. The 'keyasint' thing isn't essential @@ -200,12 +199,11 @@ type fsHashInfo struct { AUM *AUM `cbor:"2,keyasint"` } -func (c *FS) dirPrefix(h AUMHash) string { - return filepath.Join(c.base, hex.EncodeToString(h[:1])) -} - -func (c *FS) filename(h AUMHash) string { - return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(h[1:]) +// aumDir returns the directory an AUM is stored in, and its filename +// within the directory. +func (c *FS) aumDir(h AUMHash) (dir, base string) { + s := h.String() + return filepath.Join(c.base, s[:2]), s } // AUM returns the AUM with the specified digest. @@ -256,7 +254,8 @@ func (c *FS) ChildAUMs(prevAUMHash AUMHash) ([]AUM, error) { } func (c *FS) get(h AUMHash) (*fsHashInfo, error) { - f, err := os.Open(filepath.Join(c.dirPrefix(h), c.filename(h))) + dir, base := c.aumDir(h) + f, err := os.Open(filepath.Join(dir, base)) if err != nil { return nil, err } @@ -266,6 +265,9 @@ func (c *FS) get(h AUMHash) (*fsHashInfo, error) { if err := cbor.NewDecoder(f).Decode(&out); err != nil { return nil, err } + if out.AUM != nil && out.AUM.Hash() != h { + return nil, fmt.Errorf("%s: AUM does not match file name hash %s", f.Name(), out.AUM.Hash()) + } return &out, nil } @@ -297,24 +299,15 @@ func (c *FS) scanHashes(eachHashInfo func(*fsHashInfo)) error { if !prefix.IsDir() { continue } - pb, err := hex.DecodeString(prefix.Name()) - if err != nil || len(pb) != 1 { - return fmt.Errorf("invalid prefix directory %q: %v", prefix.Name(), err) - } - files, err := os.ReadDir(filepath.Join(c.base, prefix.Name())) if err != nil { return fmt.Errorf("reading prefix dir: %v", err) } for _, file := range files { - remainingHash, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(file.Name()) - if err != nil { - return fmt.Errorf("invalid aum file %s/%s: %v", prefix.Name(), file.Name(), err) - } var h AUMHash - h[0] = pb[0] - copy(h[1:], remainingHash) - + if err := h.UnmarshalText([]byte(file.Name())); err != nil { + return fmt.Errorf("invalid aum file: %s: %w", file.Name(), err) + } info, err := c.get(h) if err != nil { return fmt.Errorf("reading %x: %v", h, err) @@ -422,7 +415,8 @@ func (c *FS) commit(h AUMHash, updater func(*fsHashInfo)) error { return fmt.Errorf("cannot commit AUM with hash %x to %x", toCommit.AUM.Hash(), h) } - if err := os.MkdirAll(c.dirPrefix(h), 0755); err != nil && !os.IsExist(err) { + dir, base := c.aumDir(h) + if err := os.MkdirAll(dir, 0755); err != nil && !os.IsExist(err) { return fmt.Errorf("creating directory: %v", err) } @@ -430,5 +424,5 @@ func (c *FS) commit(h AUMHash, updater func(*fsHashInfo)) error { if err := cbor.NewEncoder(&buff).Encode(toCommit); err != nil { return fmt.Errorf("encoding: %v", err) } - return atomicfile.WriteFile(filepath.Join(c.dirPrefix(h), c.filename(h)), buff.Bytes(), 0644) + return atomicfile.WriteFile(filepath.Join(dir, base), buff.Bytes(), 0644) } diff --git a/tka/tailchonk_test.go b/tka/tailchonk_test.go index f7fec6643..3ca35dbe7 100644 --- a/tka/tailchonk_test.go +++ b/tka/tailchonk_test.go @@ -146,13 +146,17 @@ func TestTailchonkFS_Commit(t *testing.T) { t.Fatal(err) } - if got, want := chonk.filename(aum.Hash()), "HJX3LPJJQVRFSQX4QONESBU4DUO5JPORA66ZUCFS6NHZWDZTP4"; got != want { - t.Errorf("aum filename = %q, want %q", got, want) + dir, base := chonk.aumDir(aum.Hash()) + if got, want := dir, filepath.Join(chonk.base, "VU"); got != want { + t.Errorf("aum dir=%s, want %s", got, want) } - if _, err := os.Stat(filepath.Join(chonk.base, "ad", "HJX3LPJJQVRFSQX4QONESBU4DUO5JPORA66ZUCFS6NHZWDZTP4")); err != nil { + if want := "VU5G7NN5FGCWEWKC7SBZUSIGTQOR3VF52ED33GQIWLZU7GYPGN7Q"; base != want { + t.Errorf("aum base=%s, want %s", base, want) + } + if _, err := os.Stat(filepath.Join(dir, base)); err != nil { t.Errorf("stat of AUM file failed: %v", err) } - if _, err := os.Stat(filepath.Join(chonk.base, "67", "226TIYPDKQWKFD5MXUI3GRVDSDFXRBABNINTFIT5ADMCLZ464U")); err != nil { + if _, err := os.Stat(filepath.Join(chonk.base, "M7", "M7LL2NDB4NKCZIUPVS6RDM2GUOIMW6EEAFVBWMVCPUANQJPHT3SQ")); err != nil { t.Errorf("stat of AUM parent failed: %v", err) } }