tka: stable text representation of AUMHash

This makes debugging easier, you can pass an AUMHash to a printf and get
a string that is easy to debug.

Also rearrange how directories/files work in the FS store: use the first
two characters of the string representation as the prefix directory, and
use the entire AUMHash string as the file name. This is again to aid
debugging: you can `ls` a directory and line up what prints out easily
with what you get from a printf in debug code.

Signed-off-by: David Crawshaw <crawshaw@tailscale.com>
pull/5320/head
David Crawshaw 2 years ago committed by David Crawshaw
parent 40ec8617ac
commit 15b8665787

@ -7,6 +7,7 @@ package tka
import ( import (
"bytes" "bytes"
"crypto/ed25519" "crypto/ed25519"
"encoding/base32"
"errors" "errors"
"fmt" "fmt"
@ -18,6 +19,32 @@ import (
// AUMHash represents the BLAKE2s digest of an Authority Update Message (AUM). // AUMHash represents the BLAKE2s digest of an Authority Update Message (AUM).
type AUMHash [blake2s.Size]byte type AUMHash [blake2s.Size]byte
var base32StdNoPad = base32.StdEncoding.WithPadding(base32.NoPadding)
// String returns the AUMHash encoded as base32.
// This is suitable for use as a filename, and for storing in text-preferred media.
func (h AUMHash) String() string {
return base32StdNoPad.EncodeToString(h[:])
}
// UnmarshalText implements encoding.TextUnmarshaler.
func (h *AUMHash) UnmarshalText(text []byte) error {
if l := base32StdNoPad.DecodedLen(len(text)); l != len(h) {
return fmt.Errorf("tka.AUMHash.UnmarshalText: text wrong length: %d, want %d", l, len(text))
}
if _, err := base32StdNoPad.Decode(h[:], text); err != nil {
return fmt.Errorf("tka.AUMHash.UnmarshalText: %w", err)
}
return nil
}
// MarshalText implements encoding.TextMarshaler.
func (h AUMHash) MarshalText() ([]byte, error) {
b := make([]byte, base32StdNoPad.EncodedLen(len(h)))
base32StdNoPad.Encode(b, h[:])
return b, nil
}
// AUMKind describes valid AUM types. // AUMKind describes valid AUM types.
type AUMKind uint8 type AUMKind uint8

@ -6,8 +6,6 @@ package tka
import ( import (
"bytes" "bytes"
"encoding/base32"
"encoding/hex"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
@ -190,7 +188,8 @@ func ChonkDir(dir string) (*FS, error) {
// fsHashInfo describes how information about an AUMHash is represented // fsHashInfo describes how information about an AUMHash is represented
// on disk. // on disk.
// //
// The CBOR-serialization of this struct is stored to base/hex(hash[0])/base32(hash[1:]) // The CBOR-serialization of this struct is stored to base/__/base32(hash)
// where __ are the first two characters of base32(hash).
// //
// CBOR was chosen because we are already using it and it serializes // CBOR was chosen because we are already using it and it serializes
// much smaller than JSON for AUMs. The 'keyasint' thing isn't essential // much smaller than JSON for AUMs. The 'keyasint' thing isn't essential
@ -200,12 +199,11 @@ type fsHashInfo struct {
AUM *AUM `cbor:"2,keyasint"` AUM *AUM `cbor:"2,keyasint"`
} }
func (c *FS) dirPrefix(h AUMHash) string { // aumDir returns the directory an AUM is stored in, and its filename
return filepath.Join(c.base, hex.EncodeToString(h[:1])) // within the directory.
} func (c *FS) aumDir(h AUMHash) (dir, base string) {
s := h.String()
func (c *FS) filename(h AUMHash) string { return filepath.Join(c.base, s[:2]), s
return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(h[1:])
} }
// AUM returns the AUM with the specified digest. // AUM returns the AUM with the specified digest.
@ -256,7 +254,8 @@ func (c *FS) ChildAUMs(prevAUMHash AUMHash) ([]AUM, error) {
} }
func (c *FS) get(h AUMHash) (*fsHashInfo, error) { func (c *FS) get(h AUMHash) (*fsHashInfo, error) {
f, err := os.Open(filepath.Join(c.dirPrefix(h), c.filename(h))) dir, base := c.aumDir(h)
f, err := os.Open(filepath.Join(dir, base))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -266,6 +265,9 @@ func (c *FS) get(h AUMHash) (*fsHashInfo, error) {
if err := cbor.NewDecoder(f).Decode(&out); err != nil { if err := cbor.NewDecoder(f).Decode(&out); err != nil {
return nil, err return nil, err
} }
if out.AUM != nil && out.AUM.Hash() != h {
return nil, fmt.Errorf("%s: AUM does not match file name hash %s", f.Name(), out.AUM.Hash())
}
return &out, nil return &out, nil
} }
@ -297,24 +299,15 @@ func (c *FS) scanHashes(eachHashInfo func(*fsHashInfo)) error {
if !prefix.IsDir() { if !prefix.IsDir() {
continue continue
} }
pb, err := hex.DecodeString(prefix.Name())
if err != nil || len(pb) != 1 {
return fmt.Errorf("invalid prefix directory %q: %v", prefix.Name(), err)
}
files, err := os.ReadDir(filepath.Join(c.base, prefix.Name())) files, err := os.ReadDir(filepath.Join(c.base, prefix.Name()))
if err != nil { if err != nil {
return fmt.Errorf("reading prefix dir: %v", err) return fmt.Errorf("reading prefix dir: %v", err)
} }
for _, file := range files { for _, file := range files {
remainingHash, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(file.Name())
if err != nil {
return fmt.Errorf("invalid aum file %s/%s: %v", prefix.Name(), file.Name(), err)
}
var h AUMHash var h AUMHash
h[0] = pb[0] if err := h.UnmarshalText([]byte(file.Name())); err != nil {
copy(h[1:], remainingHash) return fmt.Errorf("invalid aum file: %s: %w", file.Name(), err)
}
info, err := c.get(h) info, err := c.get(h)
if err != nil { if err != nil {
return fmt.Errorf("reading %x: %v", h, err) return fmt.Errorf("reading %x: %v", h, err)
@ -422,7 +415,8 @@ func (c *FS) commit(h AUMHash, updater func(*fsHashInfo)) error {
return fmt.Errorf("cannot commit AUM with hash %x to %x", toCommit.AUM.Hash(), h) return fmt.Errorf("cannot commit AUM with hash %x to %x", toCommit.AUM.Hash(), h)
} }
if err := os.MkdirAll(c.dirPrefix(h), 0755); err != nil && !os.IsExist(err) { dir, base := c.aumDir(h)
if err := os.MkdirAll(dir, 0755); err != nil && !os.IsExist(err) {
return fmt.Errorf("creating directory: %v", err) return fmt.Errorf("creating directory: %v", err)
} }
@ -430,5 +424,5 @@ func (c *FS) commit(h AUMHash, updater func(*fsHashInfo)) error {
if err := cbor.NewEncoder(&buff).Encode(toCommit); err != nil { if err := cbor.NewEncoder(&buff).Encode(toCommit); err != nil {
return fmt.Errorf("encoding: %v", err) return fmt.Errorf("encoding: %v", err)
} }
return atomicfile.WriteFile(filepath.Join(c.dirPrefix(h), c.filename(h)), buff.Bytes(), 0644) return atomicfile.WriteFile(filepath.Join(dir, base), buff.Bytes(), 0644)
} }

@ -146,13 +146,17 @@ func TestTailchonkFS_Commit(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if got, want := chonk.filename(aum.Hash()), "HJX3LPJJQVRFSQX4QONESBU4DUO5JPORA66ZUCFS6NHZWDZTP4"; got != want { dir, base := chonk.aumDir(aum.Hash())
t.Errorf("aum filename = %q, want %q", got, want) if got, want := dir, filepath.Join(chonk.base, "VU"); got != want {
t.Errorf("aum dir=%s, want %s", got, want)
} }
if _, err := os.Stat(filepath.Join(chonk.base, "ad", "HJX3LPJJQVRFSQX4QONESBU4DUO5JPORA66ZUCFS6NHZWDZTP4")); err != nil { if want := "VU5G7NN5FGCWEWKC7SBZUSIGTQOR3VF52ED33GQIWLZU7GYPGN7Q"; base != want {
t.Errorf("aum base=%s, want %s", base, want)
}
if _, err := os.Stat(filepath.Join(dir, base)); err != nil {
t.Errorf("stat of AUM file failed: %v", err) t.Errorf("stat of AUM file failed: %v", err)
} }
if _, err := os.Stat(filepath.Join(chonk.base, "67", "226TIYPDKQWKFD5MXUI3GRVDSDFXRBABNINTFIT5ADMCLZ464U")); err != nil { if _, err := os.Stat(filepath.Join(chonk.base, "M7", "M7LL2NDB4NKCZIUPVS6RDM2GUOIMW6EEAFVBWMVCPUANQJPHT3SQ")); err != nil {
t.Errorf("stat of AUM parent failed: %v", err) t.Errorf("stat of AUM parent failed: %v", err)
} }
} }

Loading…
Cancel
Save