util/deephash: remove unnecessary formatting for structs and slices (#2571)

The index for every struct field or slice element and
the number of fields for the struct is unncessary.

The hashing of Go values is unambiguous because every type (except maps)
encodes in a parsable manner. So long as we know the type information,
we could theoretically decode every value (except for maps).

At a high level:
* numbers are encoded as fixed-width records according to precision.
* strings (and AppendTo output) are encoded with a fixed-width length,
followed by the contents of the buffer.
* slices are prefixed by a fixed-width length, followed by the encoding
of each value. So long as we know the type of each element, we could
theoretically decode each element.
* arrays are encoded just like slices, but elide the length
since it is determined from the Go type.
* maps are encoded first with a byte indicating whether it is a cycle.
If a cycle, it is followed by a fixed-width index for the pointer,
otherwise followed by the SHA-256 hash of its contents. The encoding of maps
is not decodeable, but a SHA-256 hash is sufficient to avoid ambiguities.
* interfaces are encoded first with a byte indicating whether it is nil.
If not nil, it is followed by a fixed-width index for the type,
and then the encoding for the underlying value. Having the type be encoded
first ensures that the value could theoretically be decoded next.
* pointers are encoded first with a byte indicating whether it is
1) nil, 2) a cycle, or 3) newly seen. If a cycle, it is followed by
a fixed-width index for the pointer. If newly seen, it is followed by
the encoding for the pointed-at value.

Removing unnecessary details speeds up hashing:

	name              old time/op    new time/op    delta
	Hash-8              76.0µs ± 1%    55.8µs ± 2%  -26.62%        (p=0.000 n=10+10)
	HashMapAcyclic-8    61.9µs ± 0%    62.0µs ± 0%     ~             (p=0.666 n=9+9)
	TailcfgNode-8       10.2µs ± 1%     7.5µs ± 1%  -26.90%         (p=0.000 n=10+9)
	HashArray-8         1.07µs ± 1%    0.70µs ± 1%  -34.67%         (p=0.000 n=10+9)

Signed-off-by: Joe Tsai <joetsai@digital-static.net>

util/deephash/deephash.go

@@ -8,8 +8,15 @@
 // The hash is sufficiently strong and unique such that
 // Hash(x) == Hash(y) is an appropriate replacement for x == y.
 //
-// This package, like most of the tailscale.com Go module, should be
+// The definition of equality is identical to reflect.DeepEqual except:
-// considered Tailscale-internal; we make no API promises.
+//	* Floating-point values are compared based on the raw bits,
+//	  which means that NaNs (with the same bit pattern) are treated as equal.
+//	* Types which implement interface { AppendTo([]byte) []byte } use
+//	  the AppendTo method to produce a textual representation of the value.
+//	  Thus, two values are equal if AppendTo produces the same bytes.
+//
+// WARNING: This package, like most of the tailscale.com Go module,
+// should be considered Tailscale-internal; we make no API promises.
 package deephash
 
 import (
@@ -26,6 +33,33 @@ import (
 	"unsafe"
 )
 
+// There is much overlap between the theory of serialization and hashing.
+// A hash (useful for determing equality) can be produced by printing a value
+// and hashing the output. The format must:
+//	* be deterministic such that the same value hashes to the same output, and
+//	* be parsable such that the same value can be reproduced by the output.
+//
+// The logic below hashes a value by printing it to a hash.Hash.
+// To be parsable, it assumes that we know the Go type of each value:
+//	* scalar types (e.g., bool or int32) are printed as fixed-width fields.
+//	* list types (e.g., strings, slices, and AppendTo buffers) are prefixed
+//	  by a fixed-width length field, followed by the contents of the list.
+//	* slices, arrays, and structs print each element/field consecutively.
+//	* interfaces print with a 1-byte prefix indicating whether it is nil.
+//	  If non-nil, it is followed by a fixed-width field of the type index,
+//	  followed by the format of the underlying value.
+//	* pointers print with a 1-byte prefix indicating whether the pointer is
+//	  1) nil, 2) previously seen, or 3) newly seen. Previously seen pointers are
+//	  followed by a fixed-width field with the index of the previous pointer.
+//	  Newly seen pointers are followed by the format of the underlying value.
+//	* maps print with a 1-byte prefix indicating whether the map pointer is
+//	  1) nil, 2) previously seen, or 3) newly seen. Previously seen pointers
+//	  are followed by a fixed-width field of the index of the previous pointer.
+//	  Newly seen maps are printed as a fixed-width field with the XOR of the
+//	  hash of every map entry. With a sufficiently strong hash, this value is
+//	  theoretically "parsable" by looking up the hash in a magical map that
+//	  returns the set of entries for that given hash.
+
 const scratchSize = 128
 
 // hasher is reusable state for hashing a value.
@@ -174,10 +208,7 @@ func (h *hasher) hashValue(v reflect.Value) {
 		h.hashUint8(1) // indicates visiting a pointer
 		h.hashValue(v.Elem())
 	case reflect.Struct:
-		w.WriteString("struct")
-		h.hashUint64(uint64(v.NumField()))
 		for i, n := 0, v.NumField(); i < n; i++ {
-			h.hashUint64(uint64(i))
 			h.hashValue(v.Field(i))
 		}
 	case reflect.Slice, reflect.Array:
@@ -202,7 +233,6 @@ func (h *hasher) hashValue(v reflect.Value) {
 			// TODO(dsnet): Perform cycle detection for slices,
 			// which is functionally a list of pointers.
 			// See https://github.com/google/go-cmp/blob/402949e8139bb890c71a707b6faf6dd05c92f4e5/cmp/compare.go#L438-L450
-			h.hashUint64(uint64(i))
 			h.hashValue(v.Index(i))
 		}
 	case reflect.Interface:

util/deephash/deephash_test.go

@@ -269,11 +269,7 @@ func TestPrintArray(t *testing.T) {
 	h := &hasher{bw: bw}
 	h.hashValue(reflect.ValueOf(x))
 	bw.Flush()
-	const want = "struct" +
+	const want = "\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f"
-		"\x01\x00\x00\x00\x00\x00\x00\x00" + // 1 field
-		"\x00\x00\x00\x00\x00\x00\x00\x00" + // 0th field
-		// the 32 bytes:
-		"\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f"
 	if got := got.Bytes(); string(got) != want {
 		t.Errorf("wrong:\n got: %q\nwant: %q\n", got, want)
 	}