// Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause // Package art provides a routing table that implements the Allotment Routing // Table (ART) algorithm by Donald Knuth, as described in the paper by Yoichi // Hariguchi. // // ART outperforms the traditional radix tree implementations for route lookups, // insertions, and deletions. // // For more information, see Yoichi Hariguchi's paper: // https://cseweb.ucsd.edu//~varghese/TEACH/cs228/artlookup.pdf package art import ( "bytes" "fmt" "io" "math/bits" "net/netip" "strings" "sync" ) const ( debugInsert = false debugDelete = false ) // Table is an IPv4 and IPv6 routing table. type Table[T any] struct { v4 strideTable[T] v6 strideTable[T] initOnce sync.Once } func (t *Table[T]) init() { t.initOnce.Do(func() { t.v4.prefix = netip.PrefixFrom(netip.IPv4Unspecified(), 0) t.v6.prefix = netip.PrefixFrom(netip.IPv6Unspecified(), 0) }) } // Get does a route lookup for addr and returns the associated value, or nil if // no route matched. func (t *Table[T]) Get(addr netip.Addr) *T { t.init() st := &t.v4 if addr.Is6() { st = &t.v6 } i := 0 bs := addr.AsSlice() // With path compression, we might skip over some address bits while walking // to a strideTable leaf. This means the leaf answer we find might not be // correct, because path compression took us down the wrong subtree. When // that happens, we have to backtrack and figure out which most specific // route further up the tree is relevant to addr, and return that. // // So, as we walk down the stride tables, each time we find a non-nil route // result, we have to remember it and the associated strideTable prefix. // // We could also deal with this edge case of path compression by checking // the strideTable prefix on each table as we descend, but that means we // have to pay N prefix.Contains checks on every route lookup (where N is // the number of strideTables in the path), rather than only paying M prefix // comparisons in the edge case (where M is the number of strideTables in // the path with a non-nil route of their own). strideIdx := 0 stridePrefixes := [16]netip.Prefix{} strideRoutes := [16]*T{} findLeaf: for { rt, child := st.getValAndChild(bs[i]) if rt != nil { // This strideTable contains a route that may be relevant to our // search, remember it. stridePrefixes[strideIdx] = st.prefix strideRoutes[strideIdx] = rt strideIdx++ } if child == nil { // No sub-routes further down, the last thing we recorded // in strideRoutes is tentatively the result, barring // misdirection from path compression. break findLeaf } st = child // Path compression means we may be skipping over some intermediate // tables. We have to skip forward to whatever depth st now references. i = st.prefix.Bits() / 8 } // Walk backwards through the hits we recorded in strideRoutes and // stridePrefixes, returning the first one whose subtree matches addr. // // In the common case where path compression did not mislead us, we'll // return on the first loop iteration because the last route we recorded was // the correct most-specific route. for strideIdx > 0 { strideIdx-- if stridePrefixes[strideIdx].Contains(addr) { return strideRoutes[strideIdx] } } // We either found no route hits at all (both previous loops terminated // immediately), or we went on a wild goose chase down a compressed path for // the wrong prefix, and also found no usable routes on the way back up to // the root. This is a miss. return nil } // Insert adds pfx to the table, with value val. // If pfx is already present in the table, its value is set to val. func (t *Table[T]) Insert(pfx netip.Prefix, val *T) { t.init() if val == nil { panic("Table.Insert called with nil value") } // The standard library doesn't enforce normalized prefixes (where // the non-prefix bits are all zero). These algorithms require // normalized prefixes, so do it upfront. pfx = pfx.Masked() if debugInsert { defer func() { fmt.Printf("%s", t.debugSummary()) }() fmt.Printf("\ninsert: start pfx=%s\n", pfx) } st := &t.v4 if pfx.Addr().Is6() { st = &t.v6 } // This algorithm is full of off-by-one headaches that boil down // to the fact that pfx.Bits() has (2^n)+1 values, rather than // just 2^n. For example, an IPv4 prefix length can be 0 through // 32, which is 33 values. // // This extra possible value creates a lot of problems as we do // bits and bytes math to traverse strideTables below. So, we // treat the default route 0/0 specially here, that way the rest // of the logic goes back to having 2^n values to reason about, // which can be done in a nice and regular fashion with no edge // cases. if pfx.Bits() == 0 { if debugInsert { fmt.Printf("insert: default route\n") } st.insert(0, 0, val) return } bs := pfx.Addr().AsSlice() // No matter what we do as we traverse strideTables, our final // action will be to insert the last 1-8 bits of pfx into a // strideTable somewhere. // // We calculate upfront the byte position in bs of the end of the // prefix; the number of bits within that byte that contain prefix // data; and the prefix of the strideTable into which we'll // eventually insert. // // We need this in a couple different branches of the code below, // and because the possible values are 1-indexed (1 through 32 for // ipv4, 1 through 128 for ipv6), the math is very slightly // unusual to account for the off-by-one indexing. Do it once up // here, with this large comment, rather than reproduce the subtle // math in multiple places further down. finalByteIdx := (pfx.Bits() - 1) / 8 finalBits := pfx.Bits() - (finalByteIdx * 8) finalStridePrefix, err := pfx.Addr().Prefix(finalByteIdx * 8) if err != nil { panic(fmt.Sprintf("invalid prefix requested: %s/%d", pfx.Addr(), finalByteIdx*8)) } if debugInsert { fmt.Printf("insert: finalByteIdx=%d finalBits=%d finalStridePrefix=%s\n", finalByteIdx, finalBits, finalStridePrefix) } // The strideTable we want to insert into is potentially at the // end of a chain of strideTables, each one encoding 8 bits of the // prefix. // // We're expecting to walk down a path of tables, although with // prefix compression we may end up skipping some links in the // chain, or taking wrong turns and having to course correct. // // As we walk down the tree, byteIdx is the byte of bs we're // currently examining to choose our next step, and numBits is the // number of bits that remain in pfx, starting with the byte at // byteIdx inclusive. byteIdx := 0 numBits := pfx.Bits() for { if debugInsert { fmt.Printf("insert: loop byteIdx=%d numBits=%d st.prefix=%s\n", byteIdx, numBits, st.prefix) } if numBits <= 8 { if debugInsert { fmt.Printf("insert: existing leaf st.prefix=%s addr=%d/%d\n", st.prefix, bs[finalByteIdx], finalBits) } // We've reached the end of the prefix, whichever // strideTable we're looking at now is the place where we // need to insert. st.insert(bs[finalByteIdx], finalBits, val) return } // Otherwise, we need to go down at least one more level of // strideTables. With prefix compression, each level of // descent can have one of three outcomes: we find a place // where prefix compression is possible; a place where prefix // compression made us take a "wrong turn"; or a point along // our intended path that we have to keep following. child, created := st.getOrCreateChild(bs[byteIdx]) switch { case created: // The subtree we need for pfx doesn't exist yet. The rest // of the path, if we were to create it, will consist of a // bunch of strideTables with a single child each. We can // use path compression to elide those intermediates, and // jump straight to the final strideTable that hosts this // prefix. child.prefix = finalStridePrefix child.insert(bs[finalByteIdx], finalBits, val) if debugInsert { fmt.Printf("insert: new leaf st.prefix=%s child.prefix=%s addr=%d/%d\n", st.prefix, child.prefix, bs[finalByteIdx], finalBits) } return case !prefixStrictlyContains(child.prefix, pfx): // child already exists, but its prefix does not contain // our destination. This means that the path between st // and child was compressed by a previous insertion, and // somewhere in the (implicit) compressed path we took a // wrong turn, into the wrong part of st's subtree. // // This is okay, because pfx and child.prefix must have a // common ancestor node somewhere between st and child. We // can figure out what node that is, and materialize it. // // Once we've done that, we can immediately complete the // remainder of the insertion in one of two ways, without // further traversal. See a little further down for what // those are. if debugInsert { fmt.Printf("insert: wrong turn, pfx=%s child.prefix=%s\n", pfx, child.prefix) } intermediatePrefix, addrOfExisting, addrOfNew := computePrefixSplit(child.prefix, pfx) intermediate := &strideTable[T]{prefix: intermediatePrefix} // TODO: make this whole thing be st.AddIntermediate or something? st.setChild(bs[byteIdx], intermediate) intermediate.setChild(addrOfExisting, child) if debugInsert { fmt.Printf("insert: new intermediate st.prefix=%s intermediate.prefix=%s child.prefix=%s\n", st.prefix, intermediate.prefix, child.prefix) } // Now, we have a chain of st -> intermediate -> child. // // pfx either lives in a different child of intermediate, // or in intermediate itself. For example, if we created // the intermediate 1.2.0.0/16, pfx=1.2.3.4/32 would have // to go into a new child of intermediate, but // pfx=1.2.0.0/18 would go into intermediate directly. if remain := pfx.Bits() - intermediate.prefix.Bits(); remain <= 8 { // pfx lives in intermediate. if debugInsert { fmt.Printf("insert: into intermediate intermediate.prefix=%s addr=%d/%d\n", intermediate.prefix, bs[finalByteIdx], finalBits) } intermediate.insert(bs[finalByteIdx], finalBits, val) } else { // pfx lives in a different child subtree of // intermediate. By definition this subtree doesn't // exist at all, otherwise we'd never have entered // this entire "wrong turn" codepath in the first // place. // // This means we can apply prefix compression as we // create this new child, and we're done. st, created = intermediate.getOrCreateChild(addrOfNew) if !created { panic("new child path unexpectedly exists during path decompression") } st.prefix = finalStridePrefix st.insert(bs[finalByteIdx], finalBits, val) if debugInsert { fmt.Printf("insert: new child st.prefix=%s addr=%d/%d\n", st.prefix, bs[finalByteIdx], finalBits) } } return default: // An expected child table exists along pfx's // path. Continue traversing downwards. st = child byteIdx = child.prefix.Bits() / 8 numBits = pfx.Bits() - child.prefix.Bits() if debugInsert { fmt.Printf("insert: descend st.prefix=%s\n", st.prefix) } } } } // Delete removes pfx from the table, if it is present. func (t *Table[T]) Delete(pfx netip.Prefix) { t.init() // The standard library doesn't enforce normalized prefixes (where // the non-prefix bits are all zero). These algorithms require // normalized prefixes, so do it upfront. pfx = pfx.Masked() if debugDelete { defer func() { fmt.Printf("%s", t.debugSummary()) }() fmt.Printf("\ndelete: start pfx=%s table:\n%s", pfx, t.debugSummary()) } st := &t.v4 if pfx.Addr().Is6() { st = &t.v6 } // This algorithm is full of off-by-one headaches, just like // Insert. See the comment in Insert for more details. Bottom // line: we handle the default route as a special case, and that // simplifies the rest of the code slightly. if pfx.Bits() == 0 { if debugDelete { fmt.Printf("delete: default route\n") } st.delete(0, 0) return } // Deletion may drive the refcount of some strideTables down to // zero. We need to clean up these dangling tables, so we have to // keep track of which tables we touch on the way down, and which // strideEntry index each child is registered in. // // Note that the strideIndex and strideTables entries are off-by-one. // The child table pointer is recorded at i+1, but it is referenced by a // particular index in the parent table, at index i. // // In other words: entry number strideIndexes[0] in // strideTables[0] is the same pointer as strideTables[1]. // // This results in some slightly odd array accesses further down // in this code, because in a single loop iteration we have to // write to strideTables[N] and strideIndexes[N-1]. strideIdx := 0 strideTables := [16]*strideTable[T]{st} strideIndexes := [15]int{} // Similar to Insert, navigate down the tree of strideTables, // looking for the one that houses this prefix. This part is // easier than with insertion, since we can bail if the path ends // early or takes an unexpected detour. However, unlike // insertion, there's a whole post-deletion cleanup phase later // on. // // As we walk down the tree, byteIdx is the byte of bs we're // currently examining to choose our next step, and numBits is the // number of bits that remain in pfx, starting with the byte at // byteIdx inclusive. bs := pfx.Addr().AsSlice() byteIdx := 0 numBits := pfx.Bits() for numBits > 8 { if debugDelete { fmt.Printf("delete: loop byteIdx=%d numBits=%d st.prefix=%s\n", byteIdx, numBits, st.prefix) } child, idx := st.getChild(bs[byteIdx]) if child == nil { // Prefix can't exist in the table, because one of the // necessary strideTables doesn't exist. if debugDelete { fmt.Printf("delete: missing necessary child pfx=%s\n", pfx) } return } strideIndexes[strideIdx] = idx strideTables[strideIdx+1] = child strideIdx++ // Path compression means byteIdx can jump forwards // unpredictably. Recompute the next byte to look at from the // child we just found. byteIdx = child.prefix.Bits() / 8 numBits = pfx.Bits() - child.prefix.Bits() st = child if debugDelete { fmt.Printf("delete: descend st.prefix=%s\n", st.prefix) } } // We reached a leaf stride table that seems to be in the right // spot. But path compression might have led us to the wrong // table. if !prefixStrictlyContains(st.prefix, pfx) { // Wrong table, the requested prefix can't exist since its // path led us to the wrong place. if debugDelete { fmt.Printf("delete: wrong leaf table pfx=%s\n", pfx) } return } if debugDelete { fmt.Printf("delete: delete from st.prefix=%s addr=%d/%d\n", st.prefix, bs[byteIdx], numBits) } if st.delete(bs[byteIdx], numBits) == nil { // We're in the right strideTable, but pfx wasn't in // it. Refcounts haven't changed, so we can skip cleanup. if debugDelete { fmt.Printf("delete: prefix not present pfx=%s\n", pfx) } return } // st.delete reduced st's refcount by one. This table may now be // reclaimable, and depending on how we can reclaim it, the parent // tables may also need to be reclaimed. This loop ends as soon as // an iteration takes no action, or takes an action that doesn't // alter the parent table's refcounts. // // We start our walk back at strideTables[strideIdx], which // contains st. for strideIdx > 0 { cur := strideTables[strideIdx] if debugDelete { fmt.Printf("delete: GC? strideIdx=%d st.prefix=%s\n", strideIdx, cur.prefix) } if cur.routeRefs > 0 { // the strideTable has other route entries, it cannot be // deleted or compacted. if debugDelete { fmt.Printf("delete: has other routes st.prefix=%s\n", cur.prefix) } return } switch cur.childRefs { case 0: // no routeRefs and no childRefs, this table can be // deleted. This will alter the parent table's refcount, // so we'll have to look at it as well (in the next loop // iteration). if debugDelete { fmt.Printf("delete: remove st.prefix=%s\n", cur.prefix) } strideTables[strideIdx-1].deleteChild(strideIndexes[strideIdx-1]) strideIdx-- case 1: // This table has no routes, and a single child. Compact // this table out of existence by making the parent point // directly at the one child. This does not affect the // parent's refcounts, so the parent can't be eligible for // deletion or compaction, and we can stop. child := strideTables[strideIdx].findFirstChild() // only 1 child exists, by definition parent := strideTables[strideIdx-1] if debugDelete { fmt.Printf("delete: compact parent.prefix=%s st.prefix=%s child.prefix=%s\n", parent.prefix, cur.prefix, child.prefix) } strideTables[strideIdx-1].setChildByIndex(strideIndexes[strideIdx-1], child) return default: // This table has two or more children, so it's acting as a "fork in // the road" between two prefix subtrees. It cannot be deleted, and // thus no further cleanups are possible. if debugDelete { fmt.Printf("delete: fork table st.prefix=%s\n", cur.prefix) } return } } } // debugSummary prints the tree of allocated strideTables in t, with each // strideTable's refcount. func (t *Table[T]) debugSummary() string { t.init() var ret bytes.Buffer fmt.Fprintf(&ret, "v4: ") strideSummary(&ret, &t.v4, 4) fmt.Fprintf(&ret, "v6: ") strideSummary(&ret, &t.v6, 4) return ret.String() } func strideSummary[T any](w io.Writer, st *strideTable[T], indent int) { fmt.Fprintf(w, "%s: %d routes, %d children\n", st.prefix, st.routeRefs, st.childRefs) indent += 4 st.treeDebugStringRec(w, 1, indent) for i := firstHostIndex; i <= lastHostIndex; i++ { if child := st.entries[i].child; child != nil { addr, len := inversePrefixIndex(i) fmt.Fprintf(w, "%s%d/%d (%02x/%d): ", strings.Repeat(" ", indent), addr, len, addr, len) strideSummary(w, child, indent) } } } // prefixStrictlyContains reports whether child is a prefix within // parent, but not parent itself. func prefixStrictlyContains(parent, child netip.Prefix) bool { return parent.Overlaps(child) && parent.Bits() < child.Bits() } // computePrefixSplit returns the smallest common prefix that contains // both a and b. lastCommon is 8-bit aligned, with aStride and bStride // indicating the value of the 8-bit stride immediately following // lastCommon. // // computePrefixSplit is used in constructing an intermediate // strideTable when a new prefix needs to be inserted in a compressed // table. It can be read as: given that a is already in the table, and // b is being inserted, what is the prefix of the new intermediate // strideTable that needs to be created, and at what addresses in that // new strideTable should a and b's subsequent strideTables be // attached? // // Note as a special case, this can be called with a==b. An example of // when this happens: // - We want to insert the prefix 1.2.0.0/16 // - A strideTable exists for 1.2.0.0/16, because another child // prefix already exists (e.g. 1.2.3.4/32) // - The 1.0.0.0/8 strideTable does not exist, because path // compression removed it. // // In this scenario, the caller of computePrefixSplit ends up making a // "wrong turn" while traversing strideTables: it was looking for the // 1.0.0.0/8 table, but ended up at the 1.2.0.0/16 table. When this // happens, it will invoke computePrefixSplit(1.2.0.0/16, 1.2.0.0/16), // and we return 1.0.0.0/8 as the missing intermediate. func computePrefixSplit(a, b netip.Prefix) (lastCommon netip.Prefix, aStride, bStride uint8) { a = a.Masked() b = b.Masked() if a.Bits() == 0 || b.Bits() == 0 { panic("computePrefixSplit called with a default route") } if a.Addr().Is4() != b.Addr().Is4() { panic("computePrefixSplit called with mismatched address families") } minPrefixLen := a.Bits() if b.Bits() < minPrefixLen { minPrefixLen = b.Bits() } commonBits := commonBits(a.Addr(), b.Addr(), minPrefixLen) // We want to know how many 8-bit strides are shared between a and // b. Naively, this would be commonBits/8, but this introduces an // off-by-one error. This is due to the way our ART stores // prefixes whose length falls exactly on a stride boundary. // // Consider 192.168.1.0/24 and 192.168.0.0/16. commonBits // correctly reports that these prefixes have their first 16 bits // in common. However, in the ART they only share 1 common stride: // they both use the 192.0.0.0/8 strideTable, but 192.168.0.0/16 // is stored as 168/8 within that table, and not as 0/0 in the // 192.168.0.0/16 table. // // So, when commonBits matches the length of one of the inputs and // falls on a boundary between strides, the strideTable one // further up from commonBits/8 is the one we need to create, // which means we have to adjust the stride count down by one. if commonBits == minPrefixLen { commonBits-- } commonStrides := commonBits / 8 lastCommon, err := a.Addr().Prefix(commonStrides * 8) if err != nil { panic(fmt.Sprintf("computePrefixSplit constructing common prefix: %v", err)) } if a.Addr().Is4() { aStride = a.Addr().As4()[commonStrides] bStride = b.Addr().As4()[commonStrides] } else { aStride = a.Addr().As16()[commonStrides] bStride = b.Addr().As16()[commonStrides] } return lastCommon, aStride, bStride } // commonBits returns the number of common leading bits of a and b. // If the number of common bits exceeds maxBits, it returns maxBits // instead. func commonBits(a, b netip.Addr, maxBits int) int { if a.Is4() != b.Is4() { panic("commonStrides called with mismatched address families") } var common int // The following implements an old bit-twiddling trick to compute // the number of common leading bits: if you XOR two numbers // together, equal bits become 0 and unequal bits become 1. You // can then count the number of leading zeros (which is a single // instruction on modern CPUs) to get the answer. // // This code is a little more complex than just XOR + count // leading zeros, because IPv4 and IPv6 are different sizes, and // for IPv6 we have to do the math in two 64-bit chunks because Go // lacks a uint128 type. if a.Is4() { aNum, bNum := ipv4AsUint(a), ipv4AsUint(b) common = bits.LeadingZeros32(aNum ^ bNum) } else { aNumHi, aNumLo := ipv6AsUint(a) bNumHi, bNumLo := ipv6AsUint(b) common = bits.LeadingZeros64(aNumHi ^ bNumHi) if common == 64 { common += bits.LeadingZeros64(aNumLo ^ bNumLo) } } if common > maxBits { common = maxBits } return common } // ipv4AsUint returns ip as a uint32. func ipv4AsUint(ip netip.Addr) uint32 { bs := ip.As4() return uint32(bs[0])<<24 | uint32(bs[1])<<16 | uint32(bs[2])<<8 | uint32(bs[3]) } // ipv6AsUint returns ip as a pair of uint64s. func ipv6AsUint(ip netip.Addr) (uint64, uint64) { bs := ip.As16() hi := uint64(bs[0])<<56 | uint64(bs[1])<<48 | uint64(bs[2])<<40 | uint64(bs[3])<<32 | uint64(bs[4])<<24 | uint64(bs[5])<<16 | uint64(bs[6])<<8 | uint64(bs[7]) lo := uint64(bs[8])<<56 | uint64(bs[9])<<48 | uint64(bs[10])<<40 | uint64(bs[11])<<32 | uint64(bs[12])<<24 | uint64(bs[13])<<16 | uint64(bs[14])<<8 | uint64(bs[15]) return hi, lo }