mirror of https://github.com/tailscale/tailscale/
derp/xdp,cmd/xdpderper: initial skeleton (#12390)
This commit introduces a userspace program for managing an experimental eBPF XDP STUN server program. derp/xdp contains the eBPF pseudo-C along with a Go pkg for loading it and exporting its metrics. cmd/xdpderper is a package main user of derp/xdp. Updates tailscale/corp#20689 Signed-off-by: Jordan Whited <jordan@tailscale.com>pull/10639/merge
Jordan Whited
1 week ago
committed by
GitHub
parent
6908fb0de3
commit
65888d95c9
@ -0,0 +1,76 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"tailscale.com/derp/xdp"
|
||||
"tailscale.com/tsweb"
|
||||
)
|
||||
|
||||
var (
|
||||
flagDevice = flag.String("device", "", "target device name")
|
||||
flagPort = flag.Int("dst-port", 0, "destination UDP port to serve")
|
||||
flagVerbose = flag.Bool("verbose", false, "verbose output including verifier errors")
|
||||
flagMode = flag.String("mode", "xdp", "XDP mode; valid modes: [xdp, xdpgeneric, xdpdrv, xdpoffload]")
|
||||
flagHTTP = flag.String("http", ":8230", "HTTP listen address")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
var attachFlags xdp.XDPAttachFlags
|
||||
switch strings.ToLower(*flagMode) {
|
||||
case "xdp":
|
||||
attachFlags = 0
|
||||
case "xdpgeneric":
|
||||
attachFlags = xdp.XDPGenericMode
|
||||
case "xdpdrv":
|
||||
attachFlags = xdp.XDPDriverMode
|
||||
case "xdpoffload":
|
||||
attachFlags = xdp.XDPOffloadMode
|
||||
default:
|
||||
log.Fatal("invalid mode")
|
||||
}
|
||||
server, err := xdp.NewSTUNServer(&xdp.STUNServerConfig{
|
||||
DeviceName: *flagDevice,
|
||||
DstPort: *flagPort,
|
||||
AttachFlags: attachFlags,
|
||||
FullVerifierErr: *flagVerbose,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("failed to init XDP STUN server: %v", err)
|
||||
}
|
||||
defer server.Close()
|
||||
err = prometheus.Register(server)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to register XDP STUN server as a prometheus collector: %v", err)
|
||||
}
|
||||
log.Println("XDP STUN server started")
|
||||
|
||||
mux := http.NewServeMux()
|
||||
tsweb.Debugger(mux)
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
err := http.ListenAndServe(*flagHTTP, mux)
|
||||
errCh <- err
|
||||
}()
|
||||
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
log.Printf("HTTP serve err: %v", err)
|
||||
case sig := <-sigCh:
|
||||
log.Printf("received signal: %s", sig)
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,166 @@
|
||||
// Code generated by bpf2go; DO NOT EDIT.
|
||||
//go:build mips || mips64 || ppc64 || s390x
|
||||
|
||||
package xdp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
)
|
||||
|
||||
type bpfConfig struct{ DstPort uint16 }
|
||||
|
||||
type bpfCounterKeyAf uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_UNKNOWN bpfCounterKeyAf = 0
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_IPV4 bpfCounterKeyAf = 1
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_IPV6 bpfCounterKeyAf = 2
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_LEN bpfCounterKeyAf = 3
|
||||
)
|
||||
|
||||
type bpfCounterKeyPacketsBytesAction uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL bpfCounterKeyPacketsBytesAction = 0
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL bpfCounterKeyPacketsBytesAction = 1
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL bpfCounterKeyPacketsBytesAction = 2
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL bpfCounterKeyPacketsBytesAction = 3
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL bpfCounterKeyPacketsBytesAction = 4
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL bpfCounterKeyPacketsBytesAction = 5
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL bpfCounterKeyPacketsBytesAction = 6
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL bpfCounterKeyPacketsBytesAction = 7
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN bpfCounterKeyPacketsBytesAction = 8
|
||||
)
|
||||
|
||||
type bpfCounterKeyProgEnd uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED bpfCounterKeyProgEnd = 0
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR bpfCounterKeyProgEnd = 1
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM bpfCounterKeyProgEnd = 2
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM bpfCounterKeyProgEnd = 3
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_NOT_STUN_PORT bpfCounterKeyProgEnd = 4
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL bpfCounterKeyProgEnd = 5
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_LEN bpfCounterKeyProgEnd = 6
|
||||
)
|
||||
|
||||
type bpfCountersKey struct {
|
||||
Unused uint8
|
||||
Af uint8
|
||||
Pba uint8
|
||||
ProgEnd uint8
|
||||
}
|
||||
|
||||
// loadBpf returns the embedded CollectionSpec for bpf.
|
||||
func loadBpf() (*ebpf.CollectionSpec, error) {
|
||||
reader := bytes.NewReader(_BpfBytes)
|
||||
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't load bpf: %w", err)
|
||||
}
|
||||
|
||||
return spec, err
|
||||
}
|
||||
|
||||
// loadBpfObjects loads bpf and converts it into a struct.
|
||||
//
|
||||
// The following types are suitable as obj argument:
|
||||
//
|
||||
// *bpfObjects
|
||||
// *bpfPrograms
|
||||
// *bpfMaps
|
||||
//
|
||||
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
|
||||
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
spec, err := loadBpf()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return spec.LoadAndAssign(obj, opts)
|
||||
}
|
||||
|
||||
// bpfSpecs contains maps and programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfSpecs struct {
|
||||
bpfProgramSpecs
|
||||
bpfMapSpecs
|
||||
}
|
||||
|
||||
// bpfSpecs contains programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfProgramSpecs struct {
|
||||
XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"`
|
||||
}
|
||||
|
||||
// bpfMapSpecs contains maps before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfMapSpecs struct {
|
||||
ConfigMap *ebpf.MapSpec `ebpf:"config_map"`
|
||||
CountersMap *ebpf.MapSpec `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
// bpfObjects contains all objects after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfObjects struct {
|
||||
bpfPrograms
|
||||
bpfMaps
|
||||
}
|
||||
|
||||
func (o *bpfObjects) Close() error {
|
||||
return _BpfClose(
|
||||
&o.bpfPrograms,
|
||||
&o.bpfMaps,
|
||||
)
|
||||
}
|
||||
|
||||
// bpfMaps contains all maps after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfMaps struct {
|
||||
ConfigMap *ebpf.Map `ebpf:"config_map"`
|
||||
CountersMap *ebpf.Map `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
func (m *bpfMaps) Close() error {
|
||||
return _BpfClose(
|
||||
m.ConfigMap,
|
||||
m.CountersMap,
|
||||
)
|
||||
}
|
||||
|
||||
// bpfPrograms contains all programs after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfPrograms struct {
|
||||
XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"`
|
||||
}
|
||||
|
||||
func (p *bpfPrograms) Close() error {
|
||||
return _BpfClose(
|
||||
p.XdpProgFunc,
|
||||
)
|
||||
}
|
||||
|
||||
func _BpfClose(closers ...io.Closer) error {
|
||||
for _, closer := range closers {
|
||||
if err := closer.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Do not access this directly.
|
||||
//
|
||||
//go:embed bpf_bpfeb.o
|
||||
var _BpfBytes []byte
|
||||
Binary file not shown.
@ -0,0 +1,166 @@
|
||||
// Code generated by bpf2go; DO NOT EDIT.
|
||||
//go:build 386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64
|
||||
|
||||
package xdp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
)
|
||||
|
||||
type bpfConfig struct{ DstPort uint16 }
|
||||
|
||||
type bpfCounterKeyAf uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_UNKNOWN bpfCounterKeyAf = 0
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_IPV4 bpfCounterKeyAf = 1
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_IPV6 bpfCounterKeyAf = 2
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_LEN bpfCounterKeyAf = 3
|
||||
)
|
||||
|
||||
type bpfCounterKeyPacketsBytesAction uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL bpfCounterKeyPacketsBytesAction = 0
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL bpfCounterKeyPacketsBytesAction = 1
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL bpfCounterKeyPacketsBytesAction = 2
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL bpfCounterKeyPacketsBytesAction = 3
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL bpfCounterKeyPacketsBytesAction = 4
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL bpfCounterKeyPacketsBytesAction = 5
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL bpfCounterKeyPacketsBytesAction = 6
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL bpfCounterKeyPacketsBytesAction = 7
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN bpfCounterKeyPacketsBytesAction = 8
|
||||
)
|
||||
|
||||
type bpfCounterKeyProgEnd uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED bpfCounterKeyProgEnd = 0
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR bpfCounterKeyProgEnd = 1
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM bpfCounterKeyProgEnd = 2
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM bpfCounterKeyProgEnd = 3
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_NOT_STUN_PORT bpfCounterKeyProgEnd = 4
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL bpfCounterKeyProgEnd = 5
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_LEN bpfCounterKeyProgEnd = 6
|
||||
)
|
||||
|
||||
type bpfCountersKey struct {
|
||||
Unused uint8
|
||||
Af uint8
|
||||
Pba uint8
|
||||
ProgEnd uint8
|
||||
}
|
||||
|
||||
// loadBpf returns the embedded CollectionSpec for bpf.
|
||||
func loadBpf() (*ebpf.CollectionSpec, error) {
|
||||
reader := bytes.NewReader(_BpfBytes)
|
||||
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't load bpf: %w", err)
|
||||
}
|
||||
|
||||
return spec, err
|
||||
}
|
||||
|
||||
// loadBpfObjects loads bpf and converts it into a struct.
|
||||
//
|
||||
// The following types are suitable as obj argument:
|
||||
//
|
||||
// *bpfObjects
|
||||
// *bpfPrograms
|
||||
// *bpfMaps
|
||||
//
|
||||
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
|
||||
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
spec, err := loadBpf()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return spec.LoadAndAssign(obj, opts)
|
||||
}
|
||||
|
||||
// bpfSpecs contains maps and programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfSpecs struct {
|
||||
bpfProgramSpecs
|
||||
bpfMapSpecs
|
||||
}
|
||||
|
||||
// bpfSpecs contains programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfProgramSpecs struct {
|
||||
XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"`
|
||||
}
|
||||
|
||||
// bpfMapSpecs contains maps before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfMapSpecs struct {
|
||||
ConfigMap *ebpf.MapSpec `ebpf:"config_map"`
|
||||
CountersMap *ebpf.MapSpec `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
// bpfObjects contains all objects after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfObjects struct {
|
||||
bpfPrograms
|
||||
bpfMaps
|
||||
}
|
||||
|
||||
func (o *bpfObjects) Close() error {
|
||||
return _BpfClose(
|
||||
&o.bpfPrograms,
|
||||
&o.bpfMaps,
|
||||
)
|
||||
}
|
||||
|
||||
// bpfMaps contains all maps after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfMaps struct {
|
||||
ConfigMap *ebpf.Map `ebpf:"config_map"`
|
||||
CountersMap *ebpf.Map `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
func (m *bpfMaps) Close() error {
|
||||
return _BpfClose(
|
||||
m.ConfigMap,
|
||||
m.CountersMap,
|
||||
)
|
||||
}
|
||||
|
||||
// bpfPrograms contains all programs after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfPrograms struct {
|
||||
XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"`
|
||||
}
|
||||
|
||||
func (p *bpfPrograms) Close() error {
|
||||
return _BpfClose(
|
||||
p.XdpProgFunc,
|
||||
)
|
||||
}
|
||||
|
||||
func _BpfClose(closers ...io.Closer) error {
|
||||
for _, closer := range closers {
|
||||
if err := closer.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Do not access this directly.
|
||||
//
|
||||
//go:embed bpf_bpfel.o
|
||||
var _BpfBytes []byte
|
||||
Binary file not shown.
@ -0,0 +1,32 @@
|
||||
Valid-License-Identifier: BSD-2-Clause
|
||||
SPDX-URL: https://spdx.org/licenses/BSD-2-Clause.html
|
||||
Usage-Guide:
|
||||
To use the BSD 2-clause "Simplified" License put the following SPDX
|
||||
tag/value pair into a comment according to the placement guidelines in
|
||||
the licensing rules documentation:
|
||||
SPDX-License-Identifier: BSD-2-Clause
|
||||
License-Text:
|
||||
|
||||
Copyright (c) 2015 The Libbpf Authors. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
@ -0,0 +1,99 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_ENDIAN__
|
||||
#define __BPF_ENDIAN__
|
||||
|
||||
/*
|
||||
* Isolate byte #n and put it into byte #m, for __u##b type.
|
||||
* E.g., moving byte #6 (nnnnnnnn) into byte #1 (mmmmmmmm) for __u64:
|
||||
* 1) xxxxxxxx nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx
|
||||
* 2) nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx 00000000
|
||||
* 3) 00000000 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn
|
||||
* 4) 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn 00000000
|
||||
*/
|
||||
#define ___bpf_mvb(x, b, n, m) ((__u##b)(x) << (b-(n+1)*8) >> (b-8) << (m*8))
|
||||
|
||||
#define ___bpf_swab16(x) ((__u16)( \
|
||||
___bpf_mvb(x, 16, 0, 1) | \
|
||||
___bpf_mvb(x, 16, 1, 0)))
|
||||
|
||||
#define ___bpf_swab32(x) ((__u32)( \
|
||||
___bpf_mvb(x, 32, 0, 3) | \
|
||||
___bpf_mvb(x, 32, 1, 2) | \
|
||||
___bpf_mvb(x, 32, 2, 1) | \
|
||||
___bpf_mvb(x, 32, 3, 0)))
|
||||
|
||||
#define ___bpf_swab64(x) ((__u64)( \
|
||||
___bpf_mvb(x, 64, 0, 7) | \
|
||||
___bpf_mvb(x, 64, 1, 6) | \
|
||||
___bpf_mvb(x, 64, 2, 5) | \
|
||||
___bpf_mvb(x, 64, 3, 4) | \
|
||||
___bpf_mvb(x, 64, 4, 3) | \
|
||||
___bpf_mvb(x, 64, 5, 2) | \
|
||||
___bpf_mvb(x, 64, 6, 1) | \
|
||||
___bpf_mvb(x, 64, 7, 0)))
|
||||
|
||||
/* LLVM's BPF target selects the endianness of the CPU
|
||||
* it compiles on, or the user specifies (bpfel/bpfeb),
|
||||
* respectively. The used __BYTE_ORDER__ is defined by
|
||||
* the compiler, we cannot rely on __BYTE_ORDER from
|
||||
* libc headers, since it doesn't reflect the actual
|
||||
* requested byte order.
|
||||
*
|
||||
* Note, LLVM's BPF target has different __builtin_bswapX()
|
||||
* semantics. It does map to BPF_ALU | BPF_END | BPF_TO_BE
|
||||
* in bpfel and bpfeb case, which means below, that we map
|
||||
* to cpu_to_be16(). We could use it unconditionally in BPF
|
||||
* case, but better not rely on it, so that this header here
|
||||
* can be used from application and BPF program side, which
|
||||
* use different targets.
|
||||
*/
|
||||
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||
# define __bpf_ntohs(x) __builtin_bswap16(x)
|
||||
# define __bpf_htons(x) __builtin_bswap16(x)
|
||||
# define __bpf_constant_ntohs(x) ___bpf_swab16(x)
|
||||
# define __bpf_constant_htons(x) ___bpf_swab16(x)
|
||||
# define __bpf_ntohl(x) __builtin_bswap32(x)
|
||||
# define __bpf_htonl(x) __builtin_bswap32(x)
|
||||
# define __bpf_constant_ntohl(x) ___bpf_swab32(x)
|
||||
# define __bpf_constant_htonl(x) ___bpf_swab32(x)
|
||||
# define __bpf_be64_to_cpu(x) __builtin_bswap64(x)
|
||||
# define __bpf_cpu_to_be64(x) __builtin_bswap64(x)
|
||||
# define __bpf_constant_be64_to_cpu(x) ___bpf_swab64(x)
|
||||
# define __bpf_constant_cpu_to_be64(x) ___bpf_swab64(x)
|
||||
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
# define __bpf_ntohs(x) (x)
|
||||
# define __bpf_htons(x) (x)
|
||||
# define __bpf_constant_ntohs(x) (x)
|
||||
# define __bpf_constant_htons(x) (x)
|
||||
# define __bpf_ntohl(x) (x)
|
||||
# define __bpf_htonl(x) (x)
|
||||
# define __bpf_constant_ntohl(x) (x)
|
||||
# define __bpf_constant_htonl(x) (x)
|
||||
# define __bpf_be64_to_cpu(x) (x)
|
||||
# define __bpf_cpu_to_be64(x) (x)
|
||||
# define __bpf_constant_be64_to_cpu(x) (x)
|
||||
# define __bpf_constant_cpu_to_be64(x) (x)
|
||||
#else
|
||||
# error "Fix your compiler's __BYTE_ORDER__?!"
|
||||
#endif
|
||||
|
||||
#define bpf_htons(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_htons(x) : __bpf_htons(x))
|
||||
#define bpf_ntohs(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_ntohs(x) : __bpf_ntohs(x))
|
||||
#define bpf_htonl(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_htonl(x) : __bpf_htonl(x))
|
||||
#define bpf_ntohl(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_ntohl(x) : __bpf_ntohl(x))
|
||||
#define bpf_cpu_to_be64(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_cpu_to_be64(x) : __bpf_cpu_to_be64(x))
|
||||
#define bpf_be64_to_cpu(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_be64_to_cpu(x) : __bpf_be64_to_cpu(x))
|
||||
|
||||
#endif /* __BPF_ENDIAN__ */
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,410 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_HELPERS__
|
||||
#define __BPF_HELPERS__
|
||||
|
||||
/*
|
||||
* Note that bpf programs need to include either
|
||||
* vmlinux.h (auto-generated from BTF) or linux/types.h
|
||||
* in advance since bpf_helper_defs.h uses such types
|
||||
* as __u64.
|
||||
*/
|
||||
#include "bpf_helper_defs.h"
|
||||
|
||||
#define __uint(name, val) int (*name)[val]
|
||||
#define __type(name, val) typeof(val) *name
|
||||
#define __array(name, val) typeof(val) *name[]
|
||||
#define __ulong(name, val) enum { ___bpf_concat(__unique_value, __COUNTER__) = val } name
|
||||
|
||||
/*
|
||||
* Helper macro to place programs, maps, license in
|
||||
* different sections in elf_bpf file. Section names
|
||||
* are interpreted by libbpf depending on the context (BPF programs, BPF maps,
|
||||
* extern variables, etc).
|
||||
* To allow use of SEC() with externs (e.g., for extern .maps declarations),
|
||||
* make sure __attribute__((unused)) doesn't trigger compilation warning.
|
||||
*/
|
||||
#if __GNUC__ && !__clang__
|
||||
|
||||
/*
|
||||
* Pragma macros are broken on GCC
|
||||
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578
|
||||
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90400
|
||||
*/
|
||||
#define SEC(name) __attribute__((section(name), used))
|
||||
|
||||
#else
|
||||
|
||||
#define SEC(name) \
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wignored-attributes\"") \
|
||||
__attribute__((section(name), used)) \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
|
||||
#endif
|
||||
|
||||
/* Avoid 'linux/stddef.h' definition of '__always_inline'. */
|
||||
#undef __always_inline
|
||||
#define __always_inline inline __attribute__((always_inline))
|
||||
|
||||
#ifndef __noinline
|
||||
#define __noinline __attribute__((noinline))
|
||||
#endif
|
||||
#ifndef __weak
|
||||
#define __weak __attribute__((weak))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Use __hidden attribute to mark a non-static BPF subprogram effectively
|
||||
* static for BPF verifier's verification algorithm purposes, allowing more
|
||||
* extensive and permissive BPF verification process, taking into account
|
||||
* subprogram's caller context.
|
||||
*/
|
||||
#define __hidden __attribute__((visibility("hidden")))
|
||||
|
||||
/* When utilizing vmlinux.h with BPF CO-RE, user BPF programs can't include
|
||||
* any system-level headers (such as stddef.h, linux/version.h, etc), and
|
||||
* commonly-used macros like NULL and KERNEL_VERSION aren't available through
|
||||
* vmlinux.h. This just adds unnecessary hurdles and forces users to re-define
|
||||
* them on their own. So as a convenience, provide such definitions here.
|
||||
*/
|
||||
#ifndef NULL
|
||||
#define NULL ((void *)0)
|
||||
#endif
|
||||
|
||||
#ifndef KERNEL_VERSION
|
||||
#define KERNEL_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Helper macros to manipulate data structures
|
||||
*/
|
||||
|
||||
/* offsetof() definition that uses __builtin_offset() might not preserve field
|
||||
* offset CO-RE relocation properly, so force-redefine offsetof() using
|
||||
* old-school approach which works with CO-RE correctly
|
||||
*/
|
||||
#undef offsetof
|
||||
#define offsetof(type, member) ((unsigned long)&((type *)0)->member)
|
||||
|
||||
/* redefined container_of() to ensure we use the above offsetof() macro */
|
||||
#undef container_of
|
||||
#define container_of(ptr, type, member) \
|
||||
({ \
|
||||
void *__mptr = (void *)(ptr); \
|
||||
((type *)(__mptr - offsetof(type, member))); \
|
||||
})
|
||||
|
||||
/*
|
||||
* Compiler (optimization) barrier.
|
||||
*/
|
||||
#ifndef barrier
|
||||
#define barrier() asm volatile("" ::: "memory")
|
||||
#endif
|
||||
|
||||
/* Variable-specific compiler (optimization) barrier. It's a no-op which makes
|
||||
* compiler believe that there is some black box modification of a given
|
||||
* variable and thus prevents compiler from making extra assumption about its
|
||||
* value and potential simplifications and optimizations on this variable.
|
||||
*
|
||||
* E.g., compiler might often delay or even omit 32-bit to 64-bit casting of
|
||||
* a variable, making some code patterns unverifiable. Putting barrier_var()
|
||||
* in place will ensure that cast is performed before the barrier_var()
|
||||
* invocation, because compiler has to pessimistically assume that embedded
|
||||
* asm section might perform some extra operations on that variable.
|
||||
*
|
||||
* This is a variable-specific variant of more global barrier().
|
||||
*/
|
||||
#ifndef barrier_var
|
||||
#define barrier_var(var) asm volatile("" : "+r"(var))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Helper macro to throw a compilation error if __bpf_unreachable() gets
|
||||
* built into the resulting code. This works given BPF back end does not
|
||||
* implement __builtin_trap(). This is useful to assert that certain paths
|
||||
* of the program code are never used and hence eliminated by the compiler.
|
||||
*
|
||||
* For example, consider a switch statement that covers known cases used by
|
||||
* the program. __bpf_unreachable() can then reside in the default case. If
|
||||
* the program gets extended such that a case is not covered in the switch
|
||||
* statement, then it will throw a build error due to the default case not
|
||||
* being compiled out.
|
||||
*/
|
||||
#ifndef __bpf_unreachable
|
||||
# define __bpf_unreachable() __builtin_trap()
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Helper function to perform a tail call with a constant/immediate map slot.
|
||||
*/
|
||||
#if __clang_major__ >= 8 && defined(__bpf__)
|
||||
static __always_inline void
|
||||
bpf_tail_call_static(void *ctx, const void *map, const __u32 slot)
|
||||
{
|
||||
if (!__builtin_constant_p(slot))
|
||||
__bpf_unreachable();
|
||||
|
||||
/*
|
||||
* Provide a hard guarantee that LLVM won't optimize setting r2 (map
|
||||
* pointer) and r3 (constant map index) from _different paths_ ending
|
||||
* up at the _same_ call insn as otherwise we won't be able to use the
|
||||
* jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel
|
||||
* given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key
|
||||
* tracking for prog array pokes") for details on verifier tracking.
|
||||
*
|
||||
* Note on clobber list: we need to stay in-line with BPF calling
|
||||
* convention, so even if we don't end up using r0, r4, r5, we need
|
||||
* to mark them as clobber so that LLVM doesn't end up using them
|
||||
* before / after the call.
|
||||
*/
|
||||
asm volatile("r1 = %[ctx]\n\t"
|
||||
"r2 = %[map]\n\t"
|
||||
"r3 = %[slot]\n\t"
|
||||
"call 12"
|
||||
:: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot)
|
||||
: "r0", "r1", "r2", "r3", "r4", "r5");
|
||||
}
|
||||
#endif
|
||||
|
||||
enum libbpf_pin_type {
|
||||
LIBBPF_PIN_NONE,
|
||||
/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
|
||||
LIBBPF_PIN_BY_NAME,
|
||||
};
|
||||
|
||||
enum libbpf_tristate {
|
||||
TRI_NO = 0,
|
||||
TRI_YES = 1,
|
||||
TRI_MODULE = 2,
|
||||
};
|
||||
|
||||
#define __kconfig __attribute__((section(".kconfig")))
|
||||
#define __ksym __attribute__((section(".ksyms")))
|
||||
#define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted")))
|
||||
#define __kptr __attribute__((btf_type_tag("kptr")))
|
||||
#define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr")))
|
||||
|
||||
#define bpf_ksym_exists(sym) ({ \
|
||||
_Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak"); \
|
||||
!!sym; \
|
||||
})
|
||||
|
||||
#define __arg_ctx __attribute__((btf_decl_tag("arg:ctx")))
|
||||
#define __arg_nonnull __attribute((btf_decl_tag("arg:nonnull")))
|
||||
#define __arg_nullable __attribute((btf_decl_tag("arg:nullable")))
|
||||
#define __arg_trusted __attribute((btf_decl_tag("arg:trusted")))
|
||||
#define __arg_arena __attribute((btf_decl_tag("arg:arena")))
|
||||
|
||||
#ifndef ___bpf_concat
|
||||
#define ___bpf_concat(a, b) a ## b
|
||||
#endif
|
||||
#ifndef ___bpf_apply
|
||||
#define ___bpf_apply(fn, n) ___bpf_concat(fn, n)
|
||||
#endif
|
||||
#ifndef ___bpf_nth
|
||||
#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N
|
||||
#endif
|
||||
#ifndef ___bpf_narg
|
||||
#define ___bpf_narg(...) \
|
||||
___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
|
||||
#endif
|
||||
|
||||
#define ___bpf_fill0(arr, p, x) do {} while (0)
|
||||
#define ___bpf_fill1(arr, p, x) arr[p] = x
|
||||
#define ___bpf_fill2(arr, p, x, args...) arr[p] = x; ___bpf_fill1(arr, p + 1, args)
|
||||
#define ___bpf_fill3(arr, p, x, args...) arr[p] = x; ___bpf_fill2(arr, p + 1, args)
|
||||
#define ___bpf_fill4(arr, p, x, args...) arr[p] = x; ___bpf_fill3(arr, p + 1, args)
|
||||
#define ___bpf_fill5(arr, p, x, args...) arr[p] = x; ___bpf_fill4(arr, p + 1, args)
|
||||
#define ___bpf_fill6(arr, p, x, args...) arr[p] = x; ___bpf_fill5(arr, p + 1, args)
|
||||
#define ___bpf_fill7(arr, p, x, args...) arr[p] = x; ___bpf_fill6(arr, p + 1, args)
|
||||
#define ___bpf_fill8(arr, p, x, args...) arr[p] = x; ___bpf_fill7(arr, p + 1, args)
|
||||
#define ___bpf_fill9(arr, p, x, args...) arr[p] = x; ___bpf_fill8(arr, p + 1, args)
|
||||
#define ___bpf_fill10(arr, p, x, args...) arr[p] = x; ___bpf_fill9(arr, p + 1, args)
|
||||
#define ___bpf_fill11(arr, p, x, args...) arr[p] = x; ___bpf_fill10(arr, p + 1, args)
|
||||
#define ___bpf_fill12(arr, p, x, args...) arr[p] = x; ___bpf_fill11(arr, p + 1, args)
|
||||
#define ___bpf_fill(arr, args...) \
|
||||
___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args)
|
||||
|
||||
/*
|
||||
* BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values
|
||||
* in a structure.
|
||||
*/
|
||||
#define BPF_SEQ_PRINTF(seq, fmt, args...) \
|
||||
({ \
|
||||
static const char ___fmt[] = fmt; \
|
||||
unsigned long long ___param[___bpf_narg(args)]; \
|
||||
\
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
___bpf_fill(___param, args); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
\
|
||||
bpf_seq_printf(seq, ___fmt, sizeof(___fmt), \
|
||||
___param, sizeof(___param)); \
|
||||
})
|
||||
|
||||
/*
|
||||
* BPF_SNPRINTF wraps the bpf_snprintf helper with variadic arguments instead of
|
||||
* an array of u64.
|
||||
*/
|
||||
#define BPF_SNPRINTF(out, out_size, fmt, args...) \
|
||||
({ \
|
||||
static const char ___fmt[] = fmt; \
|
||||
unsigned long long ___param[___bpf_narg(args)]; \
|
||||
\
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
___bpf_fill(___param, args); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
\
|
||||
bpf_snprintf(out, out_size, ___fmt, \
|
||||
___param, sizeof(___param)); \
|
||||
})
|
||||
|
||||
#ifdef BPF_NO_GLOBAL_DATA
|
||||
#define BPF_PRINTK_FMT_MOD
|
||||
#else
|
||||
#define BPF_PRINTK_FMT_MOD static const
|
||||
#endif
|
||||
|
||||
#define __bpf_printk(fmt, ...) \
|
||||
({ \
|
||||
BPF_PRINTK_FMT_MOD char ____fmt[] = fmt; \
|
||||
bpf_trace_printk(____fmt, sizeof(____fmt), \
|
||||
##__VA_ARGS__); \
|
||||
})
|
||||
|
||||
/*
|
||||
* __bpf_vprintk wraps the bpf_trace_vprintk helper with variadic arguments
|
||||
* instead of an array of u64.
|
||||
*/
|
||||
#define __bpf_vprintk(fmt, args...) \
|
||||
({ \
|
||||
static const char ___fmt[] = fmt; \
|
||||
unsigned long long ___param[___bpf_narg(args)]; \
|
||||
\
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
___bpf_fill(___param, args); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
\
|
||||
bpf_trace_vprintk(___fmt, sizeof(___fmt), \
|
||||
___param, sizeof(___param)); \
|
||||
})
|
||||
|
||||
/* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args
|
||||
* Otherwise use __bpf_vprintk
|
||||
*/
|
||||
#define ___bpf_pick_printk(...) \
|
||||
___bpf_nth(_, ##__VA_ARGS__, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \
|
||||
__bpf_vprintk, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \
|
||||
__bpf_vprintk, __bpf_vprintk, __bpf_printk /*3*/, __bpf_printk /*2*/,\
|
||||
__bpf_printk /*1*/, __bpf_printk /*0*/)
|
||||
|
||||
/* Helper macro to print out debug messages */
|
||||
#define bpf_printk(fmt, args...) ___bpf_pick_printk(args)(fmt, ##args)
|
||||
|
||||
struct bpf_iter_num;
|
||||
|
||||
extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __weak __ksym;
|
||||
extern int *bpf_iter_num_next(struct bpf_iter_num *it) __weak __ksym;
|
||||
extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __weak __ksym;
|
||||
|
||||
#ifndef bpf_for_each
|
||||
/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for
|
||||
* using BPF open-coded iterators without having to write mundane explicit
|
||||
* low-level loop logic. Instead, it provides for()-like generic construct
|
||||
* that can be used pretty naturally. E.g., for some hypothetical cgroup
|
||||
* iterator, you'd write:
|
||||
*
|
||||
* struct cgroup *cg, *parent_cg = <...>;
|
||||
*
|
||||
* bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) {
|
||||
* bpf_printk("Child cgroup id = %d", cg->cgroup_id);
|
||||
* if (cg->cgroup_id == 123)
|
||||
* break;
|
||||
* }
|
||||
*
|
||||
* I.e., it looks almost like high-level for each loop in other languages,
|
||||
* supports continue/break, and is verifiable by BPF verifier.
|
||||
*
|
||||
* For iterating integers, the difference betwen bpf_for_each(num, i, N, M)
|
||||
* and bpf_for(i, N, M) is in that bpf_for() provides additional proof to
|
||||
* verifier that i is in [N, M) range, and in bpf_for_each() case i is `int
|
||||
* *`, not just `int`. So for integers bpf_for() is more convenient.
|
||||
*
|
||||
* Note: this macro relies on C99 feature of allowing to declare variables
|
||||
* inside for() loop, bound to for() loop lifetime. It also utilizes GCC
|
||||
* extension: __attribute__((cleanup(<func>))), supported by both GCC and
|
||||
* Clang.
|
||||
*/
|
||||
#define bpf_for_each(type, cur, args...) for ( \
|
||||
/* initialize and define destructor */ \
|
||||
struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \
|
||||
cleanup(bpf_iter_##type##_destroy))), \
|
||||
/* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \
|
||||
*___p __attribute__((unused)) = ( \
|
||||
bpf_iter_##type##_new(&___it, ##args), \
|
||||
/* this is a workaround for Clang bug: it currently doesn't emit BTF */ \
|
||||
/* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \
|
||||
(void)bpf_iter_##type##_destroy, (void *)0); \
|
||||
/* iteration and termination check */ \
|
||||
(((cur) = bpf_iter_##type##_next(&___it))); \
|
||||
)
|
||||
#endif /* bpf_for_each */
|
||||
|
||||
#ifndef bpf_for
|
||||
/* bpf_for(i, start, end) implements a for()-like looping construct that sets
|
||||
* provided integer variable *i* to values starting from *start* through,
|
||||
* but not including, *end*. It also proves to BPF verifier that *i* belongs
|
||||
* to range [start, end), so this can be used for accessing arrays without
|
||||
* extra checks.
|
||||
*
|
||||
* Note: *start* and *end* are assumed to be expressions with no side effects
|
||||
* and whose values do not change throughout bpf_for() loop execution. They do
|
||||
* not have to be statically known or constant, though.
|
||||
*
|
||||
* Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()
|
||||
* loop bound variables and cleanup attribute, supported by GCC and Clang.
|
||||
*/
|
||||
#define bpf_for(i, start, end) for ( \
|
||||
/* initialize and define destructor */ \
|
||||
struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \
|
||||
cleanup(bpf_iter_num_destroy))), \
|
||||
/* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \
|
||||
*___p __attribute__((unused)) = ( \
|
||||
bpf_iter_num_new(&___it, (start), (end)), \
|
||||
/* this is a workaround for Clang bug: it currently doesn't emit BTF */ \
|
||||
/* for bpf_iter_num_destroy() when used from cleanup() attribute */ \
|
||||
(void)bpf_iter_num_destroy, (void *)0); \
|
||||
({ \
|
||||
/* iteration step */ \
|
||||
int *___t = bpf_iter_num_next(&___it); \
|
||||
/* termination and bounds check */ \
|
||||
(___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \
|
||||
}); \
|
||||
)
|
||||
#endif /* bpf_for */
|
||||
|
||||
#ifndef bpf_repeat
|
||||
/* bpf_repeat(N) performs N iterations without exposing iteration number
|
||||
*
|
||||
* Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()
|
||||
* loop bound variables and cleanup attribute, supported by GCC and Clang.
|
||||
*/
|
||||
#define bpf_repeat(N) for ( \
|
||||
/* initialize and define destructor */ \
|
||||
struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \
|
||||
cleanup(bpf_iter_num_destroy))), \
|
||||
/* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \
|
||||
*___p __attribute__((unused)) = ( \
|
||||
bpf_iter_num_new(&___it, 0, (N)), \
|
||||
/* this is a workaround for Clang bug: it currently doesn't emit BTF */ \
|
||||
/* for bpf_iter_num_destroy() when used from cleanup() attribute */ \
|
||||
(void)bpf_iter_num_destroy, (void *)0); \
|
||||
bpf_iter_num_next(&___it); \
|
||||
/* nothing here */ \
|
||||
)
|
||||
#endif /* bpf_repeat */
|
||||
|
||||
#endif
|
||||
@ -0,0 +1,922 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_TRACING_H__
|
||||
#define __BPF_TRACING_H__
|
||||
|
||||
#include "bpf_helpers.h"
|
||||
|
||||
/* Scan the ARCH passed in from ARCH env variable (see Makefile) */
|
||||
#if defined(__TARGET_ARCH_x86)
|
||||
#define bpf_target_x86
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_s390)
|
||||
#define bpf_target_s390
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_arm)
|
||||
#define bpf_target_arm
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_arm64)
|
||||
#define bpf_target_arm64
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_mips)
|
||||
#define bpf_target_mips
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_powerpc)
|
||||
#define bpf_target_powerpc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_sparc)
|
||||
#define bpf_target_sparc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_riscv)
|
||||
#define bpf_target_riscv
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_arc)
|
||||
#define bpf_target_arc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_loongarch)
|
||||
#define bpf_target_loongarch
|
||||
#define bpf_target_defined
|
||||
#else
|
||||
|
||||
/* Fall back to what the compiler says */
|
||||
#if defined(__x86_64__)
|
||||
#define bpf_target_x86
|
||||
#define bpf_target_defined
|
||||
#elif defined(__s390__)
|
||||
#define bpf_target_s390
|
||||
#define bpf_target_defined
|
||||
#elif defined(__arm__)
|
||||
#define bpf_target_arm
|
||||
#define bpf_target_defined
|
||||
#elif defined(__aarch64__)
|
||||
#define bpf_target_arm64
|
||||
#define bpf_target_defined
|
||||
#elif defined(__mips__)
|
||||
#define bpf_target_mips
|
||||
#define bpf_target_defined
|
||||
#elif defined(__powerpc__)
|
||||
#define bpf_target_powerpc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__sparc__)
|
||||
#define bpf_target_sparc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__riscv) && __riscv_xlen == 64
|
||||
#define bpf_target_riscv
|
||||
#define bpf_target_defined
|
||||
#elif defined(__arc__)
|
||||
#define bpf_target_arc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__loongarch__)
|
||||
#define bpf_target_loongarch
|
||||
#define bpf_target_defined
|
||||
#endif /* no compiler target */
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef __BPF_TARGET_MISSING
|
||||
#define __BPF_TARGET_MISSING "GCC error \"Must specify a BPF target arch via __TARGET_ARCH_xxx\""
|
||||
#endif
|
||||
|
||||
#if defined(bpf_target_x86)
|
||||
|
||||
/*
|
||||
* https://en.wikipedia.org/wiki/X86_calling_conventions#System_V_AMD64_ABI
|
||||
*/
|
||||
|
||||
#if defined(__KERNEL__) || defined(__VMLINUX_H__)
|
||||
|
||||
#define __PT_PARM1_REG di
|
||||
#define __PT_PARM2_REG si
|
||||
#define __PT_PARM3_REG dx
|
||||
#define __PT_PARM4_REG cx
|
||||
#define __PT_PARM5_REG r8
|
||||
#define __PT_PARM6_REG r9
|
||||
/*
|
||||
* Syscall uses r10 for PARM4. See arch/x86/entry/entry_64.S:entry_SYSCALL_64
|
||||
* comments in Linux sources. And refer to syscall(2) manpage.
|
||||
*/
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG r10
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG sp
|
||||
#define __PT_FP_REG bp
|
||||
#define __PT_RC_REG ax
|
||||
#define __PT_SP_REG sp
|
||||
#define __PT_IP_REG ip
|
||||
|
||||
#else
|
||||
|
||||
#ifdef __i386__
|
||||
|
||||
/* i386 kernel is built with -mregparm=3 */
|
||||
#define __PT_PARM1_REG eax
|
||||
#define __PT_PARM2_REG edx
|
||||
#define __PT_PARM3_REG ecx
|
||||
/* i386 syscall ABI is very different, refer to syscall(2) manpage */
|
||||
#define __PT_PARM1_SYSCALL_REG ebx
|
||||
#define __PT_PARM2_SYSCALL_REG ecx
|
||||
#define __PT_PARM3_SYSCALL_REG edx
|
||||
#define __PT_PARM4_SYSCALL_REG esi
|
||||
#define __PT_PARM5_SYSCALL_REG edi
|
||||
#define __PT_PARM6_SYSCALL_REG ebp
|
||||
|
||||
#define __PT_RET_REG esp
|
||||
#define __PT_FP_REG ebp
|
||||
#define __PT_RC_REG eax
|
||||
#define __PT_SP_REG esp
|
||||
#define __PT_IP_REG eip
|
||||
|
||||
#else /* __i386__ */
|
||||
|
||||
#define __PT_PARM1_REG rdi
|
||||
#define __PT_PARM2_REG rsi
|
||||
#define __PT_PARM3_REG rdx
|
||||
#define __PT_PARM4_REG rcx
|
||||
#define __PT_PARM5_REG r8
|
||||
#define __PT_PARM6_REG r9
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG r10
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG rsp
|
||||
#define __PT_FP_REG rbp
|
||||
#define __PT_RC_REG rax
|
||||
#define __PT_SP_REG rsp
|
||||
#define __PT_IP_REG rip
|
||||
|
||||
#endif /* __i386__ */
|
||||
|
||||
#endif /* __KERNEL__ || __VMLINUX_H__ */
|
||||
|
||||
#elif defined(bpf_target_s390)
|
||||
|
||||
/*
|
||||
* https://github.com/IBM/s390x-abi/releases/download/v1.6/lzsabi_s390x.pdf
|
||||
*/
|
||||
|
||||
struct pt_regs___s390 {
|
||||
unsigned long orig_gpr2;
|
||||
};
|
||||
|
||||
/* s390 provides user_pt_regs instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const user_pt_regs *)(x))
|
||||
#define __PT_PARM1_REG gprs[2]
|
||||
#define __PT_PARM2_REG gprs[3]
|
||||
#define __PT_PARM3_REG gprs[4]
|
||||
#define __PT_PARM4_REG gprs[5]
|
||||
#define __PT_PARM5_REG gprs[6]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG orig_gpr2
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG gprs[7]
|
||||
#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x)
|
||||
#define PT_REGS_PARM1_CORE_SYSCALL(x) \
|
||||
BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG)
|
||||
|
||||
#define __PT_RET_REG gprs[14]
|
||||
#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define __PT_RC_REG gprs[2]
|
||||
#define __PT_SP_REG gprs[15]
|
||||
#define __PT_IP_REG psw.addr
|
||||
|
||||
#elif defined(bpf_target_arm)
|
||||
|
||||
/*
|
||||
* https://github.com/ARM-software/abi-aa/blob/main/aapcs32/aapcs32.rst#machine-registers
|
||||
*/
|
||||
|
||||
#define __PT_PARM1_REG uregs[0]
|
||||
#define __PT_PARM2_REG uregs[1]
|
||||
#define __PT_PARM3_REG uregs[2]
|
||||
#define __PT_PARM4_REG uregs[3]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG uregs[4]
|
||||
#define __PT_PARM6_SYSCALL_REG uregs[5]
|
||||
#define __PT_PARM7_SYSCALL_REG uregs[6]
|
||||
|
||||
#define __PT_RET_REG uregs[14]
|
||||
#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define __PT_RC_REG uregs[0]
|
||||
#define __PT_SP_REG uregs[13]
|
||||
#define __PT_IP_REG uregs[12]
|
||||
|
||||
#elif defined(bpf_target_arm64)
|
||||
|
||||
/*
|
||||
* https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#machine-registers
|
||||
*/
|
||||
|
||||
struct pt_regs___arm64 {
|
||||
unsigned long orig_x0;
|
||||
};
|
||||
|
||||
/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x))
|
||||
#define __PT_PARM1_REG regs[0]
|
||||
#define __PT_PARM2_REG regs[1]
|
||||
#define __PT_PARM3_REG regs[2]
|
||||
#define __PT_PARM4_REG regs[3]
|
||||
#define __PT_PARM5_REG regs[4]
|
||||
#define __PT_PARM6_REG regs[5]
|
||||
#define __PT_PARM7_REG regs[6]
|
||||
#define __PT_PARM8_REG regs[7]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG orig_x0
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x)
|
||||
#define PT_REGS_PARM1_CORE_SYSCALL(x) \
|
||||
BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG)
|
||||
|
||||
#define __PT_RET_REG regs[30]
|
||||
#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define __PT_RC_REG regs[0]
|
||||
#define __PT_SP_REG sp
|
||||
#define __PT_IP_REG pc
|
||||
|
||||
#elif defined(bpf_target_mips)
|
||||
|
||||
/*
|
||||
* N64 ABI is assumed right now.
|
||||
* https://en.wikipedia.org/wiki/MIPS_architecture#Calling_conventions
|
||||
*/
|
||||
|
||||
#define __PT_PARM1_REG regs[4]
|
||||
#define __PT_PARM2_REG regs[5]
|
||||
#define __PT_PARM3_REG regs[6]
|
||||
#define __PT_PARM4_REG regs[7]
|
||||
#define __PT_PARM5_REG regs[8]
|
||||
#define __PT_PARM6_REG regs[9]
|
||||
#define __PT_PARM7_REG regs[10]
|
||||
#define __PT_PARM8_REG regs[11]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG /* only N32/N64 */
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG /* only N32/N64 */
|
||||
|
||||
#define __PT_RET_REG regs[31]
|
||||
#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define __PT_RC_REG regs[2]
|
||||
#define __PT_SP_REG regs[29]
|
||||
#define __PT_IP_REG cp0_epc
|
||||
|
||||
#elif defined(bpf_target_powerpc)
|
||||
|
||||
/*
|
||||
* http://refspecs.linux-foundation.org/elf/elfspec_ppc.pdf (page 3-14,
|
||||
* section "Function Calling Sequence")
|
||||
*/
|
||||
|
||||
#define __PT_PARM1_REG gpr[3]
|
||||
#define __PT_PARM2_REG gpr[4]
|
||||
#define __PT_PARM3_REG gpr[5]
|
||||
#define __PT_PARM4_REG gpr[6]
|
||||
#define __PT_PARM5_REG gpr[7]
|
||||
#define __PT_PARM6_REG gpr[8]
|
||||
#define __PT_PARM7_REG gpr[9]
|
||||
#define __PT_PARM8_REG gpr[10]
|
||||
|
||||
/* powerpc does not select ARCH_HAS_SYSCALL_WRAPPER. */
|
||||
#define PT_REGS_SYSCALL_REGS(ctx) ctx
|
||||
#define __PT_PARM1_SYSCALL_REG orig_gpr3
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
#if !defined(__arch64__)
|
||||
#define __PT_PARM7_SYSCALL_REG __PT_PARM7_REG /* only powerpc (not powerpc64) */
|
||||
#endif
|
||||
|
||||
#define __PT_RET_REG regs[31]
|
||||
#define __PT_FP_REG __unsupported__
|
||||
#define __PT_RC_REG gpr[3]
|
||||
#define __PT_SP_REG sp
|
||||
#define __PT_IP_REG nip
|
||||
|
||||
#elif defined(bpf_target_sparc)
|
||||
|
||||
/*
|
||||
* https://en.wikipedia.org/wiki/Calling_convention#SPARC
|
||||
*/
|
||||
|
||||
#define __PT_PARM1_REG u_regs[UREG_I0]
|
||||
#define __PT_PARM2_REG u_regs[UREG_I1]
|
||||
#define __PT_PARM3_REG u_regs[UREG_I2]
|
||||
#define __PT_PARM4_REG u_regs[UREG_I3]
|
||||
#define __PT_PARM5_REG u_regs[UREG_I4]
|
||||
#define __PT_PARM6_REG u_regs[UREG_I5]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG u_regs[UREG_I7]
|
||||
#define __PT_FP_REG __unsupported__
|
||||
#define __PT_RC_REG u_regs[UREG_I0]
|
||||
#define __PT_SP_REG u_regs[UREG_FP]
|
||||
/* Should this also be a bpf_target check for the sparc case? */
|
||||
#if defined(__arch64__)
|
||||
#define __PT_IP_REG tpc
|
||||
#else
|
||||
#define __PT_IP_REG pc
|
||||
#endif
|
||||
|
||||
#elif defined(bpf_target_riscv)
|
||||
|
||||
/*
|
||||
* https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#risc-v-calling-conventions
|
||||
*/
|
||||
|
||||
/* riscv provides struct user_regs_struct instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))
|
||||
#define __PT_PARM1_REG a0
|
||||
#define __PT_PARM2_REG a1
|
||||
#define __PT_PARM3_REG a2
|
||||
#define __PT_PARM4_REG a3
|
||||
#define __PT_PARM5_REG a4
|
||||
#define __PT_PARM6_REG a5
|
||||
#define __PT_PARM7_REG a6
|
||||
#define __PT_PARM8_REG a7
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG ra
|
||||
#define __PT_FP_REG s0
|
||||
#define __PT_RC_REG a0
|
||||
#define __PT_SP_REG sp
|
||||
#define __PT_IP_REG pc
|
||||
|
||||
#elif defined(bpf_target_arc)
|
||||
|
||||
/*
|
||||
* Section "Function Calling Sequence" (page 24):
|
||||
* https://raw.githubusercontent.com/wiki/foss-for-synopsys-dwc-arc-processors/toolchain/files/ARCv2_ABI.pdf
|
||||
*/
|
||||
|
||||
/* arc provides struct user_regs_struct instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))
|
||||
#define __PT_PARM1_REG scratch.r0
|
||||
#define __PT_PARM2_REG scratch.r1
|
||||
#define __PT_PARM3_REG scratch.r2
|
||||
#define __PT_PARM4_REG scratch.r3
|
||||
#define __PT_PARM5_REG scratch.r4
|
||||
#define __PT_PARM6_REG scratch.r5
|
||||
#define __PT_PARM7_REG scratch.r6
|
||||
#define __PT_PARM8_REG scratch.r7
|
||||
|
||||
/* arc does not select ARCH_HAS_SYSCALL_WRAPPER. */
|
||||
#define PT_REGS_SYSCALL_REGS(ctx) ctx
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG scratch.blink
|
||||
#define __PT_FP_REG scratch.fp
|
||||
#define __PT_RC_REG scratch.r0
|
||||
#define __PT_SP_REG scratch.sp
|
||||
#define __PT_IP_REG scratch.ret
|
||||
|
||||
#elif defined(bpf_target_loongarch)
|
||||
|
||||
/*
|
||||
* https://docs.kernel.org/loongarch/introduction.html
|
||||
* https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html
|
||||
*/
|
||||
|
||||
/* loongarch provides struct user_pt_regs instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x))
|
||||
#define __PT_PARM1_REG regs[4]
|
||||
#define __PT_PARM2_REG regs[5]
|
||||
#define __PT_PARM3_REG regs[6]
|
||||
#define __PT_PARM4_REG regs[7]
|
||||
#define __PT_PARM5_REG regs[8]
|
||||
#define __PT_PARM6_REG regs[9]
|
||||
#define __PT_PARM7_REG regs[10]
|
||||
#define __PT_PARM8_REG regs[11]
|
||||
|
||||
/* loongarch does not select ARCH_HAS_SYSCALL_WRAPPER. */
|
||||
#define PT_REGS_SYSCALL_REGS(ctx) ctx
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG regs[1]
|
||||
#define __PT_FP_REG regs[22]
|
||||
#define __PT_RC_REG regs[4]
|
||||
#define __PT_SP_REG regs[3]
|
||||
#define __PT_IP_REG csr_era
|
||||
|
||||
#endif
|
||||
|
||||
#if defined(bpf_target_defined)
|
||||
|
||||
struct pt_regs;
|
||||
|
||||
/* allow some architectures to override `struct pt_regs` */
|
||||
#ifndef __PT_REGS_CAST
|
||||
#define __PT_REGS_CAST(x) (x)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Different architectures support different number of arguments passed
|
||||
* through registers. i386 supports just 3, some arches support up to 8.
|
||||
*/
|
||||
#ifndef __PT_PARM4_REG
|
||||
#define __PT_PARM4_REG __unsupported__
|
||||
#endif
|
||||
#ifndef __PT_PARM5_REG
|
||||
#define __PT_PARM5_REG __unsupported__
|
||||
#endif
|
||||
#ifndef __PT_PARM6_REG
|
||||
#define __PT_PARM6_REG __unsupported__
|
||||
#endif
|
||||
#ifndef __PT_PARM7_REG
|
||||
#define __PT_PARM7_REG __unsupported__
|
||||
#endif
|
||||
#ifndef __PT_PARM8_REG
|
||||
#define __PT_PARM8_REG __unsupported__
|
||||
#endif
|
||||
/*
|
||||
* Similarly, syscall-specific conventions might differ between function call
|
||||
* conventions within each architecutre. All supported architectures pass
|
||||
* either 6 or 7 syscall arguments in registers.
|
||||
*
|
||||
* See syscall(2) manpage for succinct table with information on each arch.
|
||||
*/
|
||||
#ifndef __PT_PARM7_SYSCALL_REG
|
||||
#define __PT_PARM7_SYSCALL_REG __unsupported__
|
||||
#endif
|
||||
|
||||
#define PT_REGS_PARM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG)
|
||||
#define PT_REGS_PARM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG)
|
||||
#define PT_REGS_PARM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG)
|
||||
#define PT_REGS_PARM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG)
|
||||
#define PT_REGS_PARM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG)
|
||||
#define PT_REGS_PARM6(x) (__PT_REGS_CAST(x)->__PT_PARM6_REG)
|
||||
#define PT_REGS_PARM7(x) (__PT_REGS_CAST(x)->__PT_PARM7_REG)
|
||||
#define PT_REGS_PARM8(x) (__PT_REGS_CAST(x)->__PT_PARM8_REG)
|
||||
#define PT_REGS_RET(x) (__PT_REGS_CAST(x)->__PT_RET_REG)
|
||||
#define PT_REGS_FP(x) (__PT_REGS_CAST(x)->__PT_FP_REG)
|
||||
#define PT_REGS_RC(x) (__PT_REGS_CAST(x)->__PT_RC_REG)
|
||||
#define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG)
|
||||
#define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG)
|
||||
|
||||
#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG)
|
||||
#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG)
|
||||
#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG)
|
||||
#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG)
|
||||
#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG)
|
||||
#define PT_REGS_PARM6_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_REG)
|
||||
#define PT_REGS_PARM7_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_REG)
|
||||
#define PT_REGS_PARM8_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM8_REG)
|
||||
#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG)
|
||||
#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG)
|
||||
#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG)
|
||||
#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG)
|
||||
#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG)
|
||||
|
||||
#if defined(bpf_target_powerpc)
|
||||
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; })
|
||||
#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
|
||||
|
||||
#elif defined(bpf_target_sparc)
|
||||
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); })
|
||||
#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
|
||||
|
||||
#else
|
||||
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) \
|
||||
({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); })
|
||||
#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \
|
||||
({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); })
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef PT_REGS_PARM1_SYSCALL
|
||||
#define PT_REGS_PARM1_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM1_SYSCALL_REG)
|
||||
#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM2_SYSCALL
|
||||
#define PT_REGS_PARM2_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM2_SYSCALL_REG)
|
||||
#define PT_REGS_PARM2_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM3_SYSCALL
|
||||
#define PT_REGS_PARM3_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM3_SYSCALL_REG)
|
||||
#define PT_REGS_PARM3_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM4_SYSCALL
|
||||
#define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_SYSCALL_REG)
|
||||
#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM5_SYSCALL
|
||||
#define PT_REGS_PARM5_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM5_SYSCALL_REG)
|
||||
#define PT_REGS_PARM5_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM6_SYSCALL
|
||||
#define PT_REGS_PARM6_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM6_SYSCALL_REG)
|
||||
#define PT_REGS_PARM6_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM7_SYSCALL
|
||||
#define PT_REGS_PARM7_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM7_SYSCALL_REG)
|
||||
#define PT_REGS_PARM7_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_SYSCALL_REG)
|
||||
#endif
|
||||
|
||||
#else /* defined(bpf_target_defined) */
|
||||
|
||||
#define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM3(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM4(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM5(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM6(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM7(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM8(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_RET(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_FP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_RC(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_SP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_IP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#define PT_REGS_PARM1_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM2_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM3_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM4_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM5_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM6_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM7_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM8_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_RET_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_FP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_RC_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_SP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_IP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM2_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM3_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM4_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM5_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM6_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM7_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#define PT_REGS_PARM1_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM2_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM3_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM4_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM5_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM6_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM7_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#endif /* defined(bpf_target_defined) */
|
||||
|
||||
/*
|
||||
* When invoked from a syscall handler kprobe, returns a pointer to a
|
||||
* struct pt_regs containing syscall arguments and suitable for passing to
|
||||
* PT_REGS_PARMn_SYSCALL() and PT_REGS_PARMn_CORE_SYSCALL().
|
||||
*/
|
||||
#ifndef PT_REGS_SYSCALL_REGS
|
||||
/* By default, assume that the arch selects ARCH_HAS_SYSCALL_WRAPPER. */
|
||||
#define PT_REGS_SYSCALL_REGS(ctx) ((struct pt_regs *)PT_REGS_PARM1(ctx))
|
||||
#endif
|
||||
|
||||
#ifndef ___bpf_concat
|
||||
#define ___bpf_concat(a, b) a ## b
|
||||
#endif
|
||||
#ifndef ___bpf_apply
|
||||
#define ___bpf_apply(fn, n) ___bpf_concat(fn, n)
|
||||
#endif
|
||||
#ifndef ___bpf_nth
|
||||
#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N
|
||||
#endif
|
||||
#ifndef ___bpf_narg
|
||||
#define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
|
||||
#endif
|
||||
|
||||
#define ___bpf_ctx_cast0() ctx
|
||||
#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), (void *)ctx[0]
|
||||
#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), (void *)ctx[1]
|
||||
#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), (void *)ctx[2]
|
||||
#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), (void *)ctx[3]
|
||||
#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), (void *)ctx[4]
|
||||
#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), (void *)ctx[5]
|
||||
#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), (void *)ctx[6]
|
||||
#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), (void *)ctx[7]
|
||||
#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), (void *)ctx[8]
|
||||
#define ___bpf_ctx_cast10(x, args...) ___bpf_ctx_cast9(args), (void *)ctx[9]
|
||||
#define ___bpf_ctx_cast11(x, args...) ___bpf_ctx_cast10(args), (void *)ctx[10]
|
||||
#define ___bpf_ctx_cast12(x, args...) ___bpf_ctx_cast11(args), (void *)ctx[11]
|
||||
#define ___bpf_ctx_cast(args...) ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args)
|
||||
|
||||
/*
|
||||
* BPF_PROG is a convenience wrapper for generic tp_btf/fentry/fexit and
|
||||
* similar kinds of BPF programs, that accept input arguments as a single
|
||||
* pointer to untyped u64 array, where each u64 can actually be a typed
|
||||
* pointer or integer of different size. Instead of requring user to write
|
||||
* manual casts and work with array elements by index, BPF_PROG macro
|
||||
* allows user to declare a list of named and typed input arguments in the
|
||||
* same syntax as for normal C function. All the casting is hidden and
|
||||
* performed transparently, while user code can just assume working with
|
||||
* function arguments of specified type and name.
|
||||
*
|
||||
* Original raw context argument is preserved as well as 'ctx' argument.
|
||||
* This is useful when using BPF helpers that expect original context
|
||||
* as one of the parameters (e.g., for bpf_perf_event_output()).
|
||||
*/
|
||||
#define BPF_PROG(name, args...) \
|
||||
name(unsigned long long *ctx); \
|
||||
static __always_inline typeof(name(0)) \
|
||||
____##name(unsigned long long *ctx, ##args); \
|
||||
typeof(name(0)) name(unsigned long long *ctx) \
|
||||
{ \
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
return ____##name(___bpf_ctx_cast(args)); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
} \
|
||||
static __always_inline typeof(name(0)) \
|
||||
____##name(unsigned long long *ctx, ##args)
|
||||
|
||||
#ifndef ___bpf_nth2
|
||||
#define ___bpf_nth2(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, \
|
||||
_14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, N, ...) N
|
||||
#endif
|
||||
#ifndef ___bpf_narg2
|
||||
#define ___bpf_narg2(...) \
|
||||
___bpf_nth2(_, ##__VA_ARGS__, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, \
|
||||
6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0)
|
||||
#endif
|
||||
|
||||
#define ___bpf_treg_cnt(t) \
|
||||
__builtin_choose_expr(sizeof(t) == 1, 1, \
|
||||
__builtin_choose_expr(sizeof(t) == 2, 1, \
|
||||
__builtin_choose_expr(sizeof(t) == 4, 1, \
|
||||
__builtin_choose_expr(sizeof(t) == 8, 1, \
|
||||
__builtin_choose_expr(sizeof(t) == 16, 2, \
|
||||
(void)0)))))
|
||||
|
||||
#define ___bpf_reg_cnt0() (0)
|
||||
#define ___bpf_reg_cnt1(t, x) (___bpf_reg_cnt0() + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt2(t, x, args...) (___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt3(t, x, args...) (___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt4(t, x, args...) (___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt5(t, x, args...) (___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt6(t, x, args...) (___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt7(t, x, args...) (___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt8(t, x, args...) (___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt9(t, x, args...) (___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt10(t, x, args...) (___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt11(t, x, args...) (___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt12(t, x, args...) (___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t))
|
||||
#define ___bpf_reg_cnt(args...) ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args)
|
||||
|
||||
#define ___bpf_union_arg(t, x, n) \
|
||||
__builtin_choose_expr(sizeof(t) == 1, ({ union { __u8 z[1]; t x; } ___t = { .z = {ctx[n]}}; ___t.x; }), \
|
||||
__builtin_choose_expr(sizeof(t) == 2, ({ union { __u16 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \
|
||||
__builtin_choose_expr(sizeof(t) == 4, ({ union { __u32 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \
|
||||
__builtin_choose_expr(sizeof(t) == 8, ({ union { __u64 z[1]; t x; } ___t = {.z = {ctx[n]} }; ___t.x; }), \
|
||||
__builtin_choose_expr(sizeof(t) == 16, ({ union { __u64 z[2]; t x; } ___t = {.z = {ctx[n], ctx[n + 1]} }; ___t.x; }), \
|
||||
(void)0)))))
|
||||
|
||||
#define ___bpf_ctx_arg0(n, args...)
|
||||
#define ___bpf_ctx_arg1(n, t, x) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x))
|
||||
#define ___bpf_ctx_arg2(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args)
|
||||
#define ___bpf_ctx_arg3(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args)
|
||||
#define ___bpf_ctx_arg4(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args)
|
||||
#define ___bpf_ctx_arg5(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args)
|
||||
#define ___bpf_ctx_arg6(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args)
|
||||
#define ___bpf_ctx_arg7(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args)
|
||||
#define ___bpf_ctx_arg8(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args)
|
||||
#define ___bpf_ctx_arg9(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args)
|
||||
#define ___bpf_ctx_arg10(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args)
|
||||
#define ___bpf_ctx_arg11(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args)
|
||||
#define ___bpf_ctx_arg12(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args)
|
||||
#define ___bpf_ctx_arg(args...) ___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args)
|
||||
|
||||
#define ___bpf_ctx_decl0()
|
||||
#define ___bpf_ctx_decl1(t, x) , t x
|
||||
#define ___bpf_ctx_decl2(t, x, args...) , t x ___bpf_ctx_decl1(args)
|
||||
#define ___bpf_ctx_decl3(t, x, args...) , t x ___bpf_ctx_decl2(args)
|
||||
#define ___bpf_ctx_decl4(t, x, args...) , t x ___bpf_ctx_decl3(args)
|
||||
#define ___bpf_ctx_decl5(t, x, args...) , t x ___bpf_ctx_decl4(args)
|
||||
#define ___bpf_ctx_decl6(t, x, args...) , t x ___bpf_ctx_decl5(args)
|
||||
#define ___bpf_ctx_decl7(t, x, args...) , t x ___bpf_ctx_decl6(args)
|
||||
#define ___bpf_ctx_decl8(t, x, args...) , t x ___bpf_ctx_decl7(args)
|
||||
#define ___bpf_ctx_decl9(t, x, args...) , t x ___bpf_ctx_decl8(args)
|
||||
#define ___bpf_ctx_decl10(t, x, args...) , t x ___bpf_ctx_decl9(args)
|
||||
#define ___bpf_ctx_decl11(t, x, args...) , t x ___bpf_ctx_decl10(args)
|
||||
#define ___bpf_ctx_decl12(t, x, args...) , t x ___bpf_ctx_decl11(args)
|
||||
#define ___bpf_ctx_decl(args...) ___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args)
|
||||
|
||||
/*
|
||||
* BPF_PROG2 is an enhanced version of BPF_PROG in order to handle struct
|
||||
* arguments. Since each struct argument might take one or two u64 values
|
||||
* in the trampoline stack, argument type size is needed to place proper number
|
||||
* of u64 values for each argument. Therefore, BPF_PROG2 has different
|
||||
* syntax from BPF_PROG. For example, for the following BPF_PROG syntax:
|
||||
*
|
||||
* int BPF_PROG(test2, int a, int b) { ... }
|
||||
*
|
||||
* the corresponding BPF_PROG2 syntax is:
|
||||
*
|
||||
* int BPF_PROG2(test2, int, a, int, b) { ... }
|
||||
*
|
||||
* where type and the corresponding argument name are separated by comma.
|
||||
*
|
||||
* Use BPF_PROG2 macro if one of the arguments might be a struct/union larger
|
||||
* than 8 bytes:
|
||||
*
|
||||
* int BPF_PROG2(test_struct_arg, struct bpf_testmod_struct_arg_1, a, int, b,
|
||||
* int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret)
|
||||
* {
|
||||
* // access a, b, c, d, e, and ret directly
|
||||
* ...
|
||||
* }
|
||||
*/
|
||||
#define BPF_PROG2(name, args...) \
|
||||
name(unsigned long long *ctx); \
|
||||
static __always_inline typeof(name(0)) \
|
||||
____##name(unsigned long long *ctx ___bpf_ctx_decl(args)); \
|
||||
typeof(name(0)) name(unsigned long long *ctx) \
|
||||
{ \
|
||||
return ____##name(ctx ___bpf_ctx_arg(args)); \
|
||||
} \
|
||||
static __always_inline typeof(name(0)) \
|
||||
____##name(unsigned long long *ctx ___bpf_ctx_decl(args))
|
||||
|
||||
struct pt_regs;
|
||||
|
||||
#define ___bpf_kprobe_args0() ctx
|
||||
#define ___bpf_kprobe_args1(x) ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx)
|
||||
#define ___bpf_kprobe_args2(x, args...) ___bpf_kprobe_args1(args), (void *)PT_REGS_PARM2(ctx)
|
||||
#define ___bpf_kprobe_args3(x, args...) ___bpf_kprobe_args2(args), (void *)PT_REGS_PARM3(ctx)
|
||||
#define ___bpf_kprobe_args4(x, args...) ___bpf_kprobe_args3(args), (void *)PT_REGS_PARM4(ctx)
|
||||
#define ___bpf_kprobe_args5(x, args...) ___bpf_kprobe_args4(args), (void *)PT_REGS_PARM5(ctx)
|
||||
#define ___bpf_kprobe_args6(x, args...) ___bpf_kprobe_args5(args), (void *)PT_REGS_PARM6(ctx)
|
||||
#define ___bpf_kprobe_args7(x, args...) ___bpf_kprobe_args6(args), (void *)PT_REGS_PARM7(ctx)
|
||||
#define ___bpf_kprobe_args8(x, args...) ___bpf_kprobe_args7(args), (void *)PT_REGS_PARM8(ctx)
|
||||
#define ___bpf_kprobe_args(args...) ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args)
|
||||
|
||||
/*
|
||||
* BPF_KPROBE serves the same purpose for kprobes as BPF_PROG for
|
||||
* tp_btf/fentry/fexit BPF programs. It hides the underlying platform-specific
|
||||
* low-level way of getting kprobe input arguments from struct pt_regs, and
|
||||
* provides a familiar typed and named function arguments syntax and
|
||||
* semantics of accessing kprobe input paremeters.
|
||||
*
|
||||
* Original struct pt_regs* context is preserved as 'ctx' argument. This might
|
||||
* be necessary when using BPF helpers like bpf_perf_event_output().
|
||||
*/
|
||||
#define BPF_KPROBE(name, args...) \
|
||||
name(struct pt_regs *ctx); \
|
||||
static __always_inline typeof(name(0)) \
|
||||
____##name(struct pt_regs *ctx, ##args); \
|
||||
typeof(name(0)) name(struct pt_regs *ctx) \
|
||||
{ \
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
return ____##name(___bpf_kprobe_args(args)); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
} \
|
||||
static __always_inline typeof(name(0)) \
|
||||
____##name(struct pt_regs *ctx, ##args)
|
||||
|
||||
#define ___bpf_kretprobe_args0() ctx
|
||||
#define ___bpf_kretprobe_args1(x) ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx)
|
||||
#define ___bpf_kretprobe_args(args...) ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args)
|
||||
|
||||
/*
|
||||
* BPF_KRETPROBE is similar to BPF_KPROBE, except, it only provides optional
|
||||
* return value (in addition to `struct pt_regs *ctx`), but no input
|
||||
* arguments, because they will be clobbered by the time probed function
|
||||
* returns.
|
||||
*/
|
||||
#define BPF_KRETPROBE(name, args...) \
|
||||
name(struct pt_regs *ctx); \
|
||||
static __always_inline typeof(name(0)) \
|
||||
____##name(struct pt_regs *ctx, ##args); \
|
||||
typeof(name(0)) name(struct pt_regs *ctx) \
|
||||
{ \
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
return ____##name(___bpf_kretprobe_args(args)); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
} \
|
||||
static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args)
|
||||
|
||||
/* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */
|
||||
#define ___bpf_syscall_args0() ctx
|
||||
#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_SYSCALL(regs)
|
||||
#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_SYSCALL(regs)
|
||||
#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_SYSCALL(regs)
|
||||
#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_SYSCALL(regs)
|
||||
#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_SYSCALL(regs)
|
||||
#define ___bpf_syscall_args6(x, args...) ___bpf_syscall_args5(args), (void *)PT_REGS_PARM6_SYSCALL(regs)
|
||||
#define ___bpf_syscall_args7(x, args...) ___bpf_syscall_args6(args), (void *)PT_REGS_PARM7_SYSCALL(regs)
|
||||
#define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args)
|
||||
|
||||
/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */
|
||||
#define ___bpf_syswrap_args0() ctx
|
||||
#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs)
|
||||
#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs)
|
||||
#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs)
|
||||
#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs)
|
||||
#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs)
|
||||
#define ___bpf_syswrap_args6(x, args...) ___bpf_syswrap_args5(args), (void *)PT_REGS_PARM6_CORE_SYSCALL(regs)
|
||||
#define ___bpf_syswrap_args7(x, args...) ___bpf_syswrap_args6(args), (void *)PT_REGS_PARM7_CORE_SYSCALL(regs)
|
||||
#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args)
|
||||
|
||||
/*
|
||||
* BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for
|
||||
* tracing syscall functions, like __x64_sys_close. It hides the underlying
|
||||
* platform-specific low-level way of getting syscall input arguments from
|
||||
* struct pt_regs, and provides a familiar typed and named function arguments
|
||||
* syntax and semantics of accessing syscall input parameters.
|
||||
*
|
||||
* Original struct pt_regs * context is preserved as 'ctx' argument. This might
|
||||
* be necessary when using BPF helpers like bpf_perf_event_output().
|
||||
*
|
||||
* At the moment BPF_KSYSCALL does not transparently handle all the calling
|
||||
* convention quirks for the following syscalls:
|
||||
*
|
||||
* - mmap(): __ARCH_WANT_SYS_OLD_MMAP.
|
||||
* - clone(): CONFIG_CLONE_BACKWARDS, CONFIG_CLONE_BACKWARDS2 and
|
||||
* CONFIG_CLONE_BACKWARDS3.
|
||||
* - socket-related syscalls: __ARCH_WANT_SYS_SOCKETCALL.
|
||||
* - compat syscalls.
|
||||
*
|
||||
* This may or may not change in the future. User needs to take extra measures
|
||||
* to handle such quirks explicitly, if necessary.
|
||||
*
|
||||
* This macro relies on BPF CO-RE support and virtual __kconfig externs.
|
||||
*/
|
||||
#define BPF_KSYSCALL(name, args...) \
|
||||
name(struct pt_regs *ctx); \
|
||||
extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \
|
||||
static __always_inline typeof(name(0)) \
|
||||
____##name(struct pt_regs *ctx, ##args); \
|
||||
typeof(name(0)) name(struct pt_regs *ctx) \
|
||||
{ \
|
||||
struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER \
|
||||
? (struct pt_regs *)PT_REGS_PARM1(ctx) \
|
||||
: ctx; \
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
if (LINUX_HAS_SYSCALL_WRAPPER) \
|
||||
return ____##name(___bpf_syswrap_args(args)); \
|
||||
else \
|
||||
return ____##name(___bpf_syscall_args(args)); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
} \
|
||||
static __always_inline typeof(name(0)) \
|
||||
____##name(struct pt_regs *ctx, ##args)
|
||||
|
||||
#define BPF_KPROBE_SYSCALL BPF_KSYSCALL
|
||||
|
||||
/* BPF_UPROBE and BPF_URETPROBE are identical to BPF_KPROBE and BPF_KRETPROBE,
|
||||
* but are named way less confusingly for SEC("uprobe") and SEC("uretprobe")
|
||||
* use cases.
|
||||
*/
|
||||
#define BPF_UPROBE(name, args...) BPF_KPROBE(name, ##args)
|
||||
#define BPF_URETPROBE(name, args...) BPF_KRETPROBE(name, ##args)
|
||||
|
||||
#endif
|
||||
@ -0,0 +1,99 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
libbpfVersion = "1.4.3"
|
||||
prefix = "libbpf-" + libbpfVersion
|
||||
)
|
||||
|
||||
var (
|
||||
filesToExtract = map[string]struct{}{
|
||||
prefix + "/LICENSE.BSD-2-Clause": {},
|
||||
prefix + "/src/bpf_endian.h": {},
|
||||
prefix + "/src/bpf_helper_defs.h": {},
|
||||
prefix + "/src/bpf_helpers.h": {},
|
||||
prefix + "/src/bpf_tracing.h": {},
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
flagDest = flag.String("dest", ".", "destination directory")
|
||||
)
|
||||
|
||||
// TODO(jwhited): go generate strategy for derp/xdp
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
f, err := os.CreateTemp("", "libbpf")
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
resp, err := http.Get(fmt.Sprintf("https://github.com/libbpf/libbpf/archive/refs/tags/v%s.tar.gz", libbpfVersion))
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
_, err = io.Copy(f, resp.Body)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
|
||||
_, err = f.Seek(0, 0)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
g, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
defer g.Close()
|
||||
t := tar.NewReader(g)
|
||||
|
||||
seen := make(map[string]bool, len(filesToExtract))
|
||||
for {
|
||||
h, err := t.Next()
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
if strings.Contains(h.Name, "..") {
|
||||
continue
|
||||
}
|
||||
_, ok := filesToExtract[h.Name]
|
||||
if ok {
|
||||
if seen[h.Name] {
|
||||
log.Panicf("saw %s more than once in archive", h.Name)
|
||||
}
|
||||
seen[h.Name] = true
|
||||
p := filepath.Join(*flagDest, filepath.Base(h.Name))
|
||||
e, err := os.Create(p)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
_, err = io.Copy(e, t)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
if len(seen) == len(filesToExtract) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,585 @@
|
||||
//go:build ignore
|
||||
|
||||
#include <linux/bpf.h>
|
||||
#include <linux/if_ether.h>
|
||||
#include <linux/in.h>
|
||||
#include <linux/ip.h>
|
||||
#include <linux/ipv6.h>
|
||||
#include <linux/udp.h>
|
||||
#include <bpf_endian.h>
|
||||
#include <bpf_helpers.h>
|
||||
|
||||
struct config {
|
||||
// TODO(jwhited): if we add more fields consider endianness consistency in
|
||||
// the context of the data. cilium/ebpf uses native endian encoding for map
|
||||
// encoding even if we use big endian types here, e.g. __be16.
|
||||
__u16 dst_port;
|
||||
};
|
||||
struct config *unused_config __attribute__((unused)); // required by bpf2go -type
|
||||
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_ARRAY);
|
||||
__uint(key_size, sizeof(__u32));
|
||||
__uint(value_size, sizeof(struct config));
|
||||
__uint(max_entries, 1);
|
||||
} config_map SEC(".maps");
|
||||
|
||||
struct counters_key {
|
||||
__u8 unused;
|
||||
__u8 af;
|
||||
__u8 pba;
|
||||
__u8 prog_end;
|
||||
};
|
||||
struct counters_key *unused_counters_key __attribute__((unused)); // required by bpf2go -type
|
||||
|
||||
enum counter_key_af {
|
||||
COUNTER_KEY_AF_UNKNOWN,
|
||||
COUNTER_KEY_AF_IPV4,
|
||||
COUNTER_KEY_AF_IPV6,
|
||||
COUNTER_KEY_AF_LEN
|
||||
};
|
||||
enum counter_key_af *unused_counter_key_af __attribute__((unused)); // required by bpf2go -type
|
||||
|
||||
enum counter_key_packets_bytes_action {
|
||||
COUNTER_KEY_PACKETS_PASS_TOTAL,
|
||||
COUNTER_KEY_BYTES_PASS_TOTAL,
|
||||
COUNTER_KEY_PACKETS_ABORTED_TOTAL,
|
||||
COUNTER_KEY_BYTES_ABORTED_TOTAL,
|
||||
COUNTER_KEY_PACKETS_TX_TOTAL,
|
||||
COUNTER_KEY_BYTES_TX_TOTAL,
|
||||
COUNTER_KEY_PACKETS_DROP_TOTAL,
|
||||
COUNTER_KEY_BYTES_DROP_TOTAL,
|
||||
COUNTER_KEY_PACKETS_BYTES_ACTION_LEN
|
||||
};
|
||||
enum counter_key_packets_bytes_action *unused_counter_key_packets_bytes_action __attribute__((unused)); // required by bpf2go -type
|
||||
|
||||
enum counter_key_prog_end {
|
||||
COUNTER_KEY_END_UNSPECIFIED,
|
||||
COUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR,
|
||||
COUNTER_KEY_END_INVALID_UDP_CSUM,
|
||||
COUNTER_KEY_END_INVALID_IP_CSUM,
|
||||
COUNTER_KEY_END_NOT_STUN_PORT,
|
||||
COUNTER_KEY_END_INVALID_SW_ATTR_VAL,
|
||||
COUNTER_KEY_END_LEN
|
||||
};
|
||||
enum counter_key_prog_end *unused_counter_key_prog_end __attribute__((unused)); // required by bpf2go -type
|
||||
|
||||
#define COUNTERS_MAP_MAX_ENTRIES ((COUNTER_KEY_AF_LEN - 1) << 16) | \
|
||||
((COUNTER_KEY_PACKETS_BYTES_ACTION_LEN - 1) << 8) | \
|
||||
(COUNTER_KEY_END_LEN - 1)
|
||||
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_PERCPU_HASH);
|
||||
__uint(key_size, sizeof(struct counters_key));
|
||||
__uint(value_size, sizeof(__u64));
|
||||
__uint(max_entries, COUNTERS_MAP_MAX_ENTRIES);
|
||||
} counters_map SEC(".maps");
|
||||
|
||||
struct stunreq {
|
||||
__be16 type;
|
||||
__be16 length;
|
||||
__be32 magic;
|
||||
__be32 txid[3];
|
||||
// attributes follow
|
||||
};
|
||||
|
||||
struct stunattr {
|
||||
__be16 num;
|
||||
__be16 length;
|
||||
};
|
||||
|
||||
struct stunxor {
|
||||
__u8 unused;
|
||||
__u8 family;
|
||||
__be16 port;
|
||||
__be32 addr;
|
||||
};
|
||||
|
||||
struct stunxor6 {
|
||||
__u8 unused;
|
||||
__u8 family;
|
||||
__be16 port;
|
||||
__be32 addr[4];
|
||||
};
|
||||
|
||||
#define STUN_BINDING_REQUEST 1
|
||||
|
||||
#define STUN_MAGIC 0x2112a442
|
||||
|
||||
#define STUN_ATTR_SW 0x8022
|
||||
|
||||
#define STUN_ATTR_XOR_MAPPED_ADDR 0x0020
|
||||
|
||||
#define STUN_BINDING_RESPONSE 0x0101
|
||||
|
||||
#define STUN_MAGIC_FOR_PORT_XOR 0x2112
|
||||
|
||||
#define MAX_UDP_LEN_IPV4 1480
|
||||
|
||||
#define MAX_UDP_LEN_IPV6 1460
|
||||
|
||||
#define IP_MF 0x2000
|
||||
#define IP_OFFSET 0x1fff
|
||||
|
||||
static __always_inline __u16 csum_fold_flip(__u32 csum) {
|
||||
__u32 sum;
|
||||
sum = (csum >> 16) + (csum & 0xffff); // maximum value 0x1fffe
|
||||
sum += (sum >> 16); // maximum value 0xffff
|
||||
return ~sum;
|
||||
}
|
||||
|
||||
// csum_const_size is an alternative to bpf_csum_diff. It's a verifier
|
||||
// workaround for when we are forced to use a constant max_size + bounds
|
||||
// checking. The alternative being passing a dynamic length to bpf_csum_diff
|
||||
// {from,to}_size arguments, which the verifier can't follow. For further info
|
||||
// see: https://github.com/iovisor/bcc/issues/2463#issuecomment-512503958
|
||||
static __always_inline __u16 csum_const_size(__u32 seed, void* from, void* data_end, int max_size) {
|
||||
__u16 *buf = from;
|
||||
for (int i = 0; i < max_size; i += 2) {
|
||||
if ((void *)(buf + 1) > data_end) {
|
||||
break;
|
||||
}
|
||||
seed += *buf;
|
||||
buf++;
|
||||
}
|
||||
if ((void *)buf + 1 <= data_end) {
|
||||
seed += *(__u8 *)buf;
|
||||
}
|
||||
return csum_fold_flip(seed);
|
||||
}
|
||||
|
||||
static __always_inline __u32 pseudo_sum_ipv6(struct ipv6hdr* ip6, __u16 udp_len) {
|
||||
__u32 pseudo = 0; // TODO(jwhited): __u64 for intermediate checksum values to reduce number of ops
|
||||
for (int i = 0; i < 8; i ++) {
|
||||
pseudo += ip6->saddr.in6_u.u6_addr16[i];
|
||||
pseudo += ip6->daddr.in6_u.u6_addr16[i];
|
||||
}
|
||||
pseudo += bpf_htons(ip6->nexthdr);
|
||||
pseudo += udp_len;
|
||||
return pseudo;
|
||||
}
|
||||
|
||||
static __always_inline __u32 pseudo_sum_ipv4(struct iphdr* ip, __u16 udp_len) {
|
||||
__u32 pseudo = (__u16)ip->saddr;
|
||||
pseudo += (__u16)(ip->saddr >> 16);
|
||||
pseudo += (__u16)ip->daddr;
|
||||
pseudo += (__u16)(ip->daddr >> 16);
|
||||
pseudo += bpf_htons(ip->protocol);
|
||||
pseudo += udp_len;
|
||||
return pseudo;
|
||||
}
|
||||
|
||||
struct packet_context {
|
||||
enum counter_key_af af;
|
||||
enum counter_key_prog_end prog_end;
|
||||
};
|
||||
|
||||
static __always_inline int inc_counter(struct counters_key key, __u64 val) {
|
||||
__u64 *counter = bpf_map_lookup_elem(&counters_map, &key);
|
||||
if (!counter) {
|
||||
return bpf_map_update_elem(&counters_map, &key, &val, BPF_ANY);
|
||||
}
|
||||
*counter += val;
|
||||
return bpf_map_update_elem(&counters_map, &key, counter, BPF_ANY);
|
||||
}
|
||||
|
||||
static __always_inline int handle_counters(struct xdp_md *ctx, int action, struct packet_context *pctx) {
|
||||
void *data_end = (void *)(long)ctx->data_end;
|
||||
void *data = (void *)(long)ctx->data;
|
||||
|
||||
__u64 bytes = data_end - data;
|
||||
enum counter_key_packets_bytes_action packets_pba = COUNTER_KEY_PACKETS_PASS_TOTAL;
|
||||
enum counter_key_packets_bytes_action bytes_pba = COUNTER_KEY_BYTES_PASS_TOTAL;
|
||||
|
||||
switch (action) {
|
||||
case XDP_ABORTED:
|
||||
packets_pba = COUNTER_KEY_PACKETS_ABORTED_TOTAL;
|
||||
bytes_pba = COUNTER_KEY_BYTES_ABORTED_TOTAL;
|
||||
break;
|
||||
case XDP_PASS:
|
||||
packets_pba = COUNTER_KEY_PACKETS_PASS_TOTAL;
|
||||
bytes_pba = COUNTER_KEY_BYTES_PASS_TOTAL;
|
||||
break;
|
||||
case XDP_TX:
|
||||
packets_pba = COUNTER_KEY_PACKETS_TX_TOTAL;
|
||||
bytes_pba = COUNTER_KEY_BYTES_TX_TOTAL;
|
||||
break;
|
||||
case XDP_DROP:
|
||||
packets_pba = COUNTER_KEY_PACKETS_DROP_TOTAL;
|
||||
bytes_pba = COUNTER_KEY_BYTES_DROP_TOTAL;
|
||||
break;
|
||||
}
|
||||
|
||||
struct counters_key packets_key = {
|
||||
.af = pctx->af,
|
||||
.pba = packets_pba,
|
||||
.prog_end = pctx->prog_end,
|
||||
};
|
||||
|
||||
struct counters_key bytes_key = {
|
||||
.af = pctx->af,
|
||||
.pba = bytes_pba,
|
||||
.prog_end = pctx->prog_end,
|
||||
};
|
||||
|
||||
inc_counter(packets_key, 1);
|
||||
inc_counter(bytes_key, bytes);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define is_ipv6 (pctx->af == COUNTER_KEY_AF_IPV6)
|
||||
static __always_inline int handle_packet(struct xdp_md *ctx, struct packet_context *pctx) {
|
||||
void *data_end = (void *)(long)ctx->data_end;
|
||||
void *data = (void *)(long)ctx->data;
|
||||
pctx->af = COUNTER_KEY_AF_UNKNOWN;
|
||||
pctx->prog_end = COUNTER_KEY_END_UNSPECIFIED;
|
||||
|
||||
struct ethhdr *eth = data;
|
||||
if ((void *)(eth + 1) > data_end) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
struct iphdr *ip;
|
||||
struct ipv6hdr *ip6;
|
||||
struct udphdr *udp;
|
||||
|
||||
int validate_udp_csum;
|
||||
if (eth->h_proto == bpf_htons(ETH_P_IP)) {
|
||||
pctx->af = COUNTER_KEY_AF_IPV4;
|
||||
ip = (void *)(eth + 1);
|
||||
if ((void *)(ip + 1) > data_end) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
if (ip->ihl != 5 ||
|
||||
ip->version != 4 ||
|
||||
ip->protocol != IPPROTO_UDP ||
|
||||
(ip->frag_off & bpf_htons(IP_MF | IP_OFFSET)) != 0) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
// validate ipv4 header checksum
|
||||
__u32 cs_unfolded = bpf_csum_diff(0, 0, (void *)ip, sizeof(*ip), 0);
|
||||
__u16 cs = csum_fold_flip(cs_unfolded);
|
||||
if (cs != 0) {
|
||||
pctx->prog_end = COUNTER_KEY_END_INVALID_IP_CSUM;
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
if (bpf_ntohs(ip->tot_len) != data_end - (void *)ip) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
udp = (void *)(ip + 1);
|
||||
if ((void *)(udp + 1) > data_end) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
if (udp->check != 0) {
|
||||
// https://datatracker.ietf.org/doc/html/rfc768#page-3
|
||||
// If the computed checksum is zero, it is transmitted as all
|
||||
// ones (the equivalent in one's complement arithmetic). An all
|
||||
// zero transmitted checksum value means that the transmitter
|
||||
// generated no checksum (for debugging or for higher level
|
||||
// protocols that don't care).
|
||||
validate_udp_csum = 1;
|
||||
}
|
||||
} else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) {
|
||||
pctx->af = COUNTER_KEY_AF_IPV6;
|
||||
ip6 = (void *)(eth + 1);
|
||||
if ((void *)(ip6 + 1) > data_end) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
if (ip6->version != 6 || ip6->nexthdr != IPPROTO_UDP) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
udp = (void *)(ip6 + 1);
|
||||
if ((void *)(udp + 1) > data_end) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
if (bpf_ntohs(ip6->payload_len) != data_end - (void *)udp) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
// https://datatracker.ietf.org/doc/html/rfc8200#page-28
|
||||
// Unlike IPv4, the default behavior when UDP packets are
|
||||
// originated by an IPv6 node is that the UDP checksum is not
|
||||
// optional. That is, whenever originating a UDP packet, an IPv6
|
||||
// node must compute a UDP checksum over the packet and the
|
||||
// pseudo-header, and, if that computation yields a result of
|
||||
// zero, it must be changed to hex FFFF for placement in the UDP
|
||||
// header. IPv6 receivers must discard UDP packets containing a
|
||||
// zero checksum and should log the error.
|
||||
validate_udp_csum = 1;
|
||||
} else {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
__u32 config_key = 0;
|
||||
struct config *c = bpf_map_lookup_elem(&config_map, &config_key);
|
||||
if (!c) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
if (bpf_ntohs(udp->len) != data_end - (void *)udp) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
if (bpf_ntohs(udp->dest) != c->dst_port) {
|
||||
pctx->prog_end = COUNTER_KEY_END_NOT_STUN_PORT;
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
if (validate_udp_csum) {
|
||||
__u16 cs;
|
||||
__u32 pseudo_sum;
|
||||
if (is_ipv6) {
|
||||
pseudo_sum = pseudo_sum_ipv6(ip6, udp->len);
|
||||
cs = csum_const_size(pseudo_sum, udp, data_end, MAX_UDP_LEN_IPV6);
|
||||
} else {
|
||||
pseudo_sum = pseudo_sum_ipv4(ip, udp->len);
|
||||
cs = csum_const_size(pseudo_sum, udp, data_end, MAX_UDP_LEN_IPV4);
|
||||
}
|
||||
if (cs != 0) {
|
||||
pctx->prog_end = COUNTER_KEY_END_INVALID_UDP_CSUM;
|
||||
return XDP_PASS;
|
||||
}
|
||||
}
|
||||
|
||||
struct stunreq *req = (void *)(udp + 1);
|
||||
if ((void *)(req + 1) > data_end) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
if (req->type != bpf_htons(STUN_BINDING_REQUEST)) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
if (bpf_ntohl(req->magic) != STUN_MAGIC) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
void *attrs = (void *)(req + 1);
|
||||
__u16 attrs_len = ((char *)data_end) - ((char *)attrs);
|
||||
if (bpf_ntohs(req->length) != attrs_len) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
struct stunattr *sa = attrs;
|
||||
if ((void *)(sa + 1) > data_end) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
// Assume the order and contents of attributes. We *could* loop through
|
||||
// them, but parsing their lengths and performing arithmetic against the
|
||||
// packet pointer is more pain than it's worth. Bounds checks are invisible
|
||||
// to the verifier in certain circumstances where things move from registers
|
||||
// to the stack and/or compilation optimizations remove them entirely. There
|
||||
// have only ever been two attributes included by the client, and we are
|
||||
// only interested in one of them, anyway. Verify the software attribute,
|
||||
// but ignore the fingerprint attribute as it's only useful where STUN is
|
||||
// multiplexed with other traffic on the same port/socket, which is not the
|
||||
// case here.
|
||||
void *attr_data = (void *)(sa + 1);
|
||||
if (bpf_ntohs(sa->length) != 8 || bpf_ntohs(sa->num) != STUN_ATTR_SW) {
|
||||
pctx->prog_end = COUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR;
|
||||
return XDP_PASS;
|
||||
}
|
||||
if (attr_data + 8 > data_end) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
char want_sw[] = {0x74, 0x61, 0x69, 0x6c, 0x6e, 0x6f, 0x64, 0x65}; // tailnode
|
||||
char *got_sw = attr_data;
|
||||
for (int j = 0; j < 8; j++) {
|
||||
if (got_sw[j] != want_sw[j]) {
|
||||
pctx->prog_end = COUNTER_KEY_END_INVALID_SW_ATTR_VAL;
|
||||
return XDP_PASS;
|
||||
}
|
||||
}
|
||||
|
||||
// Begin transforming packet into a STUN_BINDING_RESPONSE. From here
|
||||
// onwards we return XDP_ABORTED instead of XDP_PASS when transformations or
|
||||
// bounds checks fail as it would be nonsensical to pass a mangled packet
|
||||
// through to the kernel, and we may be interested in debugging via
|
||||
// tracepoint.
|
||||
|
||||
// Set success response and new length. Magic cookie and txid remain the
|
||||
// same.
|
||||
req->type = bpf_htons(STUN_BINDING_RESPONSE);
|
||||
if (is_ipv6) {
|
||||
req->length = bpf_htons(sizeof(struct stunattr) + sizeof(struct stunxor6));
|
||||
} else {
|
||||
req->length = bpf_htons(sizeof(struct stunattr) + sizeof(struct stunxor));
|
||||
}
|
||||
|
||||
// Set attr type. Length remains unchanged, but set it again for future
|
||||
// safety reasons.
|
||||
sa->num = bpf_htons(STUN_ATTR_XOR_MAPPED_ADDR);
|
||||
if (is_ipv6) {
|
||||
sa->length = bpf_htons(sizeof(struct stunxor6));
|
||||
} else {
|
||||
sa->length = bpf_htons(sizeof(struct stunxor));
|
||||
}
|
||||
|
||||
struct stunxor *xor;
|
||||
struct stunxor6 *xor6;
|
||||
|
||||
// Adjust tail and reset header pointers.
|
||||
int adjust_tail_by;
|
||||
if (is_ipv6) {
|
||||
xor6 = attr_data;
|
||||
adjust_tail_by = (void *)(xor6 + 1) - data_end;
|
||||
} else {
|
||||
xor = attr_data;
|
||||
adjust_tail_by = (void *)(xor + 1) - data_end;
|
||||
}
|
||||
if (bpf_xdp_adjust_tail(ctx, adjust_tail_by)) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
data_end = (void *)(long)ctx->data_end;
|
||||
data = (void *)(long)ctx->data;
|
||||
eth = data;
|
||||
if ((void *)(eth + 1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
if (is_ipv6) {
|
||||
ip6 = (void *)(eth + 1);
|
||||
if ((void *)(ip6 + 1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
udp = (void *)(ip6 + 1);
|
||||
if ((void *)(udp + 1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
} else {
|
||||
ip = (void *)(eth + 1);
|
||||
if ((void *)(ip + 1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
udp = (void *)(ip + 1);
|
||||
if ((void *)(udp + 1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
}
|
||||
req = (void *)(udp + 1);
|
||||
if ((void *)(req + 1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
sa = (void *)(req + 1);
|
||||
if ((void *)(sa + 1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
|
||||
// Set attr data.
|
||||
if (is_ipv6) {
|
||||
xor6 = (void *)(sa + 1);
|
||||
if ((void *)(xor6 + 1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
xor6->unused = 0x00; // unused byte
|
||||
xor6->family = 0x02;
|
||||
xor6->port = udp->source ^ bpf_htons(STUN_MAGIC_FOR_PORT_XOR);
|
||||
xor6->addr[0] = ip6->saddr.in6_u.u6_addr32[0] ^ bpf_htonl(STUN_MAGIC);
|
||||
for (int i = 1; i < 4; i++) {
|
||||
// All three are __be32, no endianness flips.
|
||||
xor6->addr[i] = ip6->saddr.in6_u.u6_addr32[i] ^ req->txid[i-1];
|
||||
}
|
||||
} else {
|
||||
xor = (void *)(sa + 1);
|
||||
if ((void *)(xor + 1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
xor->unused = 0x00; // unused byte
|
||||
xor->family = 0x01;
|
||||
xor->port = udp->source ^ bpf_htons(STUN_MAGIC_FOR_PORT_XOR);
|
||||
xor->addr = ip->saddr ^ bpf_htonl(STUN_MAGIC);
|
||||
}
|
||||
|
||||
// Flip ethernet header source and destination address.
|
||||
__u8 eth_tmp[ETH_ALEN];
|
||||
__builtin_memcpy(eth_tmp, eth->h_source, ETH_ALEN);
|
||||
__builtin_memcpy(eth->h_source, eth->h_dest, ETH_ALEN);
|
||||
__builtin_memcpy(eth->h_dest, eth_tmp, ETH_ALEN);
|
||||
|
||||
// Flip ip header source and destination address.
|
||||
if (is_ipv6) {
|
||||
struct in6_addr ip_tmp = ip6->saddr;
|
||||
ip6->saddr = ip6->daddr;
|
||||
ip6->daddr = ip_tmp;
|
||||
} else {
|
||||
__be32 ip_tmp = ip->saddr;
|
||||
ip->saddr = ip->daddr;
|
||||
ip->daddr = ip_tmp;
|
||||
}
|
||||
|
||||
// Flip udp header source and destination ports;
|
||||
__be16 port_tmp = udp->source;
|
||||
udp->source = udp->dest;
|
||||
udp->dest = port_tmp;
|
||||
|
||||
// Update total length, TTL, and checksum.
|
||||
__u32 cs = 0;
|
||||
if (is_ipv6) {
|
||||
if ((void *)(ip6 +1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
__u16 payload_len = data_end - (void *)(ip6 + 1);
|
||||
ip6->payload_len = bpf_htons(payload_len);
|
||||
ip6->hop_limit = IPDEFTTL;
|
||||
} else {
|
||||
__u16 tot_len = data_end - (void *)ip;
|
||||
ip->tot_len = bpf_htons(tot_len);
|
||||
ip->ttl = IPDEFTTL;
|
||||
ip->check = 0;
|
||||
cs = bpf_csum_diff(0, 0, (void *)ip, sizeof(*ip), cs);
|
||||
ip->check = csum_fold_flip(cs);
|
||||
}
|
||||
|
||||
// Avoid dynamic length math against the packet pointer, which is just a big
|
||||
// verifier headache. Instead sizeof() all the things.
|
||||
int to_csum_len = sizeof(*udp) + sizeof(*req) + sizeof(*sa);
|
||||
// Update udp header length and checksum.
|
||||
if (is_ipv6) {
|
||||
to_csum_len += sizeof(*xor6);
|
||||
udp = (void *)(ip6 + 1);
|
||||
if ((void *)(udp +1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
__u16 udp_len = data_end - (void *)udp;
|
||||
udp->len = bpf_htons(udp_len);
|
||||
udp->check = 0;
|
||||
cs = pseudo_sum_ipv6(ip6, udp->len);
|
||||
} else {
|
||||
to_csum_len += sizeof(*xor);
|
||||
udp = (void *)(ip + 1);
|
||||
if ((void *)(udp +1) > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
__u16 udp_len = data_end - (void *)udp;
|
||||
udp->len = bpf_htons(udp_len);
|
||||
udp->check = 0;
|
||||
cs = pseudo_sum_ipv4(ip, udp->len);
|
||||
}
|
||||
if ((void *)udp + to_csum_len > data_end) {
|
||||
return XDP_ABORTED;
|
||||
}
|
||||
cs = bpf_csum_diff(0, 0, (void*)udp, to_csum_len, cs);
|
||||
udp->check = csum_fold_flip(cs);
|
||||
return XDP_TX;
|
||||
}
|
||||
#undef is_ipv6
|
||||
|
||||
SEC("xdp")
|
||||
int xdp_prog_func(struct xdp_md *ctx) {
|
||||
struct packet_context pctx = {
|
||||
.af = COUNTER_KEY_AF_UNKNOWN,
|
||||
.prog_end = COUNTER_KEY_END_UNSPECIFIED,
|
||||
};
|
||||
int action = XDP_PASS;
|
||||
action = handle_packet(ctx, &pctx);
|
||||
handle_counters(ctx, action, &pctx);
|
||||
return action;
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !linux
|
||||
|
||||
package xdp
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// STUNServer is unimplemented on these platforms, see xdp_linux.go.
|
||||
type STUNServer struct {
|
||||
}
|
||||
|
||||
func NewSTUNServer(config *STUNServerConfig, opts ...STUNServerOption) (*STUNServer, error) {
|
||||
return nil, errors.New("unimplemented on this GOOS")
|
||||
}
|
||||
|
||||
func (s *STUNServer) Close() error {
|
||||
return errors.New("unimplemented on this GOOS")
|
||||
}
|
||||
|
||||
func (s *STUNServer) Describe(descCh chan<- *prometheus.Desc) {}
|
||||
|
||||
func (s *STUNServer) Collect(metricCh chan<- prometheus.Metric) {}
|
||||
@ -0,0 +1,282 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package xdp
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"net"
|
||||
"sync"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
"github.com/cilium/ebpf/link"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -type config -type counters_key -type counter_key_af -type counter_key_packets_bytes_action -type counter_key_prog_end bpf xdp.c -- -I headers
|
||||
|
||||
// STUNServer manages loading and unloading of an eBPF XDP program that serves
|
||||
// the STUN protocol. It exports statistics for the XDP program via its
|
||||
// implementation of the prometheus.Collector interface.
|
||||
type STUNServer struct {
|
||||
mu sync.Mutex
|
||||
objs *bpfObjects
|
||||
metrics *stunServerMetrics
|
||||
}
|
||||
|
||||
//lint:ignore U1000 used in xdp_linux_test.go, which has a build tag
|
||||
type noAttachOption struct{}
|
||||
|
||||
//lint:ignore u1000 Used in xdp_linux_test.go, which has a build tag
|
||||
func (n noAttachOption) apply(opts *stunServerOptions) {
|
||||
opts.noAttach = true
|
||||
}
|
||||
|
||||
func (s *STUNServerConfig) validate() error {
|
||||
if len(s.DeviceName) < 1 {
|
||||
return errors.New("DeviceName is unspecified")
|
||||
}
|
||||
if s.DstPort < 0 || s.DstPort > math.MaxUint16 {
|
||||
return errors.New("DstPort is outside of uint16 bounds")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewSTUNServer returns an instance of a STUNServer that has attached the STUN
|
||||
// XDP program to the netdev and destination port specified by config.
|
||||
func NewSTUNServer(config *STUNServerConfig, opts ...STUNServerOption) (*STUNServer, error) {
|
||||
o := &stunServerOptions{}
|
||||
for _, opt := range opts {
|
||||
opt.apply(o)
|
||||
}
|
||||
err := config.validate()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid config: %v", err)
|
||||
}
|
||||
objs := new(bpfObjects)
|
||||
err = loadBpfObjects(objs, nil)
|
||||
if err != nil {
|
||||
var ve *ebpf.VerifierError
|
||||
if config.FullVerifierErr && errors.As(err, &ve) {
|
||||
err = fmt.Errorf("verifier error: %+v", ve)
|
||||
}
|
||||
return nil, fmt.Errorf("error loading XDP program: %w", err)
|
||||
}
|
||||
server := &STUNServer{
|
||||
objs: objs,
|
||||
metrics: newSTUNServerMetrics(),
|
||||
}
|
||||
var key uint32
|
||||
xdpConfig := bpfConfig{
|
||||
DstPort: uint16(config.DstPort),
|
||||
}
|
||||
err = objs.ConfigMap.Put(key, &xdpConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error loading config in eBPF map: %w", err)
|
||||
}
|
||||
if o.noAttach {
|
||||
return server, nil
|
||||
}
|
||||
iface, err := net.InterfaceByName(config.DeviceName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error finding device: %w", err)
|
||||
}
|
||||
_, err = link.AttachXDP(link.XDPOptions{
|
||||
Program: objs.XdpProgFunc,
|
||||
Interface: iface.Index,
|
||||
Flags: link.XDPAttachFlags(config.AttachFlags),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error attaching XDP program to dev: %w", err)
|
||||
}
|
||||
return server, nil
|
||||
}
|
||||
|
||||
// Close unloads the XDP program and associated maps.
|
||||
func (s *STUNServer) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.objs.Close()
|
||||
}
|
||||
|
||||
type stunServerMetrics struct {
|
||||
last map[bpfCountersKey]uint64
|
||||
registry *prometheus.Registry
|
||||
packets *prometheus.CounterVec
|
||||
bytes *prometheus.CounterVec
|
||||
}
|
||||
|
||||
func newSTUNServerMetrics() *stunServerMetrics {
|
||||
last := make(map[bpfCountersKey]uint64)
|
||||
registry := prometheus.NewRegistry()
|
||||
packets := prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "xdp",
|
||||
Subsystem: "stun_server",
|
||||
Name: "packets_total",
|
||||
}, []string{addressFamilyKey, xdpOutcomeKey, progEndKey})
|
||||
bytes := prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "xdp",
|
||||
Subsystem: "stun_server",
|
||||
Name: "bytes_total",
|
||||
}, []string{addressFamilyKey, xdpOutcomeKey, progEndKey})
|
||||
registry.MustRegister(packets, bytes)
|
||||
return &stunServerMetrics{
|
||||
last: last,
|
||||
registry: registry,
|
||||
packets: packets,
|
||||
bytes: bytes,
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
xdpOutcomeKey = "xdp_outcome"
|
||||
progEndKey = "prog_end"
|
||||
)
|
||||
|
||||
const (
|
||||
xdpOutcomePass = "pass"
|
||||
xdpOutcomeAborted = "aborted"
|
||||
xdpOutcomeDrop = "drop"
|
||||
xdpOutcomeTX = "tx"
|
||||
)
|
||||
|
||||
func sum(vals []uint64) uint64 {
|
||||
var s uint64
|
||||
for _, v := range vals {
|
||||
s += v
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
const (
|
||||
addressFamilyKey = "address_family"
|
||||
)
|
||||
|
||||
const (
|
||||
addressFamilyUnknown = "unknown"
|
||||
addressFamilyIPv4 = "ipv4"
|
||||
addressFamilyIPv6 = "ipv6"
|
||||
)
|
||||
|
||||
var (
|
||||
// TODO(jwhited): go generate these maps or equivalent switch logic behind bpf2go
|
||||
pbaToOutcomeLV = map[bpfCounterKeyPacketsBytesAction]string{
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL: xdpOutcomePass,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL: xdpOutcomePass,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL: xdpOutcomeAborted,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL: xdpOutcomeAborted,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL: xdpOutcomeTX,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL: xdpOutcomeTX,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL: xdpOutcomeDrop,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL: xdpOutcomeDrop,
|
||||
}
|
||||
|
||||
progEndLV = map[bpfCounterKeyProgEnd]string{
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED: "unspecified",
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR: "unexpected_first_stun_attr",
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM: "invalid_udp_csum",
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM: "invalid_ip_csum",
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_NOT_STUN_PORT: "not_stun_port",
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL: "invalid_sw_attr_val",
|
||||
}
|
||||
|
||||
packetCounterKeys = map[bpfCounterKeyPacketsBytesAction]bool{
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL: true,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL: true,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL: true,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL: true,
|
||||
}
|
||||
|
||||
//lint:ignore U1000 used in xdp_linux_test.go, which has a build tag
|
||||
bytesCounterKeys = map[bpfCounterKeyPacketsBytesAction]bool{
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL: true,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL: true,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL: true,
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL: true,
|
||||
}
|
||||
)
|
||||
|
||||
// increase returns the difference between "from" and "to" assuming they
|
||||
// originated from the same counter gathered at different times, i.e. "from"
|
||||
// was incremented by a non-negative value into "to". In the case of wraps
|
||||
// increase returns the difference between "to" and zero.
|
||||
func increase(from, to uint64) uint64 {
|
||||
if to >= from {
|
||||
return to - from
|
||||
}
|
||||
return to
|
||||
}
|
||||
|
||||
func (s *stunServerMetrics) updateFromMapKV(key bpfCountersKey, vals []uint64) error {
|
||||
if key.Unused != 0 ||
|
||||
key.Af >= uint8(bpfCounterKeyAfCOUNTER_KEY_AF_LEN) ||
|
||||
key.Pba >= uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN) ||
|
||||
key.ProgEnd >= uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_LEN) {
|
||||
return fmt.Errorf("unexpected counter key: %+v", key)
|
||||
}
|
||||
previousAllCPUs := s.last[key]
|
||||
allCPUs := sum(vals)
|
||||
s.last[key] = allCPUs
|
||||
inc := increase(previousAllCPUs, allCPUs)
|
||||
if inc == 0 {
|
||||
return nil
|
||||
}
|
||||
var af string
|
||||
switch key.Af {
|
||||
case uint8(bpfCounterKeyAfCOUNTER_KEY_AF_UNKNOWN):
|
||||
af = addressFamilyUnknown
|
||||
case uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4):
|
||||
af = addressFamilyIPv4
|
||||
case uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6):
|
||||
af = addressFamilyIPv6
|
||||
}
|
||||
labels := prometheus.Labels{
|
||||
addressFamilyKey: af,
|
||||
xdpOutcomeKey: pbaToOutcomeLV[bpfCounterKeyPacketsBytesAction(key.Pba)],
|
||||
progEndKey: progEndLV[bpfCounterKeyProgEnd(key.ProgEnd)],
|
||||
}
|
||||
var metric *prometheus.CounterVec
|
||||
if packetCounterKeys[bpfCounterKeyPacketsBytesAction(key.Pba)] {
|
||||
metric = s.packets
|
||||
} else {
|
||||
metric = s.bytes
|
||||
}
|
||||
metric.With(labels).Add(float64(inc))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Describe is part of the implementation of prometheus.Collector.
|
||||
func (s *STUNServer) Describe(descCh chan<- *prometheus.Desc) {
|
||||
s.metrics.registry.Describe(descCh)
|
||||
}
|
||||
|
||||
// Collect is part of the implementation of prometheus.Collector.
|
||||
func (s *STUNServer) Collect(metricCh chan<- prometheus.Metric) {
|
||||
err := s.updateMetrics()
|
||||
if err != nil {
|
||||
log.Printf("xdp: error collecting metrics: %v", err)
|
||||
}
|
||||
s.metrics.registry.Collect(metricCh)
|
||||
}
|
||||
|
||||
func (s *STUNServer) updateMetrics() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
iter := s.objs.CountersMap.Iterate()
|
||||
var key bpfCountersKey
|
||||
numCPU, err := ebpf.PossibleCPU()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vals := make([]uint64, numCPU)
|
||||
for iter.Next(&key, &vals) {
|
||||
err := s.metrics.updateFromMapKV(key, vals)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return iter.Err()
|
||||
}
|
||||
@ -0,0 +1,975 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build linux
|
||||
|
||||
package xdp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
"golang.org/x/sys/unix"
|
||||
"gvisor.dev/gvisor/pkg/tcpip"
|
||||
"gvisor.dev/gvisor/pkg/tcpip/checksum"
|
||||
"gvisor.dev/gvisor/pkg/tcpip/header"
|
||||
"tailscale.com/net/stun"
|
||||
)
|
||||
|
||||
type xdpAction uint32
|
||||
|
||||
func (x xdpAction) String() string {
|
||||
switch x {
|
||||
case xdpActionAborted:
|
||||
return "XDP_ABORTED"
|
||||
case xdpActionDrop:
|
||||
return "XDP_DROP"
|
||||
case xdpActionPass:
|
||||
return "XDP_PASS"
|
||||
case xdpActionTX:
|
||||
return "XDP_TX"
|
||||
case xdpActionRedirect:
|
||||
return "XDP_REDIRECT"
|
||||
default:
|
||||
return fmt.Sprintf("unknown(%d)", x)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
xdpActionAborted xdpAction = iota
|
||||
xdpActionDrop
|
||||
xdpActionPass
|
||||
xdpActionTX
|
||||
xdpActionRedirect
|
||||
)
|
||||
|
||||
const (
|
||||
ethHLen = 14
|
||||
udpHLen = 8
|
||||
ipv4HLen = 20
|
||||
ipv6HLen = 40
|
||||
)
|
||||
|
||||
const (
|
||||
defaultSTUNPort = 3478
|
||||
defaultTTL = 64
|
||||
reqSrcPort = uint16(1025)
|
||||
)
|
||||
|
||||
var (
|
||||
reqEthSrc = tcpip.LinkAddress([]byte{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01})
|
||||
reqEthDst = tcpip.LinkAddress([]byte{0x00, 0x00, 0x5e, 0x00, 0x53, 0x02})
|
||||
reqIPv4Src = netip.MustParseAddr("192.0.2.1")
|
||||
reqIPv4Dst = netip.MustParseAddr("192.0.2.2")
|
||||
reqIPv6Src = netip.MustParseAddr("2001:db8::1")
|
||||
reqIPv6Dst = netip.MustParseAddr("2001:db8::2")
|
||||
)
|
||||
|
||||
var testTXID = stun.TxID([12]byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b})
|
||||
|
||||
type ipv4Mutations struct {
|
||||
ipHeaderFn func(header.IPv4)
|
||||
udpHeaderFn func(header.UDP)
|
||||
stunReqFn func([]byte)
|
||||
}
|
||||
|
||||
func getIPv4STUNBindingReq(mutations *ipv4Mutations) []byte {
|
||||
req := stun.Request(testTXID)
|
||||
if mutations != nil && mutations.stunReqFn != nil {
|
||||
mutations.stunReqFn(req)
|
||||
}
|
||||
payloadLen := len(req)
|
||||
totalLen := ipv4HLen + udpHLen + payloadLen
|
||||
b := make([]byte, ethHLen+totalLen)
|
||||
ipv4H := header.IPv4(b[ethHLen:])
|
||||
ethH := header.Ethernet(b)
|
||||
ethFields := header.EthernetFields{
|
||||
SrcAddr: reqEthSrc,
|
||||
DstAddr: reqEthDst,
|
||||
Type: unix.ETH_P_IP,
|
||||
}
|
||||
ethH.Encode(ðFields)
|
||||
ipFields := header.IPv4Fields{
|
||||
SrcAddr: tcpip.AddrFrom4(reqIPv4Src.As4()),
|
||||
DstAddr: tcpip.AddrFrom4(reqIPv4Dst.As4()),
|
||||
Protocol: unix.IPPROTO_UDP,
|
||||
TTL: defaultTTL,
|
||||
TotalLength: uint16(totalLen),
|
||||
}
|
||||
ipv4H.Encode(&ipFields)
|
||||
ipv4H.SetChecksum(^ipv4H.CalculateChecksum())
|
||||
if mutations != nil && mutations.ipHeaderFn != nil {
|
||||
mutations.ipHeaderFn(ipv4H)
|
||||
}
|
||||
udpH := header.UDP(b[ethHLen+ipv4HLen:])
|
||||
udpFields := header.UDPFields{
|
||||
SrcPort: reqSrcPort,
|
||||
DstPort: defaultSTUNPort,
|
||||
Length: uint16(udpHLen + payloadLen),
|
||||
Checksum: 0,
|
||||
}
|
||||
udpH.Encode(&udpFields)
|
||||
copy(b[ethHLen+ipv4HLen+udpHLen:], req)
|
||||
cs := header.PseudoHeaderChecksum(
|
||||
unix.IPPROTO_UDP,
|
||||
ipv4H.SourceAddress(),
|
||||
ipv4H.DestinationAddress(),
|
||||
uint16(udpHLen+payloadLen),
|
||||
)
|
||||
cs = checksum.Checksum(req, cs)
|
||||
udpH.SetChecksum(^udpH.CalculateChecksum(cs))
|
||||
if mutations != nil && mutations.udpHeaderFn != nil {
|
||||
mutations.udpHeaderFn(udpH)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
type ipv6Mutations struct {
|
||||
ipHeaderFn func(header.IPv6)
|
||||
udpHeaderFn func(header.UDP)
|
||||
stunReqFn func([]byte)
|
||||
}
|
||||
|
||||
func getIPv6STUNBindingReq(mutations *ipv6Mutations) []byte {
|
||||
req := stun.Request(testTXID)
|
||||
if mutations != nil && mutations.stunReqFn != nil {
|
||||
mutations.stunReqFn(req)
|
||||
}
|
||||
payloadLen := len(req)
|
||||
src := netip.MustParseAddr("2001:db8::1")
|
||||
dst := netip.MustParseAddr("2001:db8::2")
|
||||
b := make([]byte, ethHLen+ipv6HLen+udpHLen+payloadLen)
|
||||
ipv6H := header.IPv6(b[ethHLen:])
|
||||
ethH := header.Ethernet(b)
|
||||
ethFields := header.EthernetFields{
|
||||
SrcAddr: tcpip.LinkAddress([]byte{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01}),
|
||||
DstAddr: tcpip.LinkAddress([]byte{0x00, 0x00, 0x5e, 0x00, 0x53, 0x02}),
|
||||
Type: unix.ETH_P_IPV6,
|
||||
}
|
||||
ethH.Encode(ðFields)
|
||||
ipFields := header.IPv6Fields{
|
||||
SrcAddr: tcpip.AddrFrom16(src.As16()),
|
||||
DstAddr: tcpip.AddrFrom16(dst.As16()),
|
||||
TransportProtocol: unix.IPPROTO_UDP,
|
||||
HopLimit: 64,
|
||||
PayloadLength: uint16(udpHLen + payloadLen),
|
||||
}
|
||||
ipv6H.Encode(&ipFields)
|
||||
if mutations != nil && mutations.ipHeaderFn != nil {
|
||||
mutations.ipHeaderFn(ipv6H)
|
||||
}
|
||||
udpH := header.UDP(b[ethHLen+ipv6HLen:])
|
||||
udpFields := header.UDPFields{
|
||||
SrcPort: 1025,
|
||||
DstPort: defaultSTUNPort,
|
||||
Length: uint16(udpHLen + payloadLen),
|
||||
Checksum: 0,
|
||||
}
|
||||
udpH.Encode(&udpFields)
|
||||
copy(b[ethHLen+ipv6HLen+udpHLen:], req)
|
||||
cs := header.PseudoHeaderChecksum(
|
||||
unix.IPPROTO_UDP,
|
||||
ipv6H.SourceAddress(),
|
||||
ipv6H.DestinationAddress(),
|
||||
uint16(udpHLen+payloadLen),
|
||||
)
|
||||
cs = checksum.Checksum(req, cs)
|
||||
udpH.SetChecksum(^udpH.CalculateChecksum(cs))
|
||||
if mutations != nil && mutations.udpHeaderFn != nil {
|
||||
mutations.udpHeaderFn(udpH)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func getIPv4STUNBindingResp() []byte {
|
||||
addrPort := netip.AddrPortFrom(reqIPv4Src, reqSrcPort)
|
||||
resp := stun.Response(testTXID, addrPort)
|
||||
payloadLen := len(resp)
|
||||
totalLen := ipv4HLen + udpHLen + payloadLen
|
||||
b := make([]byte, ethHLen+totalLen)
|
||||
ipv4H := header.IPv4(b[ethHLen:])
|
||||
ethH := header.Ethernet(b)
|
||||
ethFields := header.EthernetFields{
|
||||
SrcAddr: reqEthDst,
|
||||
DstAddr: reqEthSrc,
|
||||
Type: unix.ETH_P_IP,
|
||||
}
|
||||
ethH.Encode(ðFields)
|
||||
ipFields := header.IPv4Fields{
|
||||
SrcAddr: tcpip.AddrFrom4(reqIPv4Dst.As4()),
|
||||
DstAddr: tcpip.AddrFrom4(reqIPv4Src.As4()),
|
||||
Protocol: unix.IPPROTO_UDP,
|
||||
TTL: defaultTTL,
|
||||
TotalLength: uint16(totalLen),
|
||||
}
|
||||
ipv4H.Encode(&ipFields)
|
||||
ipv4H.SetChecksum(^ipv4H.CalculateChecksum())
|
||||
udpH := header.UDP(b[ethHLen+ipv4HLen:])
|
||||
udpFields := header.UDPFields{
|
||||
SrcPort: defaultSTUNPort,
|
||||
DstPort: reqSrcPort,
|
||||
Length: uint16(udpHLen + payloadLen),
|
||||
Checksum: 0,
|
||||
}
|
||||
udpH.Encode(&udpFields)
|
||||
copy(b[ethHLen+ipv4HLen+udpHLen:], resp)
|
||||
cs := header.PseudoHeaderChecksum(
|
||||
unix.IPPROTO_UDP,
|
||||
ipv4H.SourceAddress(),
|
||||
ipv4H.DestinationAddress(),
|
||||
uint16(udpHLen+payloadLen),
|
||||
)
|
||||
cs = checksum.Checksum(resp, cs)
|
||||
udpH.SetChecksum(^udpH.CalculateChecksum(cs))
|
||||
return b
|
||||
}
|
||||
|
||||
func getIPv6STUNBindingResp() []byte {
|
||||
addrPort := netip.AddrPortFrom(reqIPv6Src, reqSrcPort)
|
||||
resp := stun.Response(testTXID, addrPort)
|
||||
payloadLen := len(resp)
|
||||
totalLen := ipv6HLen + udpHLen + payloadLen
|
||||
b := make([]byte, ethHLen+totalLen)
|
||||
ipv6H := header.IPv6(b[ethHLen:])
|
||||
ethH := header.Ethernet(b)
|
||||
ethFields := header.EthernetFields{
|
||||
SrcAddr: reqEthDst,
|
||||
DstAddr: reqEthSrc,
|
||||
Type: unix.ETH_P_IPV6,
|
||||
}
|
||||
ethH.Encode(ðFields)
|
||||
ipFields := header.IPv6Fields{
|
||||
SrcAddr: tcpip.AddrFrom16(reqIPv6Dst.As16()),
|
||||
DstAddr: tcpip.AddrFrom16(reqIPv6Src.As16()),
|
||||
TransportProtocol: unix.IPPROTO_UDP,
|
||||
HopLimit: defaultTTL,
|
||||
PayloadLength: uint16(udpHLen + payloadLen),
|
||||
}
|
||||
ipv6H.Encode(&ipFields)
|
||||
udpH := header.UDP(b[ethHLen+ipv6HLen:])
|
||||
udpFields := header.UDPFields{
|
||||
SrcPort: defaultSTUNPort,
|
||||
DstPort: reqSrcPort,
|
||||
Length: uint16(udpHLen + payloadLen),
|
||||
Checksum: 0,
|
||||
}
|
||||
udpH.Encode(&udpFields)
|
||||
copy(b[ethHLen+ipv6HLen+udpHLen:], resp)
|
||||
cs := header.PseudoHeaderChecksum(
|
||||
unix.IPPROTO_UDP,
|
||||
ipv6H.SourceAddress(),
|
||||
ipv6H.DestinationAddress(),
|
||||
uint16(udpHLen+payloadLen),
|
||||
)
|
||||
cs = checksum.Checksum(resp, cs)
|
||||
udpH.SetChecksum(^udpH.CalculateChecksum(cs))
|
||||
return b
|
||||
}
|
||||
|
||||
func TestXDP(t *testing.T) {
|
||||
ipv4STUNBindingReqTX := getIPv4STUNBindingReq(nil)
|
||||
ipv6STUNBindingReqTX := getIPv6STUNBindingReq(nil)
|
||||
|
||||
ipv4STUNBindingReqIPCsumPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
ipHeaderFn: func(ipv4H header.IPv4) {
|
||||
oldCS := ipv4H.Checksum()
|
||||
newCS := oldCS
|
||||
for newCS == 0 || newCS == oldCS {
|
||||
newCS++
|
||||
}
|
||||
ipv4H.SetChecksum(newCS)
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqIHLPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
ipHeaderFn: func(ipv4H header.IPv4) {
|
||||
ipv4H[0] &= 0xF0
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqIPVerPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
ipHeaderFn: func(ipv4H header.IPv4) {
|
||||
ipv4H[0] &= 0x0F
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqIPProtoPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
ipHeaderFn: func(ipv4H header.IPv4) {
|
||||
ipv4H[9] = unix.IPPROTO_TCP
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqFragOffsetPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
ipHeaderFn: func(ipv4H header.IPv4) {
|
||||
ipv4H.SetFlagsFragmentOffset(ipv4H.Flags(), 8)
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqFlagsMFPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
ipHeaderFn: func(ipv4H header.IPv4) {
|
||||
ipv4H.SetFlagsFragmentOffset(header.IPv4FlagMoreFragments, 0)
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqTotLenPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
ipHeaderFn: func(ipv4H header.IPv4) {
|
||||
ipv4H.SetTotalLength(ipv4H.TotalLength() + 1)
|
||||
ipv4H.SetChecksum(0)
|
||||
ipv4H.SetChecksum(^ipv4H.CalculateChecksum())
|
||||
},
|
||||
})
|
||||
|
||||
ipv6STUNBindingReqIPVerPass := getIPv6STUNBindingReq(&ipv6Mutations{
|
||||
ipHeaderFn: func(ipv6H header.IPv6) {
|
||||
ipv6H[0] &= 0x0F
|
||||
},
|
||||
udpHeaderFn: func(udp header.UDP) {},
|
||||
})
|
||||
|
||||
ipv6STUNBindingReqNextHdrPass := getIPv6STUNBindingReq(&ipv6Mutations{
|
||||
ipHeaderFn: func(ipv6H header.IPv6) {
|
||||
ipv6H.SetNextHeader(unix.IPPROTO_TCP)
|
||||
},
|
||||
udpHeaderFn: func(udp header.UDP) {},
|
||||
})
|
||||
|
||||
ipv6STUNBindingReqPayloadLenPass := getIPv6STUNBindingReq(&ipv6Mutations{
|
||||
ipHeaderFn: func(ipv6H header.IPv6) {
|
||||
ipv6H.SetPayloadLength(ipv6H.PayloadLength() + 1)
|
||||
},
|
||||
udpHeaderFn: func(udp header.UDP) {},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqUDPCsumPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
udpHeaderFn: func(udpH header.UDP) {
|
||||
oldCS := udpH.Checksum()
|
||||
newCS := oldCS
|
||||
for newCS == 0 || newCS == oldCS {
|
||||
newCS++
|
||||
}
|
||||
udpH.SetChecksum(newCS)
|
||||
},
|
||||
})
|
||||
|
||||
ipv6STUNBindingReqUDPCsumPass := getIPv6STUNBindingReq(&ipv6Mutations{
|
||||
udpHeaderFn: func(udpH header.UDP) {
|
||||
oldCS := udpH.Checksum()
|
||||
newCS := oldCS
|
||||
for newCS == 0 || newCS == oldCS {
|
||||
newCS++
|
||||
}
|
||||
udpH.SetChecksum(newCS)
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqSTUNTypePass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[1] = ^req[1]
|
||||
},
|
||||
})
|
||||
|
||||
ipv6STUNBindingReqSTUNTypePass := getIPv6STUNBindingReq(&ipv6Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[1] = ^req[1]
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqSTUNMagicPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[4] = ^req[4]
|
||||
},
|
||||
})
|
||||
|
||||
ipv6STUNBindingReqSTUNMagicPass := getIPv6STUNBindingReq(&ipv6Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[4] = ^req[4]
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqSTUNAttrsLenPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[2] = ^req[2]
|
||||
},
|
||||
})
|
||||
|
||||
ipv6STUNBindingReqSTUNAttrsLenPass := getIPv6STUNBindingReq(&ipv6Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[2] = ^req[2]
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqSTUNSWValPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[24] = ^req[24]
|
||||
},
|
||||
})
|
||||
|
||||
ipv6STUNBindingReqSTUNSWValPass := getIPv6STUNBindingReq(&ipv6Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[24] = ^req[24]
|
||||
},
|
||||
})
|
||||
|
||||
ipv4STUNBindingReqSTUNFirstAttrPass := getIPv4STUNBindingReq(&ipv4Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[21] = ^req[21]
|
||||
},
|
||||
})
|
||||
|
||||
ipv6STUNBindingReqSTUNFirstAttrPass := getIPv6STUNBindingReq(&ipv6Mutations{
|
||||
stunReqFn: func(req []byte) {
|
||||
req[21] = ^req[21]
|
||||
},
|
||||
})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
packetIn []byte
|
||||
wantCode xdpAction
|
||||
wantPacketOut []byte
|
||||
wantMetrics map[bpfCountersKey]uint64
|
||||
}{
|
||||
{
|
||||
name: "ipv4 STUN Binding Request TX",
|
||||
packetIn: ipv4STUNBindingReqTX,
|
||||
wantCode: xdpActionTX,
|
||||
wantPacketOut: getIPv4STUNBindingResp(),
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(getIPv4STUNBindingResp())),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request TX",
|
||||
packetIn: ipv6STUNBindingReqTX,
|
||||
wantCode: xdpActionTX,
|
||||
wantPacketOut: getIPv6STUNBindingResp(),
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(getIPv6STUNBindingResp())),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request invalid ip csum PASS",
|
||||
packetIn: ipv4STUNBindingReqIPCsumPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqIPCsumPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM),
|
||||
}: uint64(len(ipv4STUNBindingReqIPCsumPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request ihl PASS",
|
||||
packetIn: ipv4STUNBindingReqIHLPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqIHLPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv4STUNBindingReqIHLPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request ip version PASS",
|
||||
packetIn: ipv4STUNBindingReqIPVerPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqIPVerPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv4STUNBindingReqIPVerPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request ip proto PASS",
|
||||
packetIn: ipv4STUNBindingReqIPProtoPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqIPProtoPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv4STUNBindingReqIPProtoPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request frag offset PASS",
|
||||
packetIn: ipv4STUNBindingReqFragOffsetPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqFragOffsetPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv4STUNBindingReqFragOffsetPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request flags mf PASS",
|
||||
packetIn: ipv4STUNBindingReqFlagsMFPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqFlagsMFPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv4STUNBindingReqFlagsMFPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request tot len PASS",
|
||||
packetIn: ipv4STUNBindingReqTotLenPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqTotLenPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv4STUNBindingReqTotLenPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request ip version PASS",
|
||||
packetIn: ipv6STUNBindingReqIPVerPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv6STUNBindingReqIPVerPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv6STUNBindingReqIPVerPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request next hdr PASS",
|
||||
packetIn: ipv6STUNBindingReqNextHdrPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv6STUNBindingReqNextHdrPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv6STUNBindingReqNextHdrPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request payload len PASS",
|
||||
packetIn: ipv6STUNBindingReqPayloadLenPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv6STUNBindingReqPayloadLenPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv6STUNBindingReqPayloadLenPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request UDP csum PASS",
|
||||
packetIn: ipv4STUNBindingReqUDPCsumPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqUDPCsumPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM),
|
||||
}: uint64(len(ipv4STUNBindingReqUDPCsumPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request UDP csum PASS",
|
||||
packetIn: ipv6STUNBindingReqUDPCsumPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv6STUNBindingReqUDPCsumPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM),
|
||||
}: uint64(len(ipv6STUNBindingReqUDPCsumPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request STUN type PASS",
|
||||
packetIn: ipv4STUNBindingReqSTUNTypePass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqSTUNTypePass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv4STUNBindingReqSTUNTypePass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request STUN type PASS",
|
||||
packetIn: ipv6STUNBindingReqSTUNTypePass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv6STUNBindingReqSTUNTypePass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv6STUNBindingReqSTUNTypePass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request STUN magic PASS",
|
||||
packetIn: ipv4STUNBindingReqSTUNMagicPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqSTUNMagicPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv4STUNBindingReqSTUNMagicPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request STUN magic PASS",
|
||||
packetIn: ipv6STUNBindingReqSTUNMagicPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv6STUNBindingReqSTUNMagicPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv6STUNBindingReqSTUNMagicPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request STUN attrs len PASS",
|
||||
packetIn: ipv4STUNBindingReqSTUNAttrsLenPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqSTUNAttrsLenPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv4STUNBindingReqSTUNAttrsLenPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request STUN attrs len PASS",
|
||||
packetIn: ipv6STUNBindingReqSTUNAttrsLenPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv6STUNBindingReqSTUNAttrsLenPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(len(ipv6STUNBindingReqSTUNAttrsLenPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request STUN SW val PASS",
|
||||
packetIn: ipv4STUNBindingReqSTUNSWValPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqSTUNSWValPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL),
|
||||
}: uint64(len(ipv4STUNBindingReqSTUNSWValPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request STUN SW val PASS",
|
||||
packetIn: ipv6STUNBindingReqSTUNSWValPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv6STUNBindingReqSTUNSWValPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL),
|
||||
}: uint64(len(ipv6STUNBindingReqSTUNSWValPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv4 STUN Binding Request STUN first attr PASS",
|
||||
packetIn: ipv4STUNBindingReqSTUNFirstAttrPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv4STUNBindingReqSTUNFirstAttrPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR),
|
||||
}: uint64(len(ipv4STUNBindingReqSTUNFirstAttrPass)),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ipv6 STUN Binding Request STUN first attr PASS",
|
||||
packetIn: ipv6STUNBindingReqSTUNFirstAttrPass,
|
||||
wantCode: xdpActionPass,
|
||||
wantPacketOut: ipv6STUNBindingReqSTUNFirstAttrPass,
|
||||
wantMetrics: map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR),
|
||||
}: 1,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR),
|
||||
}: uint64(len(ipv6STUNBindingReqSTUNFirstAttrPass)),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
server, err := NewSTUNServer(&STUNServerConfig{DeviceName: "fake", DstPort: defaultSTUNPort},
|
||||
&noAttachOption{})
|
||||
if err != nil {
|
||||
if errors.Is(err, unix.EPERM) {
|
||||
// TODO(jwhited): get this running
|
||||
t.Skip("skipping due to EPERM error; test requires elevated privileges")
|
||||
}
|
||||
t.Fatalf("error constructing STUN server: %v", err)
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
clearCounters := func() error {
|
||||
server.metrics.last = make(map[bpfCountersKey]uint64)
|
||||
var cur, next bpfCountersKey
|
||||
keys := make([]bpfCountersKey, 0)
|
||||
for err = server.objs.CountersMap.NextKey(nil, &next); ; err = server.objs.CountersMap.NextKey(cur, &next) {
|
||||
if err != nil {
|
||||
if errors.Is(err, ebpf.ErrKeyNotExist) {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
keys = append(keys, next)
|
||||
cur = next
|
||||
}
|
||||
for _, key := range keys {
|
||||
err = server.objs.CountersMap.Delete(&key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = server.objs.CountersMap.NextKey(nil, &next)
|
||||
if !errors.Is(err, ebpf.ErrKeyNotExist) {
|
||||
return errors.New("counters map is not empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
err = clearCounters()
|
||||
if err != nil {
|
||||
t.Fatalf("error clearing counters: %v", err)
|
||||
}
|
||||
opts := ebpf.RunOptions{
|
||||
Data: c.packetIn,
|
||||
DataOut: make([]byte, 1514),
|
||||
}
|
||||
got, err := server.objs.XdpProgFunc.Run(&opts)
|
||||
if err != nil {
|
||||
t.Fatalf("error running program: %v", err)
|
||||
}
|
||||
if xdpAction(got) != c.wantCode {
|
||||
t.Fatalf("got code: %s != %s", xdpAction(got), c.wantCode)
|
||||
}
|
||||
if !bytes.Equal(opts.DataOut, c.wantPacketOut) {
|
||||
t.Fatal("packets not equal")
|
||||
}
|
||||
err = server.updateMetrics()
|
||||
if err != nil {
|
||||
t.Fatalf("error updating metrics: %v", err)
|
||||
}
|
||||
if c.wantMetrics != nil {
|
||||
for k, v := range c.wantMetrics {
|
||||
gotCounter, ok := server.metrics.last[k]
|
||||
if !ok {
|
||||
t.Errorf("expected counter at key %+v not found", k)
|
||||
}
|
||||
if gotCounter != v {
|
||||
t.Errorf("key: %+v gotCounter: %d != %d", k, gotCounter, v)
|
||||
}
|
||||
}
|
||||
for k := range server.metrics.last {
|
||||
_, ok := c.wantMetrics[k]
|
||||
if !ok {
|
||||
t.Errorf("counter at key: %+v incremented unexpectedly", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountersMapKey(t *testing.T) {
|
||||
if bpfCounterKeyAfCOUNTER_KEY_AF_LEN > 256 {
|
||||
t.Error("COUNTER_KEY_AF_LEN no longer fits within uint8")
|
||||
}
|
||||
if bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN > 256 {
|
||||
t.Error("COUNTER_KEY_PACKETS_BYTES_ACTION no longer fits within uint8")
|
||||
}
|
||||
if bpfCounterKeyProgEndCOUNTER_KEY_END_LEN > 256 {
|
||||
t.Error("COUNTER_KEY_END_LEN no longer fits within uint8")
|
||||
}
|
||||
if len(pbaToOutcomeLV) != int(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN) {
|
||||
t.Error("pbaToOutcomeLV is not in sync with xdp.c")
|
||||
}
|
||||
if len(progEndLV) != int(bpfCounterKeyProgEndCOUNTER_KEY_END_LEN) {
|
||||
t.Error("progEndLV is not in sync with xdp.c")
|
||||
}
|
||||
if len(packetCounterKeys)+len(bytesCounterKeys) != int(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN) {
|
||||
t.Error("packetCounterKeys and/or bytesCounterKeys is not in sync with xdp.c")
|
||||
}
|
||||
if len(pbaToOutcomeLV) != int(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN) {
|
||||
t.Error("pbaToOutcomeLV is not in sync with xdp.c")
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue