mirror of https://github.com/tailscale/tailscale/
derp/xdp,cmd/xdpderper: initial skeleton (#12390)
This commit introduces a userspace program for managing an experimental eBPF XDP STUN server program. derp/xdp contains the eBPF pseudo-C along with a Go pkg for loading it and exporting its metrics. cmd/xdpderper is a package main user of derp/xdp. Updates tailscale/corp#20689 Signed-off-by: Jordan Whited <jordan@tailscale.com>dependabot/npm_and_yarn/cmd/tsconnect/braces-3.0.3
Jordan Whited
4 months ago
committed by
GitHub
parent
6908fb0de3
commit
65888d95c9
@ -0,0 +1,76 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"tailscale.com/derp/xdp"
|
||||
"tailscale.com/tsweb"
|
||||
)
|
||||
|
||||
var (
|
||||
flagDevice = flag.String("device", "", "target device name")
|
||||
flagPort = flag.Int("dst-port", 0, "destination UDP port to serve")
|
||||
flagVerbose = flag.Bool("verbose", false, "verbose output including verifier errors")
|
||||
flagMode = flag.String("mode", "xdp", "XDP mode; valid modes: [xdp, xdpgeneric, xdpdrv, xdpoffload]")
|
||||
flagHTTP = flag.String("http", ":8230", "HTTP listen address")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
var attachFlags xdp.XDPAttachFlags
|
||||
switch strings.ToLower(*flagMode) {
|
||||
case "xdp":
|
||||
attachFlags = 0
|
||||
case "xdpgeneric":
|
||||
attachFlags = xdp.XDPGenericMode
|
||||
case "xdpdrv":
|
||||
attachFlags = xdp.XDPDriverMode
|
||||
case "xdpoffload":
|
||||
attachFlags = xdp.XDPOffloadMode
|
||||
default:
|
||||
log.Fatal("invalid mode")
|
||||
}
|
||||
server, err := xdp.NewSTUNServer(&xdp.STUNServerConfig{
|
||||
DeviceName: *flagDevice,
|
||||
DstPort: *flagPort,
|
||||
AttachFlags: attachFlags,
|
||||
FullVerifierErr: *flagVerbose,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("failed to init XDP STUN server: %v", err)
|
||||
}
|
||||
defer server.Close()
|
||||
err = prometheus.Register(server)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to register XDP STUN server as a prometheus collector: %v", err)
|
||||
}
|
||||
log.Println("XDP STUN server started")
|
||||
|
||||
mux := http.NewServeMux()
|
||||
tsweb.Debugger(mux)
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
err := http.ListenAndServe(*flagHTTP, mux)
|
||||
errCh <- err
|
||||
}()
|
||||
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
log.Printf("HTTP serve err: %v", err)
|
||||
case sig := <-sigCh:
|
||||
log.Printf("received signal: %s", sig)
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,166 @@
|
||||
// Code generated by bpf2go; DO NOT EDIT.
|
||||
//go:build mips || mips64 || ppc64 || s390x
|
||||
|
||||
package xdp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
)
|
||||
|
||||
type bpfConfig struct{ DstPort uint16 }
|
||||
|
||||
type bpfCounterKeyAf uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_UNKNOWN bpfCounterKeyAf = 0
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_IPV4 bpfCounterKeyAf = 1
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_IPV6 bpfCounterKeyAf = 2
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_LEN bpfCounterKeyAf = 3
|
||||
)
|
||||
|
||||
type bpfCounterKeyPacketsBytesAction uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL bpfCounterKeyPacketsBytesAction = 0
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL bpfCounterKeyPacketsBytesAction = 1
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL bpfCounterKeyPacketsBytesAction = 2
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL bpfCounterKeyPacketsBytesAction = 3
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL bpfCounterKeyPacketsBytesAction = 4
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL bpfCounterKeyPacketsBytesAction = 5
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL bpfCounterKeyPacketsBytesAction = 6
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL bpfCounterKeyPacketsBytesAction = 7
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN bpfCounterKeyPacketsBytesAction = 8
|
||||
)
|
||||
|
||||
type bpfCounterKeyProgEnd uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED bpfCounterKeyProgEnd = 0
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR bpfCounterKeyProgEnd = 1
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM bpfCounterKeyProgEnd = 2
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM bpfCounterKeyProgEnd = 3
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_NOT_STUN_PORT bpfCounterKeyProgEnd = 4
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL bpfCounterKeyProgEnd = 5
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_LEN bpfCounterKeyProgEnd = 6
|
||||
)
|
||||
|
||||
type bpfCountersKey struct {
|
||||
Unused uint8
|
||||
Af uint8
|
||||
Pba uint8
|
||||
ProgEnd uint8
|
||||
}
|
||||
|
||||
// loadBpf returns the embedded CollectionSpec for bpf.
|
||||
func loadBpf() (*ebpf.CollectionSpec, error) {
|
||||
reader := bytes.NewReader(_BpfBytes)
|
||||
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't load bpf: %w", err)
|
||||
}
|
||||
|
||||
return spec, err
|
||||
}
|
||||
|
||||
// loadBpfObjects loads bpf and converts it into a struct.
|
||||
//
|
||||
// The following types are suitable as obj argument:
|
||||
//
|
||||
// *bpfObjects
|
||||
// *bpfPrograms
|
||||
// *bpfMaps
|
||||
//
|
||||
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
|
||||
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
spec, err := loadBpf()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return spec.LoadAndAssign(obj, opts)
|
||||
}
|
||||
|
||||
// bpfSpecs contains maps and programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfSpecs struct {
|
||||
bpfProgramSpecs
|
||||
bpfMapSpecs
|
||||
}
|
||||
|
||||
// bpfSpecs contains programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfProgramSpecs struct {
|
||||
XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"`
|
||||
}
|
||||
|
||||
// bpfMapSpecs contains maps before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfMapSpecs struct {
|
||||
ConfigMap *ebpf.MapSpec `ebpf:"config_map"`
|
||||
CountersMap *ebpf.MapSpec `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
// bpfObjects contains all objects after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfObjects struct {
|
||||
bpfPrograms
|
||||
bpfMaps
|
||||
}
|
||||
|
||||
func (o *bpfObjects) Close() error {
|
||||
return _BpfClose(
|
||||
&o.bpfPrograms,
|
||||
&o.bpfMaps,
|
||||
)
|
||||
}
|
||||
|
||||
// bpfMaps contains all maps after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfMaps struct {
|
||||
ConfigMap *ebpf.Map `ebpf:"config_map"`
|
||||
CountersMap *ebpf.Map `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
func (m *bpfMaps) Close() error {
|
||||
return _BpfClose(
|
||||
m.ConfigMap,
|
||||
m.CountersMap,
|
||||
)
|
||||
}
|
||||
|
||||
// bpfPrograms contains all programs after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfPrograms struct {
|
||||
XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"`
|
||||
}
|
||||
|
||||
func (p *bpfPrograms) Close() error {
|
||||
return _BpfClose(
|
||||
p.XdpProgFunc,
|
||||
)
|
||||
}
|
||||
|
||||
func _BpfClose(closers ...io.Closer) error {
|
||||
for _, closer := range closers {
|
||||
if err := closer.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Do not access this directly.
|
||||
//
|
||||
//go:embed bpf_bpfeb.o
|
||||
var _BpfBytes []byte
|
||||
Binary file not shown.
@ -0,0 +1,166 @@
|
||||
// Code generated by bpf2go; DO NOT EDIT.
|
||||
//go:build 386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64
|
||||
|
||||
package xdp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
)
|
||||
|
||||
type bpfConfig struct{ DstPort uint16 }
|
||||
|
||||
type bpfCounterKeyAf uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_UNKNOWN bpfCounterKeyAf = 0
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_IPV4 bpfCounterKeyAf = 1
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_IPV6 bpfCounterKeyAf = 2
|
||||
bpfCounterKeyAfCOUNTER_KEY_AF_LEN bpfCounterKeyAf = 3
|
||||
)
|
||||
|
||||
type bpfCounterKeyPacketsBytesAction uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL bpfCounterKeyPacketsBytesAction = 0
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL bpfCounterKeyPacketsBytesAction = 1
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL bpfCounterKeyPacketsBytesAction = 2
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL bpfCounterKeyPacketsBytesAction = 3
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL bpfCounterKeyPacketsBytesAction = 4
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL bpfCounterKeyPacketsBytesAction = 5
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL bpfCounterKeyPacketsBytesAction = 6
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL bpfCounterKeyPacketsBytesAction = 7
|
||||
bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN bpfCounterKeyPacketsBytesAction = 8
|
||||
)
|
||||
|
||||
type bpfCounterKeyProgEnd uint32
|
||||
|
||||
const (
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED bpfCounterKeyProgEnd = 0
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR bpfCounterKeyProgEnd = 1
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM bpfCounterKeyProgEnd = 2
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM bpfCounterKeyProgEnd = 3
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_NOT_STUN_PORT bpfCounterKeyProgEnd = 4
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL bpfCounterKeyProgEnd = 5
|
||||
bpfCounterKeyProgEndCOUNTER_KEY_END_LEN bpfCounterKeyProgEnd = 6
|
||||
)
|
||||
|
||||
type bpfCountersKey struct {
|
||||
Unused uint8
|
||||
Af uint8
|
||||
Pba uint8
|
||||
ProgEnd uint8
|
||||
}
|
||||
|
||||
// loadBpf returns the embedded CollectionSpec for bpf.
|
||||
func loadBpf() (*ebpf.CollectionSpec, error) {
|
||||
reader := bytes.NewReader(_BpfBytes)
|
||||
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't load bpf: %w", err)
|
||||
}
|
||||
|
||||
return spec, err
|
||||
}
|
||||
|
||||
// loadBpfObjects loads bpf and converts it into a struct.
|
||||
//
|
||||
// The following types are suitable as obj argument:
|
||||
//
|
||||
// *bpfObjects
|
||||
// *bpfPrograms
|
||||
// *bpfMaps
|
||||
//
|
||||
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
|
||||
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
spec, err := loadBpf()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return spec.LoadAndAssign(obj, opts)
|
||||
}
|
||||
|
||||
// bpfSpecs contains maps and programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfSpecs struct {
|
||||
bpfProgramSpecs
|
||||
bpfMapSpecs
|
||||
}
|
||||
|
||||
// bpfSpecs contains programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfProgramSpecs struct {
|
||||
XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"`
|
||||
}
|
||||
|
||||
// bpfMapSpecs contains maps before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfMapSpecs struct {
|
||||
ConfigMap *ebpf.MapSpec `ebpf:"config_map"`
|
||||
CountersMap *ebpf.MapSpec `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
// bpfObjects contains all objects after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfObjects struct {
|
||||
bpfPrograms
|
||||
bpfMaps
|
||||
}
|
||||
|
||||
func (o *bpfObjects) Close() error {
|
||||
return _BpfClose(
|
||||
&o.bpfPrograms,
|
||||
&o.bpfMaps,
|
||||
)
|
||||
}
|
||||
|
||||
// bpfMaps contains all maps after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfMaps struct {
|
||||
ConfigMap *ebpf.Map `ebpf:"config_map"`
|
||||
CountersMap *ebpf.Map `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
func (m *bpfMaps) Close() error {
|
||||
return _BpfClose(
|
||||
m.ConfigMap,
|
||||
m.CountersMap,
|
||||
)
|
||||
}
|
||||
|
||||
// bpfPrograms contains all programs after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfPrograms struct {
|
||||
XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"`
|
||||
}
|
||||
|
||||
func (p *bpfPrograms) Close() error {
|
||||
return _BpfClose(
|
||||
p.XdpProgFunc,
|
||||
)
|
||||
}
|
||||
|
||||
func _BpfClose(closers ...io.Closer) error {
|
||||
for _, closer := range closers {
|
||||
if err := closer.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Do not access this directly.
|
||||
//
|
||||
//go:embed bpf_bpfel.o
|
||||
var _BpfBytes []byte
|
||||
Binary file not shown.
@ -0,0 +1,32 @@
|
||||
Valid-License-Identifier: BSD-2-Clause
|
||||
SPDX-URL: https://spdx.org/licenses/BSD-2-Clause.html
|
||||
Usage-Guide:
|
||||
To use the BSD 2-clause "Simplified" License put the following SPDX
|
||||
tag/value pair into a comment according to the placement guidelines in
|
||||
the licensing rules documentation:
|
||||
SPDX-License-Identifier: BSD-2-Clause
|
||||
License-Text:
|
||||
|
||||
Copyright (c) 2015 The Libbpf Authors. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
@ -0,0 +1,99 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_ENDIAN__
|
||||
#define __BPF_ENDIAN__
|
||||
|
||||
/*
|
||||
* Isolate byte #n and put it into byte #m, for __u##b type.
|
||||
* E.g., moving byte #6 (nnnnnnnn) into byte #1 (mmmmmmmm) for __u64:
|
||||
* 1) xxxxxxxx nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx
|
||||
* 2) nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx 00000000
|
||||
* 3) 00000000 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn
|
||||
* 4) 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn 00000000
|
||||
*/
|
||||
#define ___bpf_mvb(x, b, n, m) ((__u##b)(x) << (b-(n+1)*8) >> (b-8) << (m*8))
|
||||
|
||||
#define ___bpf_swab16(x) ((__u16)( \
|
||||
___bpf_mvb(x, 16, 0, 1) | \
|
||||
___bpf_mvb(x, 16, 1, 0)))
|
||||
|
||||
#define ___bpf_swab32(x) ((__u32)( \
|
||||
___bpf_mvb(x, 32, 0, 3) | \
|
||||
___bpf_mvb(x, 32, 1, 2) | \
|
||||
___bpf_mvb(x, 32, 2, 1) | \
|
||||
___bpf_mvb(x, 32, 3, 0)))
|
||||
|
||||
#define ___bpf_swab64(x) ((__u64)( \
|
||||
___bpf_mvb(x, 64, 0, 7) | \
|
||||
___bpf_mvb(x, 64, 1, 6) | \
|
||||
___bpf_mvb(x, 64, 2, 5) | \
|
||||
___bpf_mvb(x, 64, 3, 4) | \
|
||||
___bpf_mvb(x, 64, 4, 3) | \
|
||||
___bpf_mvb(x, 64, 5, 2) | \
|
||||
___bpf_mvb(x, 64, 6, 1) | \
|
||||
___bpf_mvb(x, 64, 7, 0)))
|
||||
|
||||
/* LLVM's BPF target selects the endianness of the CPU
|
||||
* it compiles on, or the user specifies (bpfel/bpfeb),
|
||||
* respectively. The used __BYTE_ORDER__ is defined by
|
||||
* the compiler, we cannot rely on __BYTE_ORDER from
|
||||
* libc headers, since it doesn't reflect the actual
|
||||
* requested byte order.
|
||||
*
|
||||
* Note, LLVM's BPF target has different __builtin_bswapX()
|
||||
* semantics. It does map to BPF_ALU | BPF_END | BPF_TO_BE
|
||||
* in bpfel and bpfeb case, which means below, that we map
|
||||
* to cpu_to_be16(). We could use it unconditionally in BPF
|
||||
* case, but better not rely on it, so that this header here
|
||||
* can be used from application and BPF program side, which
|
||||
* use different targets.
|
||||
*/
|
||||
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||
# define __bpf_ntohs(x) __builtin_bswap16(x)
|
||||
# define __bpf_htons(x) __builtin_bswap16(x)
|
||||
# define __bpf_constant_ntohs(x) ___bpf_swab16(x)
|
||||
# define __bpf_constant_htons(x) ___bpf_swab16(x)
|
||||
# define __bpf_ntohl(x) __builtin_bswap32(x)
|
||||
# define __bpf_htonl(x) __builtin_bswap32(x)
|
||||
# define __bpf_constant_ntohl(x) ___bpf_swab32(x)
|
||||
# define __bpf_constant_htonl(x) ___bpf_swab32(x)
|
||||
# define __bpf_be64_to_cpu(x) __builtin_bswap64(x)
|
||||
# define __bpf_cpu_to_be64(x) __builtin_bswap64(x)
|
||||
# define __bpf_constant_be64_to_cpu(x) ___bpf_swab64(x)
|
||||
# define __bpf_constant_cpu_to_be64(x) ___bpf_swab64(x)
|
||||
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
# define __bpf_ntohs(x) (x)
|
||||
# define __bpf_htons(x) (x)
|
||||
# define __bpf_constant_ntohs(x) (x)
|
||||
# define __bpf_constant_htons(x) (x)
|
||||
# define __bpf_ntohl(x) (x)
|
||||
# define __bpf_htonl(x) (x)
|
||||
# define __bpf_constant_ntohl(x) (x)
|
||||
# define __bpf_constant_htonl(x) (x)
|
||||
# define __bpf_be64_to_cpu(x) (x)
|
||||
# define __bpf_cpu_to_be64(x) (x)
|
||||
# define __bpf_constant_be64_to_cpu(x) (x)
|
||||
# define __bpf_constant_cpu_to_be64(x) (x)
|
||||
#else
|
||||
# error "Fix your compiler's __BYTE_ORDER__?!"
|
||||
#endif
|
||||
|
||||
#define bpf_htons(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_htons(x) : __bpf_htons(x))
|
||||
#define bpf_ntohs(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_ntohs(x) : __bpf_ntohs(x))
|
||||
#define bpf_htonl(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_htonl(x) : __bpf_htonl(x))
|
||||
#define bpf_ntohl(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_ntohl(x) : __bpf_ntohl(x))
|
||||
#define bpf_cpu_to_be64(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_cpu_to_be64(x) : __bpf_cpu_to_be64(x))
|
||||
#define bpf_be64_to_cpu(x) \
|
||||
(__builtin_constant_p(x) ? \
|
||||
__bpf_constant_be64_to_cpu(x) : __bpf_be64_to_cpu(x))
|
||||
|
||||
#endif /* __BPF_ENDIAN__ */
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,410 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_HELPERS__
|
||||
#define __BPF_HELPERS__
|
||||
|
||||
/*
|
||||
* Note that bpf programs need to include either
|
||||
* vmlinux.h (auto-generated from BTF) or linux/types.h
|
||||
* in advance since bpf_helper_defs.h uses such types
|
||||
* as __u64.
|
||||
*/
|
||||
#include "bpf_helper_defs.h"
|
||||
|
||||
#define __uint(name, val) int (*name)[val]
|
||||
#define __type(name, val) typeof(val) *name
|
||||
#define __array(name, val) typeof(val) *name[]
|
||||
#define __ulong(name, val) enum { ___bpf_concat(__unique_value, __COUNTER__) = val } name
|
||||
|
||||
/*
|
||||
* Helper macro to place programs, maps, license in
|
||||
* different sections in elf_bpf file. Section names
|
||||
* are interpreted by libbpf depending on the context (BPF programs, BPF maps,
|
||||
* extern variables, etc).
|
||||
* To allow use of SEC() with externs (e.g., for extern .maps declarations),
|
||||
* make sure __attribute__((unused)) doesn't trigger compilation warning.
|
||||
*/
|
||||
#if __GNUC__ && !__clang__
|
||||
|
||||
/*
|
||||
* Pragma macros are broken on GCC
|
||||
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578
|
||||
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90400
|
||||
*/
|
||||
#define SEC(name) __attribute__((section(name), used))
|
||||
|
||||
#else
|
||||
|
||||
#define SEC(name) \
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wignored-attributes\"") \
|
||||
__attribute__((section(name), used)) \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
|
||||
#endif
|
||||
|
||||
/* Avoid 'linux/stddef.h' definition of '__always_inline'. */
|
||||
#undef __always_inline
|
||||
#define __always_inline inline __attribute__((always_inline))
|
||||
|
||||
#ifndef __noinline
|
||||
#define __noinline __attribute__((noinline))
|
||||
#endif
|
||||
#ifndef __weak
|
||||
#define __weak __attribute__((weak))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Use __hidden attribute to mark a non-static BPF subprogram effectively
|
||||
* static for BPF verifier's verification algorithm purposes, allowing more
|
||||
* extensive and permissive BPF verification process, taking into account
|
||||
* subprogram's caller context.
|
||||
*/
|
||||
#define __hidden __attribute__((visibility("hidden")))
|
||||
|
||||
/* When utilizing vmlinux.h with BPF CO-RE, user BPF programs can't include
|
||||
* any system-level headers (such as stddef.h, linux/version.h, etc), and
|
||||
* commonly-used macros like NULL and KERNEL_VERSION aren't available through
|
||||
* vmlinux.h. This just adds unnecessary hurdles and forces users to re-define
|
||||
* them on their own. So as a convenience, provide such definitions here.
|
||||
*/
|
||||
#ifndef NULL
|
||||
#define NULL ((void *)0)
|
||||
#endif
|
||||
|
||||
#ifndef KERNEL_VERSION
|
||||
#define KERNEL_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Helper macros to manipulate data structures
|
||||
*/
|
||||
|
||||
/* offsetof() definition that uses __builtin_offset() might not preserve field
|
||||
* offset CO-RE relocation properly, so force-redefine offsetof() using
|
||||
* old-school approach which works with CO-RE correctly
|
||||
*/
|
||||
#undef offsetof
|
||||
#define offsetof(type, member) ((unsigned long)&((type *)0)->member)
|
||||
|
||||
/* redefined container_of() to ensure we use the above offsetof() macro */
|
||||
#undef container_of
|
||||
#define container_of(ptr, type, member) \
|
||||
({ \
|
||||
void *__mptr = (void *)(ptr); \
|
||||
((type *)(__mptr - offsetof(type, member))); \
|
||||
})
|
||||
|
||||
/*
|
||||
* Compiler (optimization) barrier.
|
||||
*/
|
||||
#ifndef barrier
|
||||
#define barrier() asm volatile("" ::: "memory")
|
||||
#endif
|
||||
|
||||
/* Variable-specific compiler (optimization) barrier. It's a no-op which makes
|
||||
* compiler believe that there is some black box modification of a given
|
||||
* variable and thus prevents compiler from making extra assumption about its
|
||||
* value and potential simplifications and optimizations on this variable.
|
||||
*
|
||||
* E.g., compiler might often delay or even omit 32-bit to 64-bit casting of
|
||||
* a variable, making some code patterns unverifiable. Putting barrier_var()
|
||||
* in place will ensure that cast is performed before the barrier_var()
|
||||
* invocation, because compiler has to pessimistically assume that embedded
|
||||
* asm section might perform some extra operations on that variable.
|
||||
*
|
||||
* This is a variable-specific variant of more global barrier().
|
||||
*/
|
||||
#ifndef barrier_var
|
||||
#define barrier_var(var) asm volatile("" : "+r"(var))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Helper macro to throw a compilation error if __bpf_unreachable() gets
|
||||
* built into the resulting code. This works given BPF back end does not
|
||||
* implement __builtin_trap(). This is useful to assert that certain paths
|
||||
* of the program code are never used and hence eliminated by the compiler.
|
||||
*
|
||||
* For example, consider a switch statement that covers known cases used by
|
||||
* the program. __bpf_unreachable() can then reside in the default case. If
|
||||
* the program gets extended such that a case is not covered in the switch
|
||||
* statement, then it will throw a build error due to the default case not
|
||||
* being compiled out.
|
||||
*/
|
||||
#ifndef __bpf_unreachable
|
||||
# define __bpf_unreachable() __builtin_trap()
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Helper function to perform a tail call with a constant/immediate map slot.
|
||||
*/
|
||||
#if __clang_major__ >= 8 && defined(__bpf__)
|
||||
static __always_inline void
|
||||
bpf_tail_call_static(void *ctx, const void *map, const __u32 slot)
|
||||
{
|
||||
if (!__builtin_constant_p(slot))
|
||||
__bpf_unreachable();
|
||||
|
||||
/*
|
||||
* Provide a hard guarantee that LLVM won't optimize setting r2 (map
|
||||
* pointer) and r3 (constant map index) from _different paths_ ending
|
||||
* up at the _same_ call insn as otherwise we won't be able to use the
|
||||
* jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel
|
||||
* given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key
|
||||
* tracking for prog array pokes") for details on verifier tracking.
|
||||
*
|
||||
* Note on clobber list: we need to stay in-line with BPF calling
|
||||
* convention, so even if we don't end up using r0, r4, r5, we need
|
||||
* to mark them as clobber so that LLVM doesn't end up using them
|
||||
* before / after the call.
|
||||
*/
|
||||
asm volatile("r1 = %[ctx]\n\t"
|
||||
"r2 = %[map]\n\t"
|
||||
"r3 = %[slot]\n\t"
|
||||
"call 12"
|
||||
:: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot)
|
||||
: "r0", "r1", "r2", "r3", "r4", "r5");
|
||||
}
|
||||
#endif
|
||||
|
||||
enum libbpf_pin_type {
|
||||
LIBBPF_PIN_NONE,
|
||||
/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
|
||||
LIBBPF_PIN_BY_NAME,
|
||||
};
|
||||
|
||||
enum libbpf_tristate {
|
||||
TRI_NO = 0,
|
||||
TRI_YES = 1,
|
||||
TRI_MODULE = 2,
|
||||
};
|
||||
|
||||
#define __kconfig __attribute__((section(".kconfig")))
|
||||
#define __ksym __attribute__((section(".ksyms")))
|
||||
#define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted")))
|
||||
#define __kptr __attribute__((btf_type_tag("kptr")))
|
||||
#define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr")))
|
||||
|
||||
#define bpf_ksym_exists(sym) ({ \
|
||||
_Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak"); \
|
||||
!!sym; \
|
||||
})
|
||||
|
||||
#define __arg_ctx __attribute__((btf_decl_tag("arg:ctx")))
|
||||
#define __arg_nonnull __attribute((btf_decl_tag("arg:nonnull")))
|
||||
#define __arg_nullable __attribute((btf_decl_tag("arg:nullable")))
|
||||
#define __arg_trusted __attribute((btf_decl_tag("arg:trusted")))
|
||||
#define __arg_arena __attribute((btf_decl_tag("arg:arena")))
|
||||
|
||||
#ifndef ___bpf_concat
|
||||
#define ___bpf_concat(a, b) a ## b
|
||||
#endif
|
||||
#ifndef ___bpf_apply
|
||||
#define ___bpf_apply(fn, n) ___bpf_concat(fn, n)
|
||||
#endif
|
||||
#ifndef ___bpf_nth
|
||||
#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N
|
||||
#endif
|
||||
#ifndef ___bpf_narg
|
||||
#define ___bpf_narg(...) \
|
||||
___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
|
||||
#endif
|
||||
|
||||
#define ___bpf_fill0(arr, p, x) do {} while (0)
|
||||
#define ___bpf_fill1(arr, p, x) arr[p] = x
|
||||
#define ___bpf_fill2(arr, p, x, args...) arr[p] = x; ___bpf_fill1(arr, p + 1, args)
|
||||
#define ___bpf_fill3(arr, p, x, args...) arr[p] = x; ___bpf_fill2(arr, p + 1, args)
|
||||
#define ___bpf_fill4(arr, p, x, args...) arr[p] = x; ___bpf_fill3(arr, p + 1, args)
|
||||
#define ___bpf_fill5(arr, p, x, args...) arr[p] = x; ___bpf_fill4(arr, p + 1, args)
|
||||
#define ___bpf_fill6(arr, p, x, args...) arr[p] = x; ___bpf_fill5(arr, p + 1, args)
|
||||
#define ___bpf_fill7(arr, p, x, args...) arr[p] = x; ___bpf_fill6(arr, p + 1, args)
|
||||
#define ___bpf_fill8(arr, p, x, args...) arr[p] = x; ___bpf_fill7(arr, p + 1, args)
|
||||
#define ___bpf_fill9(arr, p, x, args...) arr[p] = x; ___bpf_fill8(arr, p + 1, args)
|
||||
#define ___bpf_fill10(arr, p, x, args...) arr[p] = x; ___bpf_fill9(arr, p + 1, args)
|
||||
#define ___bpf_fill11(arr, p, x, args...) arr[p] = x; ___bpf_fill10(arr, p + 1, args)
|
||||
#define ___bpf_fill12(arr, p, x, args...) arr[p] = x; ___bpf_fill11(arr, p + 1, args)
|
||||
#define ___bpf_fill(arr, args...) \
|
||||
___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args)
|
||||
|
||||
/*
|
||||
* BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values
|
||||
* in a structure.
|
||||
*/
|
||||
#define BPF_SEQ_PRINTF(seq, fmt, args...) \
|
||||
({ \
|
||||
static const char ___fmt[] = fmt; \
|
||||
unsigned long long ___param[___bpf_narg(args)]; \
|
||||
\
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
___bpf_fill(___param, args); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
\
|
||||
bpf_seq_printf(seq, ___fmt, sizeof(___fmt), \
|
||||
___param, sizeof(___param)); \
|
||||
})
|
||||
|
||||
/*
|
||||
* BPF_SNPRINTF wraps the bpf_snprintf helper with variadic arguments instead of
|
||||
* an array of u64.
|
||||
*/
|
||||
#define BPF_SNPRINTF(out, out_size, fmt, args...) \
|
||||
({ \
|
||||
static const char ___fmt[] = fmt; \
|
||||
unsigned long long ___param[___bpf_narg(args)]; \
|
||||
\
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
___bpf_fill(___param, args); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
\
|
||||
bpf_snprintf(out, out_size, ___fmt, \
|
||||
___param, sizeof(___param)); \
|
||||
})
|
||||
|
||||
#ifdef BPF_NO_GLOBAL_DATA
|
||||
#define BPF_PRINTK_FMT_MOD
|
||||
#else
|
||||
#define BPF_PRINTK_FMT_MOD static const
|
||||
#endif
|
||||
|
||||
#define __bpf_printk(fmt, ...) \
|
||||
({ \
|
||||
BPF_PRINTK_FMT_MOD char ____fmt[] = fmt; \
|
||||
bpf_trace_printk(____fmt, sizeof(____fmt), \
|
||||
##__VA_ARGS__); \
|
||||
})
|
||||
|
||||
/*
|
||||
* __bpf_vprintk wraps the bpf_trace_vprintk helper with variadic arguments
|
||||
* instead of an array of u64.
|
||||
*/
|
||||
#define __bpf_vprintk(fmt, args...) \
|
||||
({ \
|
||||
static const char ___fmt[] = fmt; \
|
||||
unsigned long long ___param[___bpf_narg(args)]; \
|
||||
\
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
|
||||
___bpf_fill(___param, args); \
|
||||
_Pragma("GCC diagnostic pop") \
|
||||
\
|
||||
bpf_trace_vprintk(___fmt, sizeof(___fmt), \
|
||||
___param, sizeof(___param)); \
|
||||
})
|
||||
|
||||
/* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args
|
||||
* Otherwise use __bpf_vprintk
|
||||
*/
|
||||
#define ___bpf_pick_printk(...) \
|
||||
___bpf_nth(_, ##__VA_ARGS__, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \
|
||||
__bpf_vprintk, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \
|
||||
__bpf_vprintk, __bpf_vprintk, __bpf_printk /*3*/, __bpf_printk /*2*/,\
|
||||
__bpf_printk /*1*/, __bpf_printk /*0*/)
|
||||
|
||||
/* Helper macro to print out debug messages */
|
||||
#define bpf_printk(fmt, args...) ___bpf_pick_printk(args)(fmt, ##args)
|
||||
|
||||
struct bpf_iter_num;
|
||||
|
||||
extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __weak __ksym;
|
||||
extern int *bpf_iter_num_next(struct bpf_iter_num *it) __weak __ksym;
|
||||
extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __weak __ksym;
|
||||
|
||||
#ifndef bpf_for_each
|
||||
/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for
|
||||
* using BPF open-coded iterators without having to write mundane explicit
|
||||
* low-level loop logic. Instead, it provides for()-like generic construct
|
||||
* that can be used pretty naturally. E.g., for some hypothetical cgroup
|
||||
* iterator, you'd write:
|
||||
*
|
||||
* struct cgroup *cg, *parent_cg = <...>;
|
||||
*
|
||||
* bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) {
|
||||
* bpf_printk("Child cgroup id = %d", cg->cgroup_id);
|
||||
* if (cg->cgroup_id == 123)
|
||||
* break;
|
||||
* }
|
||||
*
|
||||
* I.e., it looks almost like high-level for each loop in other languages,
|
||||
* supports continue/break, and is verifiable by BPF verifier.
|
||||
*
|
||||
* For iterating integers, the difference betwen bpf_for_each(num, i, N, M)
|
||||
* and bpf_for(i, N, M) is in that bpf_for() provides additional proof to
|
||||
* verifier that i is in [N, M) range, and in bpf_for_each() case i is `int
|
||||
* *`, not just `int`. So for integers bpf_for() is more convenient.
|
||||
*
|
||||
* Note: this macro relies on C99 feature of allowing to declare variables
|
||||
* inside for() loop, bound to for() loop lifetime. It also utilizes GCC
|
||||
* extension: __attribute__((cleanup(<func>))), supported by both GCC and
|
||||
* Clang.
|
||||
*/
|
||||
#define bpf_for_each(type, cur, args...) for ( \
|
||||
/* initialize and define destructor */ \
|
||||
struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \
|
||||
cleanup(bpf_iter_##type##_destroy))), \
|
||||
/* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \
|
||||
*___p __attribute__((unused)) = ( \
|
||||
bpf_iter_##type##_new(&___it, ##args), \
|
||||
/* this is a workaround for Clang bug: it currently doesn't emit BTF */ \
|
||||
/* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \
|
||||
(void)bpf_iter_##type##_destroy, (void *)0); \
|
||||
/* iteration and termination check */ \
|
||||
(((cur) = bpf_iter_##type##_next(&___it))); \
|
||||
)
|
||||
#endif /* bpf_for_each */
|
||||
|
||||
#ifndef bpf_for
|
||||
/* bpf_for(i, start, end) implements a for()-like looping construct that sets
|
||||
* provided integer variable *i* to values starting from *start* through,
|
||||
* but not including, *end*. It also proves to BPF verifier that *i* belongs
|
||||
* to range [start, end), so this can be used for accessing arrays without
|
||||
* extra checks.
|
||||
*
|
||||
* Note: *start* and *end* are assumed to be expressions with no side effects
|
||||
* and whose values do not change throughout bpf_for() loop execution. They do
|
||||
* not have to be statically known or constant, though.
|
||||
*
|
||||
* Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()
|
||||
* loop bound variables and cleanup attribute, supported by GCC and Clang.
|
||||
*/
|
||||
#define bpf_for(i, start, end) for ( \
|
||||
/* initialize and define destructor */ \
|
||||
struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \
|
||||
cleanup(bpf_iter_num_destroy))), \
|
||||
/* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \
|
||||
*___p __attribute__((unused)) = ( \
|
||||
bpf_iter_num_new(&___it, (start), (end)), \
|
||||
/* this is a workaround for Clang bug: it currently doesn't emit BTF */ \
|
||||
/* for bpf_iter_num_destroy() when used from cleanup() attribute */ \
|
||||
(void)bpf_iter_num_destroy, (void *)0); \
|
||||
({ \
|
||||
/* iteration step */ \
|
||||
int *___t = bpf_iter_num_next(&___it); \
|
||||
/* termination and bounds check */ \
|
||||
(___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \
|
||||
}); \
|
||||
)
|
||||
#endif /* bpf_for */
|
||||
|
||||
#ifndef bpf_repeat
|
||||
/* bpf_repeat(N) performs N iterations without exposing iteration number
|
||||
*
|
||||
* Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()
|
||||
* loop bound variables and cleanup attribute, supported by GCC and Clang.
|
||||
*/
|
||||
#define bpf_repeat(N) for ( \
|
||||
/* initialize and define destructor */ \
|
||||
struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \
|
||||
cleanup(bpf_iter_num_destroy))), \
|
||||
/* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \
|
||||
*___p __attribute__((unused)) = ( \
|
||||
bpf_iter_num_new(&___it, 0, (N)), \
|
||||
/* this is a workaround for Clang bug: it currently doesn't emit BTF */ \
|
||||
/* for bpf_iter_num_destroy() when used from cleanup() attribute */ \
|
||||
(void)bpf_iter_num_destroy, (void *)0); \
|
||||
bpf_iter_num_next(&___it); \
|
||||
/* nothing here */ \
|
||||
)
|
||||
#endif /* bpf_repeat */
|
||||
|
||||
#endif
|
||||
@ -0,0 +1,922 @@
|
||||
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
|
||||
#ifndef __BPF_TRACING_H__
|
||||
#define __BPF_TRACING_H__
|
||||
|
||||
#include "bpf_helpers.h"
|
||||
|
||||
/* Scan the ARCH passed in from ARCH env variable (see Makefile) */
|
||||
#if defined(__TARGET_ARCH_x86)
|
||||
#define bpf_target_x86
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_s390)
|
||||
#define bpf_target_s390
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_arm)
|
||||
#define bpf_target_arm
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_arm64)
|
||||
#define bpf_target_arm64
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_mips)
|
||||
#define bpf_target_mips
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_powerpc)
|
||||
#define bpf_target_powerpc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_sparc)
|
||||
#define bpf_target_sparc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_riscv)
|
||||
#define bpf_target_riscv
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_arc)
|
||||
#define bpf_target_arc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__TARGET_ARCH_loongarch)
|
||||
#define bpf_target_loongarch
|
||||
#define bpf_target_defined
|
||||
#else
|
||||
|
||||
/* Fall back to what the compiler says */
|
||||
#if defined(__x86_64__)
|
||||
#define bpf_target_x86
|
||||
#define bpf_target_defined
|
||||
#elif defined(__s390__)
|
||||
#define bpf_target_s390
|
||||
#define bpf_target_defined
|
||||
#elif defined(__arm__)
|
||||
#define bpf_target_arm
|
||||
#define bpf_target_defined
|
||||
#elif defined(__aarch64__)
|
||||
#define bpf_target_arm64
|
||||
#define bpf_target_defined
|
||||
#elif defined(__mips__)
|
||||
#define bpf_target_mips
|
||||
#define bpf_target_defined
|
||||
#elif defined(__powerpc__)
|
||||
#define bpf_target_powerpc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__sparc__)
|
||||
#define bpf_target_sparc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__riscv) && __riscv_xlen == 64
|
||||
#define bpf_target_riscv
|
||||
#define bpf_target_defined
|
||||
#elif defined(__arc__)
|
||||
#define bpf_target_arc
|
||||
#define bpf_target_defined
|
||||
#elif defined(__loongarch__)
|
||||
#define bpf_target_loongarch
|
||||
#define bpf_target_defined
|
||||
#endif /* no compiler target */
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef __BPF_TARGET_MISSING
|
||||
#define __BPF_TARGET_MISSING "GCC error \"Must specify a BPF target arch via __TARGET_ARCH_xxx\""
|
||||
#endif
|
||||
|
||||
#if defined(bpf_target_x86)
|
||||
|
||||
/*
|
||||
* https://en.wikipedia.org/wiki/X86_calling_conventions#System_V_AMD64_ABI
|
||||
*/
|
||||
|
||||
#if defined(__KERNEL__) || defined(__VMLINUX_H__)
|
||||
|
||||
#define __PT_PARM1_REG di
|
||||
#define __PT_PARM2_REG si
|
||||
#define __PT_PARM3_REG dx
|
||||
#define __PT_PARM4_REG cx
|
||||
#define __PT_PARM5_REG r8
|
||||
#define __PT_PARM6_REG r9
|
||||
/*
|
||||
* Syscall uses r10 for PARM4. See arch/x86/entry/entry_64.S:entry_SYSCALL_64
|
||||
* comments in Linux sources. And refer to syscall(2) manpage.
|
||||
*/
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG r10
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG sp
|
||||
#define __PT_FP_REG bp
|
||||
#define __PT_RC_REG ax
|
||||
#define __PT_SP_REG sp
|
||||
#define __PT_IP_REG ip
|
||||
|
||||
#else
|
||||
|
||||
#ifdef __i386__
|
||||
|
||||
/* i386 kernel is built with -mregparm=3 */
|
||||
#define __PT_PARM1_REG eax
|
||||
#define __PT_PARM2_REG edx
|
||||
#define __PT_PARM3_REG ecx
|
||||
/* i386 syscall ABI is very different, refer to syscall(2) manpage */
|
||||
#define __PT_PARM1_SYSCALL_REG ebx
|
||||
#define __PT_PARM2_SYSCALL_REG ecx
|
||||
#define __PT_PARM3_SYSCALL_REG edx
|
||||
#define __PT_PARM4_SYSCALL_REG esi
|
||||
#define __PT_PARM5_SYSCALL_REG edi
|
||||
#define __PT_PARM6_SYSCALL_REG ebp
|
||||
|
||||
#define __PT_RET_REG esp
|
||||
#define __PT_FP_REG ebp
|
||||
#define __PT_RC_REG eax
|
||||
#define __PT_SP_REG esp
|
||||
#define __PT_IP_REG eip
|
||||
|
||||
#else /* __i386__ */
|
||||
|
||||
#define __PT_PARM1_REG rdi
|
||||
#define __PT_PARM2_REG rsi
|
||||
#define __PT_PARM3_REG rdx
|
||||
#define __PT_PARM4_REG rcx
|
||||
#define __PT_PARM5_REG r8
|
||||
#define __PT_PARM6_REG r9
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG r10
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG rsp
|
||||
#define __PT_FP_REG rbp
|
||||
#define __PT_RC_REG rax
|
||||
#define __PT_SP_REG rsp
|
||||
#define __PT_IP_REG rip
|
||||
|
||||
#endif /* __i386__ */
|
||||
|
||||
#endif /* __KERNEL__ || __VMLINUX_H__ */
|
||||
|
||||
#elif defined(bpf_target_s390)
|
||||
|
||||
/*
|
||||
* https://github.com/IBM/s390x-abi/releases/download/v1.6/lzsabi_s390x.pdf
|
||||
*/
|
||||
|
||||
struct pt_regs___s390 {
|
||||
unsigned long orig_gpr2;
|
||||
};
|
||||
|
||||
/* s390 provides user_pt_regs instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const user_pt_regs *)(x))
|
||||
#define __PT_PARM1_REG gprs[2]
|
||||
#define __PT_PARM2_REG gprs[3]
|
||||
#define __PT_PARM3_REG gprs[4]
|
||||
#define __PT_PARM4_REG gprs[5]
|
||||
#define __PT_PARM5_REG gprs[6]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG orig_gpr2
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG gprs[7]
|
||||
#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x)
|
||||
#define PT_REGS_PARM1_CORE_SYSCALL(x) \
|
||||
BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG)
|
||||
|
||||
#define __PT_RET_REG gprs[14]
|
||||
#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define __PT_RC_REG gprs[2]
|
||||
#define __PT_SP_REG gprs[15]
|
||||
#define __PT_IP_REG psw.addr
|
||||
|
||||
#elif defined(bpf_target_arm)
|
||||
|
||||
/*
|
||||
* https://github.com/ARM-software/abi-aa/blob/main/aapcs32/aapcs32.rst#machine-registers
|
||||
*/
|
||||
|
||||
#define __PT_PARM1_REG uregs[0]
|
||||
#define __PT_PARM2_REG uregs[1]
|
||||
#define __PT_PARM3_REG uregs[2]
|
||||
#define __PT_PARM4_REG uregs[3]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG uregs[4]
|
||||
#define __PT_PARM6_SYSCALL_REG uregs[5]
|
||||
#define __PT_PARM7_SYSCALL_REG uregs[6]
|
||||
|
||||
#define __PT_RET_REG uregs[14]
|
||||
#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define __PT_RC_REG uregs[0]
|
||||
#define __PT_SP_REG uregs[13]
|
||||
#define __PT_IP_REG uregs[12]
|
||||
|
||||
#elif defined(bpf_target_arm64)
|
||||
|
||||
/*
|
||||
* https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#machine-registers
|
||||
*/
|
||||
|
||||
struct pt_regs___arm64 {
|
||||
unsigned long orig_x0;
|
||||
};
|
||||
|
||||
/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x))
|
||||
#define __PT_PARM1_REG regs[0]
|
||||
#define __PT_PARM2_REG regs[1]
|
||||
#define __PT_PARM3_REG regs[2]
|
||||
#define __PT_PARM4_REG regs[3]
|
||||
#define __PT_PARM5_REG regs[4]
|
||||
#define __PT_PARM6_REG regs[5]
|
||||
#define __PT_PARM7_REG regs[6]
|
||||
#define __PT_PARM8_REG regs[7]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG orig_x0
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x)
|
||||
#define PT_REGS_PARM1_CORE_SYSCALL(x) \
|
||||
BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG)
|
||||
|
||||
#define __PT_RET_REG regs[30]
|
||||
#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define __PT_RC_REG regs[0]
|
||||
#define __PT_SP_REG sp
|
||||
#define __PT_IP_REG pc
|
||||
|
||||
#elif defined(bpf_target_mips)
|
||||
|
||||
/*
|
||||
* N64 ABI is assumed right now.
|
||||
* https://en.wikipedia.org/wiki/MIPS_architecture#Calling_conventions
|
||||
*/
|
||||
|
||||
#define __PT_PARM1_REG regs[4]
|
||||
#define __PT_PARM2_REG regs[5]
|
||||
#define __PT_PARM3_REG regs[6]
|
||||
#define __PT_PARM4_REG regs[7]
|
||||
#define __PT_PARM5_REG regs[8]
|
||||
#define __PT_PARM6_REG regs[9]
|
||||
#define __PT_PARM7_REG regs[10]
|
||||
#define __PT_PARM8_REG regs[11]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG /* only N32/N64 */
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG /* only N32/N64 */
|
||||
|
||||
#define __PT_RET_REG regs[31]
|
||||
#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */
|
||||
#define __PT_RC_REG regs[2]
|
||||
#define __PT_SP_REG regs[29]
|
||||
#define __PT_IP_REG cp0_epc
|
||||
|
||||
#elif defined(bpf_target_powerpc)
|
||||
|
||||
/*
|
||||
* http://refspecs.linux-foundation.org/elf/elfspec_ppc.pdf (page 3-14,
|
||||
* section "Function Calling Sequence")
|
||||
*/
|
||||
|
||||
#define __PT_PARM1_REG gpr[3]
|
||||
#define __PT_PARM2_REG gpr[4]
|
||||
#define __PT_PARM3_REG gpr[5]
|
||||
#define __PT_PARM4_REG gpr[6]
|
||||
#define __PT_PARM5_REG gpr[7]
|
||||
#define __PT_PARM6_REG gpr[8]
|
||||
#define __PT_PARM7_REG gpr[9]
|
||||
#define __PT_PARM8_REG gpr[10]
|
||||
|
||||
/* powerpc does not select ARCH_HAS_SYSCALL_WRAPPER. */
|
||||
#define PT_REGS_SYSCALL_REGS(ctx) ctx
|
||||
#define __PT_PARM1_SYSCALL_REG orig_gpr3
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
#if !defined(__arch64__)
|
||||
#define __PT_PARM7_SYSCALL_REG __PT_PARM7_REG /* only powerpc (not powerpc64) */
|
||||
#endif
|
||||
|
||||
#define __PT_RET_REG regs[31]
|
||||
#define __PT_FP_REG __unsupported__
|
||||
#define __PT_RC_REG gpr[3]
|
||||
#define __PT_SP_REG sp
|
||||
#define __PT_IP_REG nip
|
||||
|
||||
#elif defined(bpf_target_sparc)
|
||||
|
||||
/*
|
||||
* https://en.wikipedia.org/wiki/Calling_convention#SPARC
|
||||
*/
|
||||
|
||||
#define __PT_PARM1_REG u_regs[UREG_I0]
|
||||
#define __PT_PARM2_REG u_regs[UREG_I1]
|
||||
#define __PT_PARM3_REG u_regs[UREG_I2]
|
||||
#define __PT_PARM4_REG u_regs[UREG_I3]
|
||||
#define __PT_PARM5_REG u_regs[UREG_I4]
|
||||
#define __PT_PARM6_REG u_regs[UREG_I5]
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG u_regs[UREG_I7]
|
||||
#define __PT_FP_REG __unsupported__
|
||||
#define __PT_RC_REG u_regs[UREG_I0]
|
||||
#define __PT_SP_REG u_regs[UREG_FP]
|
||||
/* Should this also be a bpf_target check for the sparc case? */
|
||||
#if defined(__arch64__)
|
||||
#define __PT_IP_REG tpc
|
||||
#else
|
||||
#define __PT_IP_REG pc
|
||||
#endif
|
||||
|
||||
#elif defined(bpf_target_riscv)
|
||||
|
||||
/*
|
||||
* https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#risc-v-calling-conventions
|
||||
*/
|
||||
|
||||
/* riscv provides struct user_regs_struct instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))
|
||||
#define __PT_PARM1_REG a0
|
||||
#define __PT_PARM2_REG a1
|
||||
#define __PT_PARM3_REG a2
|
||||
#define __PT_PARM4_REG a3
|
||||
#define __PT_PARM5_REG a4
|
||||
#define __PT_PARM6_REG a5
|
||||
#define __PT_PARM7_REG a6
|
||||
#define __PT_PARM8_REG a7
|
||||
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG ra
|
||||
#define __PT_FP_REG s0
|
||||
#define __PT_RC_REG a0
|
||||
#define __PT_SP_REG sp
|
||||
#define __PT_IP_REG pc
|
||||
|
||||
#elif defined(bpf_target_arc)
|
||||
|
||||
/*
|
||||
* Section "Function Calling Sequence" (page 24):
|
||||
* https://raw.githubusercontent.com/wiki/foss-for-synopsys-dwc-arc-processors/toolchain/files/ARCv2_ABI.pdf
|
||||
*/
|
||||
|
||||
/* arc provides struct user_regs_struct instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))
|
||||
#define __PT_PARM1_REG scratch.r0
|
||||
#define __PT_PARM2_REG scratch.r1
|
||||
#define __PT_PARM3_REG scratch.r2
|
||||
#define __PT_PARM4_REG scratch.r3
|
||||
#define __PT_PARM5_REG scratch.r4
|
||||
#define __PT_PARM6_REG scratch.r5
|
||||
#define __PT_PARM7_REG scratch.r6
|
||||
#define __PT_PARM8_REG scratch.r7
|
||||
|
||||
/* arc does not select ARCH_HAS_SYSCALL_WRAPPER. */
|
||||
#define PT_REGS_SYSCALL_REGS(ctx) ctx
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG scratch.blink
|
||||
#define __PT_FP_REG scratch.fp
|
||||
#define __PT_RC_REG scratch.r0
|
||||
#define __PT_SP_REG scratch.sp
|
||||
#define __PT_IP_REG scratch.ret
|
||||
|
||||
#elif defined(bpf_target_loongarch)
|
||||
|
||||
/*
|
||||
* https://docs.kernel.org/loongarch/introduction.html
|
||||
* https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html
|
||||
*/
|
||||
|
||||
/* loongarch provides struct user_pt_regs instead of struct pt_regs to userspace */
|
||||
#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x))
|
||||
#define __PT_PARM1_REG regs[4]
|
||||
#define __PT_PARM2_REG regs[5]
|
||||
#define __PT_PARM3_REG regs[6]
|
||||
#define __PT_PARM4_REG regs[7]
|
||||
#define __PT_PARM5_REG regs[8]
|
||||
#define __PT_PARM6_REG regs[9]
|
||||
#define __PT_PARM7_REG regs[10]
|
||||
#define __PT_PARM8_REG regs[11]
|
||||
|
||||
/* loongarch does not select ARCH_HAS_SYSCALL_WRAPPER. */
|
||||
#define PT_REGS_SYSCALL_REGS(ctx) ctx
|
||||
#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
|
||||
#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
|
||||
#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
|
||||
#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
|
||||
#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
|
||||
#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
|
||||
|
||||
#define __PT_RET_REG regs[1]
|
||||
#define __PT_FP_REG regs[22]
|
||||
#define __PT_RC_REG regs[4]
|
||||
#define __PT_SP_REG regs[3]
|
||||
#define __PT_IP_REG csr_era
|
||||
|
||||
#endif
|
||||
|
||||
#if defined(bpf_target_defined)
|
||||
|
||||
struct pt_regs;
|
||||
|
||||
/* allow some architectures to override `struct pt_regs` */
|
||||
#ifndef __PT_REGS_CAST
|
||||
#define __PT_REGS_CAST(x) (x)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Different architectures support different number of arguments passed
|
||||
* through registers. i386 supports just 3, some arches support up to 8.
|
||||
*/
|
||||
#ifndef __PT_PARM4_REG
|
||||
#define __PT_PARM4_REG __unsupported__
|
||||
#endif
|
||||
#ifndef __PT_PARM5_REG
|
||||
#define __PT_PARM5_REG __unsupported__
|
||||
#endif
|
||||
#ifndef __PT_PARM6_REG
|
||||
#define __PT_PARM6_REG __unsupported__
|
||||
#endif
|
||||
#ifndef __PT_PARM7_REG
|
||||
#define __PT_PARM7_REG __unsupported__
|
||||
#endif
|
||||
#ifndef __PT_PARM8_REG
|
||||
#define __PT_PARM8_REG __unsupported__
|
||||
#endif
|
||||
/*
|
||||
* Similarly, syscall-specific conventions might differ between function call
|
||||
* conventions within each architecutre. All supported architectures pass
|
||||
* either 6 or 7 syscall arguments in registers.
|
||||
*
|
||||
* See syscall(2) manpage for succinct table with information on each arch.
|
||||
*/
|
||||
#ifndef __PT_PARM7_SYSCALL_REG
|
||||
#define __PT_PARM7_SYSCALL_REG __unsupported__
|
||||
#endif
|
||||
|
||||
#define PT_REGS_PARM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG)
|
||||
#define PT_REGS_PARM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG)
|
||||
#define PT_REGS_PARM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG)
|
||||
#define PT_REGS_PARM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG)
|
||||
#define PT_REGS_PARM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG)
|
||||
#define PT_REGS_PARM6(x) (__PT_REGS_CAST(x)->__PT_PARM6_REG)
|
||||
#define PT_REGS_PARM7(x) (__PT_REGS_CAST(x)->__PT_PARM7_REG)
|
||||
#define PT_REGS_PARM8(x) (__PT_REGS_CAST(x)->__PT_PARM8_REG)
|
||||
#define PT_REGS_RET(x) (__PT_REGS_CAST(x)->__PT_RET_REG)
|
||||
#define PT_REGS_FP(x) (__PT_REGS_CAST(x)->__PT_FP_REG)
|
||||
#define PT_REGS_RC(x) (__PT_REGS_CAST(x)->__PT_RC_REG)
|
||||
#define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG)
|
||||
#define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG)
|
||||
|
||||
#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG)
|
||||
#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG)
|
||||
#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG)
|
||||
#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG)
|
||||
#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG)
|
||||
#define PT_REGS_PARM6_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_REG)
|
||||
#define PT_REGS_PARM7_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_REG)
|
||||
#define PT_REGS_PARM8_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM8_REG)
|
||||
#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG)
|
||||
#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG)
|
||||
#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG)
|
||||
#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG)
|
||||
#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG)
|
||||
|
||||
#if defined(bpf_target_powerpc)
|
||||
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; })
|
||||
#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
|
||||
|
||||
#elif defined(bpf_target_sparc)
|
||||
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); })
|
||||
#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
|
||||
|
||||
#else
|
||||
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) \
|
||||
({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); })
|
||||
#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \
|
||||
({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); })
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef PT_REGS_PARM1_SYSCALL
|
||||
#define PT_REGS_PARM1_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM1_SYSCALL_REG)
|
||||
#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM2_SYSCALL
|
||||
#define PT_REGS_PARM2_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM2_SYSCALL_REG)
|
||||
#define PT_REGS_PARM2_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM3_SYSCALL
|
||||
#define PT_REGS_PARM3_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM3_SYSCALL_REG)
|
||||
#define PT_REGS_PARM3_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM4_SYSCALL
|
||||
#define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_SYSCALL_REG)
|
||||
#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM5_SYSCALL
|
||||
#define PT_REGS_PARM5_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM5_SYSCALL_REG)
|
||||
#define PT_REGS_PARM5_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM6_SYSCALL
|
||||
#define PT_REGS_PARM6_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM6_SYSCALL_REG)
|
||||
#define PT_REGS_PARM6_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_SYSCALL_REG)
|
||||
#endif
|
||||
#ifndef PT_REGS_PARM7_SYSCALL
|
||||
#define PT_REGS_PARM7_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM7_SYSCALL_REG)
|
||||
#define PT_REGS_PARM7_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_SYSCALL_REG)
|
||||
#endif
|
||||
|
||||
#else /* defined(bpf_target_defined) */
|
||||
|
||||
#define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM3(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM4(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM5(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM6(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM7(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM8(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_RET(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_FP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_RC(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_SP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_IP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#define PT_REGS_PARM1_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM2_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM3_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM4_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM5_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM6_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM7_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM8_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_RET_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_FP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_RC_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_SP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_IP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM2_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM3_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM4_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM5_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM6_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM7_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#define PT_REGS_PARM1_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM2_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM3_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM4_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM5_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM6_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
#define PT_REGS_PARM7_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
|
||||
|
||||
#endif /* defined(bpf_target_defined) */
|
||||
|
||||
/*
|
||||
* When invoked from a syscall handler kprobe, returns a pointer to a
|
||||
* struct pt_regs containing syscall arguments and suitable for passing to
|
||||
* PT_REGS_PARMn_SYSCALL() and PT_REGS_PARMn_CORE_SYSCALL().
|
||||
*/
|
||||
#ifndef PT_REGS_SYSCALL_REGS
|
||||
/* By default, assume that the arch selects ARCH_HAS_SYSCALL_WRAPPER. */
|
||||
#define PT_REGS_SYSCALL_REGS(ctx) ((struct pt_regs *)PT_REGS_PARM1(ctx))
|
||||
#endif
|
||||
|
||||
#ifndef ___bpf_concat
|
||||
#define ___bpf_concat(a, b) a ## b
|
||||
#endif
|
||||