From 8d6793fd7047d1c9b59e939e7644edbb75b4790f Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 11 Apr 2022 18:25:35 -0700 Subject: [PATCH] wgengine/router,util/kmod: load & log xt_mark Attempt to load the xt_mark kernel module when it is not present. If the load fails, log error information. It may be tempting to promote this failure to an error once it has been in use for some time, so as to avoid reaching an error with the iptables invocation, however, there are conditions under which the two stages may disagree - this change adds more useful breadcrumbs. Example new output from tailscaled running under my WSL2: ``` router: ensure module xt_mark: "/usr/sbin/modprobe xt_mark" failed: exit status 1; modprobe: FATAL: Module xt_mark not found in directory /lib/modules/5.10.43.3-microsoft-standard-WSL2 ``` Background: There are two places to lookup modules, one is `/proc/modules` "old", the other is `/sys/module/` "new". There was query_modules(2) in linux <2.6, alas, it is gone. In a docker container in the default configuration, you would get /proc/modules and /sys/module/ both populated. lsmod may work file, modprobe will fail with EPERM at `finit_module()` for an unpriviliged container. In a priviliged container the load may *succeed*, if some conditions are met. This condition should be avoided, but the code landing in this change does not attempt to avoid this scenario as it is both difficult to detect, and has a very uncertain impact. In an nspawn container `/proc/modules` is populated, but `/sys/module` does not exist. Modern `lsmod` versions will fail to gather most module information, without sysfs being populated with module information. In WSL2 modules are likely missing, as the in-use kernel typically is not provided by the distribution filesystem, and WSL does not mount in a module filesystem of its own. Notably the WSL2 kernel supports iptables marks without listing the xt_mark module in /sys/module, and /proc/modules is empty. On a recent kernel, we can ask the capabilities system about SYS_MODULE, that will help to disambiguate between the non-privileged container case and just being root. On older kernels these calls may fail. Update #4329 Signed-off-by: James Tucker --- cmd/tailscaled/depaware.txt | 9 ++ go.mod | 4 + go.sum | 9 ++ util/kmod/cmd/ensuremod/ensuremod.go | 31 ++++++ util/kmod/kmod.go | 161 +++++++++++++++++++++++++++ wgengine/router/router_linux.go | 11 ++ 6 files changed, 225 insertions(+) create mode 100644 util/kmod/cmd/ensuremod/ensuremod.go create mode 100644 util/kmod/kmod.go diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index b2e977957..e8144c606 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -165,9 +165,13 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de inet.af/netaddr from inet.af/wf+ inet.af/peercred from tailscale.com/ipn/ipnserver W 💣 inet.af/wf from tailscale.com/wf + L 💣 kernel.org/pub/linux/libs/security/libcap/cap from tailscale.com/util/kmod + L kernel.org/pub/linux/libs/security/libcap/psx from kernel.org/pub/linux/libs/security/libcap/cap L nhooyr.io/websocket from tailscale.com/derp/derphttp+ L nhooyr.io/websocket/internal/errd from nhooyr.io/websocket L nhooyr.io/websocket/internal/xsync from nhooyr.io/websocket + L pault.ag/go/modprobe from tailscale.com/util/kmod + L pault.ag/go/topsort from pault.ag/go/modprobe tailscale.com from tailscale.com/version tailscale.com/atomicfile from tailscale.com/ipn+ LD tailscale.com/chirp from tailscale.com/cmd/tailscaled @@ -260,6 +264,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/util/dnsname from tailscale.com/hostinfo+ LW tailscale.com/util/endian from tailscale.com/net/dns+ tailscale.com/util/groupmember from tailscale.com/ipn/ipnserver + L tailscale.com/util/kmod from tailscale.com/wgengine/router tailscale.com/util/lineread from tailscale.com/hostinfo+ tailscale.com/util/multierr from tailscale.com/cmd/tailscaled+ tailscale.com/util/netconv from tailscale.com/wgengine/magicsock @@ -329,6 +334,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de bytes from bufio+ compress/flate from compress/gzip+ compress/gzip from golang.org/x/net/http2+ + L compress/zlib from debug/elf container/heap from gvisor.dev/gvisor/pkg/tcpip/transport/tcp container/list from crypto/tls+ context from crypto/tls+ @@ -352,6 +358,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/tls from github.com/aws/aws-sdk-go-v2/aws/transport/http+ crypto/x509 from crypto/tls+ crypto/x509/pkix from crypto/x509+ + L debug/dwarf from debug/elf + L debug/elf from pault.ag/go/modprobe embed from crypto/elliptic+ encoding from encoding/json+ encoding/asn1 from crypto/x509+ @@ -366,6 +374,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de flag from tailscale.com/cmd/tailscaled+ fmt from compress/flate+ hash from crypto+ + L hash/adler32 from compress/zlib hash/crc32 from compress/gzip+ hash/fnv from gvisor.dev/gvisor/pkg/tcpip/network/ipv6+ hash/maphash from go4.org/mem diff --git a/go.mod b/go.mod index eeca2e4b9..96eb83e07 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,9 @@ require ( inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 inet.af/peercred v0.0.0-20210906144145-0893ea02156a inet.af/wf v0.0.0-20211204062712-86aaea0a7310 + kernel.org/pub/linux/libs/security/libcap/cap v1.2.64 nhooyr.io/websocket v1.8.7 + pault.ag/go/modprobe v0.1.2 ) require ( @@ -260,9 +262,11 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect howett.net/plist v1.0.0 // indirect + kernel.org/pub/linux/libs/security/libcap/psx v1.2.64 // indirect mvdan.cc/gofumpt v0.2.0 // indirect mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect mvdan.cc/unparam v0.0.0-20211002134041-24922b6997ca // indirect + pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a // indirect software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78 // indirect ) diff --git a/go.sum b/go.sum index 31c0171de..aeccb0bf7 100644 --- a/go.sum +++ b/go.sum @@ -1447,6 +1447,7 @@ golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1827,6 +1828,10 @@ inet.af/peercred v0.0.0-20210906144145-0893ea02156a h1:qdkS8Q5/i10xU2ArJMKYhVa1D inet.af/peercred v0.0.0-20210906144145-0893ea02156a/go.mod h1:FjawnflS/udxX+SvpsMgZfdqx2aykOlkISeAsADi5IU= inet.af/wf v0.0.0-20211204062712-86aaea0a7310 h1:0jKHTf+W75kYRyg5bto1UT+r18QmAz2u/5pAs/fx4zo= inet.af/wf v0.0.0-20211204062712-86aaea0a7310/go.mod h1:ViGMZRA6+RA318D7GCncrjv5gHUrPYrNDejjU12tikA= +kernel.org/pub/linux/libs/security/libcap/cap v1.2.64 h1:E1U4GNGSXEdzQUT+mop0iYawCNXDUU46Y8nfodb+ZY0= +kernel.org/pub/linux/libs/security/libcap/cap v1.2.64/go.mod h1:gtBlgvjXflnxHng9/3bXyXG3XmBYKDt35zu+lNmB+IA= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.64 h1:zlw/KoDjEObyddpFcvLiuu8frEvyEwVNc62WZQBp68w= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.64/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24= mvdan.cc/gofumpt v0.0.0-20200802201014-ab5a8192947d/go.mod h1:bzrjFmaD6+xqohD3KYP0H2FEuxknnBmyyOxdhLdaIws= mvdan.cc/gofumpt v0.0.0-20201129102820-5c11c50e9475/go.mod h1:E4LOcu9JQEtnYXtB1Y51drqh2Qr2Ngk9J3YrRCwcbd0= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= @@ -1842,6 +1847,10 @@ mvdan.cc/unparam v0.0.0-20211002134041-24922b6997ca h1:xzXXnoG5a3NUnKAcVMpE2cs3+ mvdan.cc/unparam v0.0.0-20211002134041-24922b6997ca/go.mod h1:Mb96j26qXgU/+SOj6MSgC36X30UgAlRYaxckYuYyEmo= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +pault.ag/go/modprobe v0.1.2 h1:bblunaPhqpTxGDJ5TVFW/4gheohBPleF2dIV6j6sWkI= +pault.ag/go/modprobe v0.1.2/go.mod h1:afr2STC/2Maz/qi4+Bma1s0dszZgO/PcM8AKar9DWhM= +pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a h1:WwS7vlB5H2AtwKj1jsGwp2ZLud1x6WXRXh2fXsRqrcA= +pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a/go.mod h1:INqx0ClF7kmPAMk2zVTX8DRnhZ/yaA/Mg52g8KFKE7k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/util/kmod/cmd/ensuremod/ensuremod.go b/util/kmod/cmd/ensuremod/ensuremod.go new file mode 100644 index 000000000..89a17a8d6 --- /dev/null +++ b/util/kmod/cmd/ensuremod/ensuremod.go @@ -0,0 +1,31 @@ +// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux +// +build linux + +package main + +import ( + "fmt" + "os" + + "tailscale.com/util/kmod" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "error: a module name must be supplied") + os.Exit(1) + } + + done, err := kmod.EnsureModule(os.Args[1]) + if done { + os.Exit(0) + } + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + } + os.Exit(1) +} diff --git a/util/kmod/kmod.go b/util/kmod/kmod.go new file mode 100644 index 000000000..728357bc1 --- /dev/null +++ b/util/kmod/kmod.go @@ -0,0 +1,161 @@ +// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux +// +build linux + +// Package kmod provides a simple API to attempt to ensure that a kernel +// module is loaded in a wide variety of environments, and otherwise +// report descriptive loggable error strings. +// This package does not have extensive unit testing, as the broader set +// of challenges associated with the package come from a wide variety of +// distribution and linux version differences that are problematic to +// mock/stub/emulate, including syscall boundary behaviors. The program +// `ensuremod` is kept nearby the source that provides a method for +// integration testing. +package kmod + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + + "go4.org/mem" + "golang.org/x/sys/unix" + "kernel.org/pub/linux/libs/security/libcap/cap" + "pault.ag/go/modprobe" + "tailscale.com/util/lineread" + "tailscale.com/util/multierr" +) + +// hasKernelModule attempts to find a kernel module by name using procfs and +// sysfs. If the module is found to be loaded, true is returned, in all other +// cases false is returned, regardless of errors or a missing module. +func hasKernelModule(name string) (bool, error) { + if _, err := os.Stat(filepath.Join("/sys/module", name)); err == nil { + return true, nil + } + + prefix := mem.S(name + " ") + stopFound := errors.New("") + + err := lineread.File("/proc/modules", func(line []byte) error { + if mem.HasPrefix(mem.B(line), prefix) { + return stopFound + } + return nil + }) + if err == stopFound { + return true, nil + } + if err != nil { + err = fmt.Errorf("module %s not found in /sys/module or /proc/modules: %w", name, err) + } + return false, err +} + +// canInstallModule attempts to determine if the current process has sufficient +// privilege to install modules. If the capabilities API can be queried without +// error, then the result depends on the SYS_MODULE effective capability, +// otherwise returns true only if the current process is running as root. A +// result of true implies that it may be worth trying to install a module, not +// that doing so will work. +func canInstallModule() (bool, error) { + caps, err := cap.GetPID(0) // 0 = current process + if err == nil { + // errors from GetFlag are either due to the receiver being + // uninitialized, or the kernel gave junk results, both of which aren't + // very meaningful out of context to a user, so this error is mostly + // ignored. + b, err := caps.GetFlag(cap.Effective, cap.SYS_MODULE) + if err == nil { + return b, nil + } + } + + // could not determine a well known result from capabilities, make an + // assumption based on uid. + if os.Getuid() == 0 { + return true, nil + } + return false, fmt.Errorf("not running as root, and unable to check kernel module capabilities") +} + +// firstExecutable checks paths for a path that exists and is executable by the current user. +func firstExecutable(paths ...string) string { + for _, path := range paths { + if unix.Access(path, unix.X_OK) == nil { + return path + } + } + return "" +} + +// runModprobe runs `modprobePath name` and reports summary error output on error. +func runModprobe(name, modprobePath string) error { + cmd := exec.Command(modprobePath, name) + out, err := cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("%q failed: %w; %s", fmt.Sprintf("%s %s", modprobePath, name), err, bytes.TrimSpace(out)) + } + return err +} + +// tryInstallModule attempts to find a modprobe binary to run either in +// well-known paths, or in $PATH, and runs it. If it can not find a modprobe to +// run, it instead falls back to a syscall interface to attempt to install a +// module. +func tryInstallModule(name string) error { + path := firstExecutable("/usr/sbin/modprobe", "/sbin/modprobe") + if path != "" { + return runModprobe(name, path) + } + path, err := exec.LookPath("modprobe") + if err == nil { + return runModprobe(name, path) + } + + err = modprobe.Load(name, "") + if err != nil { + err = fmt.Errorf("unable to find modprobe(1), and load of module %s failed with: %w", name, err) + } + return err +} + +// EnsureModule attempts to ensure that the given module is installed, returning +// true only if it has been found or successfully installed, otherwise false is +// returned along with a list of informational errors about probe attempts. +func EnsureModule(name string) (bool, error) { + has, hasErr := hasKernelModule(name) + if has { + return has, nil + } + var errors []error + if hasErr != nil { + errors = append(errors, hasErr) + } + + can, canErr := canInstallModule() + if can && canErr != nil { + errors = append(errors, canErr) + } + if !can { + if canErr == nil { + errors = append(errors, fmt.Errorf("module %q not found, and current user can not install modules", name)) + } + } + + if can { + if err := tryInstallModule(name); err == nil { + return true, nil + } else { + errors = append(errors, err) + } + } + + return false, multierr.New(errors...) +} diff --git a/wgengine/router/router_linux.go b/wgengine/router/router_linux.go index 687cc411b..a3218a139 100644 --- a/wgengine/router/router_linux.go +++ b/wgengine/router/router_linux.go @@ -27,11 +27,14 @@ import ( "tailscale.com/syncs" "tailscale.com/types/logger" "tailscale.com/types/preftype" + "tailscale.com/util/kmod" "tailscale.com/util/multierr" "tailscale.com/version/distro" "tailscale.com/wgengine/monitor" ) +var disableModprobe = envknob.Bool("TS_DISABLE_MODPROBE") + const ( netfilterOff = preftype.NetfilterOff netfilterNoDivert = preftype.NetfilterNoDivert @@ -175,6 +178,14 @@ func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, linkMon *monit } } + if !disableModprobe { + // xt_mark is required, so attempt to load it, and log errors that occur. + _, err := kmod.EnsureModule("xt_mark") + if err != nil { + r.logf("ensure module xt_mark: %s", err) + } + } + return r, nil }