From 2c403cbb313c89741068c7d9d700169c1bbf3ad5 Mon Sep 17 00:00:00 2001 From: Maisem Ali Date: Tue, 12 Oct 2021 14:24:07 -0700 Subject: [PATCH] docs/k8s: add instructions on how to run as a sidecar or a proxy. Signed-off-by: Maisem Ali --- build_docker.sh | 2 +- docs/k8s/Dockerfile | 7 ++ docs/k8s/Makefile | 34 ++++++++++ docs/k8s/README.md | 110 +++++++++++++++++++++++++++++--- docs/k8s/proxy.yaml | 47 ++++++++++++++ docs/k8s/role.yaml | 12 +++- docs/k8s/rolebinding.yaml | 6 +- docs/k8s/run.sh | 59 +++++++++++++++++ docs/k8s/sa.yaml | 6 +- docs/k8s/sidecar.yaml | 31 +++++++++ docs/k8s/userspace-sidecar.yaml | 30 +++++++++ 11 files changed, 326 insertions(+), 18 deletions(-) create mode 100755 docs/k8s/Dockerfile create mode 100644 docs/k8s/Makefile create mode 100644 docs/k8s/proxy.yaml create mode 100755 docs/k8s/run.sh create mode 100644 docs/k8s/sidecar.yaml create mode 100644 docs/k8s/userspace-sidecar.yaml diff --git a/build_docker.sh b/build_docker.sh index 797d41d38..27561f2b6 100755 --- a/build_docker.sh +++ b/build_docker.sh @@ -25,4 +25,4 @@ docker build \ --build-arg VERSION_LONG=$VERSION_LONG \ --build-arg VERSION_SHORT=$VERSION_SHORT \ --build-arg VERSION_GIT_HASH=$VERSION_GIT_HASH \ - -t tailscale:tailscale . + -t tailscale:$VERSION_SHORT -t tailscale:latest . diff --git a/docs/k8s/Dockerfile b/docs/k8s/Dockerfile new file mode 100755 index 000000000..308e87eb5 --- /dev/null +++ b/docs/k8s/Dockerfile @@ -0,0 +1,7 @@ +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +FROM tailscale:latest +COPY run.sh /run.sh +CMD "/run.sh" \ No newline at end of file diff --git a/docs/k8s/Makefile b/docs/k8s/Makefile new file mode 100644 index 000000000..abb664484 --- /dev/null +++ b/docs/k8s/Makefile @@ -0,0 +1,34 @@ +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +ifndef IMAGE_TAG + $(error "IMAGE_TAG is not set") +endif + +ROUTES ?= "" +SA_NAME ?= tailscale +KUBE_SECRET ?= tailscale + +build: + @docker build . -t $(IMAGE_TAG) + +push: build + @docker push $(IMAGE_TAG) + +rbac: + @sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" role.yaml | kubectl apply -f - + @sed -e "s;{{SA_NAME}};$(SA_NAME);g" rolebinding.yaml | kubectl apply -f - + @sed -e "s;{{SA_NAME}};$(SA_NAME);g" sa.yaml | kubectl apply -f - + +sidecar: + @kubectl delete -f sidecar.yaml --ignore-not-found --grace-period=0 + @sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" sidecar.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | kubectl create -f- + +userspace-sidecar: + @kubectl delete -f userspace-sidecar.yaml --ignore-not-found --grace-period=0 + @sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" userspace-sidecar.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | kubectl create -f- + +proxy: + @kubectl delete -f proxy.yaml --ignore-not-found --grace-period=0 + @sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" proxy.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | sed -e "s;{{DEST_IP}};$(DEST_IP);g" | kubectl create -f- diff --git a/docs/k8s/README.md b/docs/k8s/README.md index cd76b8150..4af300ea2 100644 --- a/docs/k8s/README.md +++ b/docs/k8s/README.md @@ -1,20 +1,110 @@ -# Using Kubernetes Secrets as the state store for Tailscale -Tailscale supports using Kubernetes Secrets as the state store, however there is some configuration required in order for it to work. +# Overview +There are quite a few ways of running Tailscale inside a Kubernetes Cluster, some of the common ones are covered in this doc. +## Instructions +### Setup +1. (Optional) Create the following secret which will automate login.
+ You will need to get an [auth key](https://tailscale.com/kb/1085/auth-keys/) from [Tailscale Admin Console](https://login.tailscale.com/admin/authkeys).
+ If you don't provide the key, you can still authenticate using the url in the logs. -**Note: this only works if `tailscaled` runs inside a pod in the cluster.** + ```yaml + apiVersion: v1 + kind: Secret + metadata: + name: tailscale-auth + stringData: + AUTH_KEY: tskey-... + ``` + +1. Build and push the container -1. Create a service account for Tailscale (optional) + ```bash + export IMAGE_TAG=tailscale-k8s:latest + make push ``` - kubectl create -f sa.yaml + +1. Tailscale (v1.16+) supports storing state inside a Kubernetes Secret. + + Configure RBAC to allow the Tailscale pod to read/write the `tailscale` secret. + ```bash + export SA_NAME=tailscale + export KUBE_SECRET=tailscale + make rbac + ``` + +### Sample Sidecar +Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivty between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow. + +1. Create and login to the sample nginx pod with a Tailscale sidecar + + ```bash + make sidecar + # If not using an auth key, authenticate by grabbing the Login URL here: + kubectl logs nginx ts-sidecar ``` -1. Create role and role bindings for the service account +1. Check if you can to connect to nginx over Tailscale: + + ```bash + curl http://nginx ``` - kubectl create -f role.yaml - kubectl create -f rolebinding.yaml + Or, if you have [MagicDNS](https://tailscale.com/kb/1081/magicdns/) disabled: + ```bash + curl "http://$(tailscale ip -4 nginx)" ``` -1. Launch `tailscaled` with a Kubernetes Secret as the state store. +#### Userspace Sidecar +You can also run the sidecar in userspace mode. The obvious benefit is reducing the amount of permissions Tailscale needs to run, the downside is that for outbound connectivity from the pod to the Tailnet you would need to use either the [SOCKS proxy](https://tailscale.com/kb/1112/userspace-networking) or HTTP proxy. + +1. Create and login to the sample nginx pod with a Tailscale sidecar + + ```bash + make userspace-sidecar + # If not using an auth key, authenticate by grabbing the Login URL here: + kubectl logs nginx ts-sidecar + ``` + +1. Check if you can to connect to nginx over Tailscale: + + ```bash + curl http://nginx ``` - tailscaled --state=kube:tailscale + Or, if you have [MagicDNS](https://tailscale.com/kb/1081/magicdns/) disabled: + ```bash + curl "http://$(tailscale ip -4 nginx)" ``` + +### Sample Proxy +Running a Tailscale proxy allows you to provide inbound connectivity to a Kubernetes Service. + +1. Provide the `ClusterIP` of the service you want to reach by either: + + **Creating a new deployment** + ```bash + kubectl create deployment nginx --image nginx + kubectl expose deployment nginx --port 80 + export DEST_IP="$(kubectl get svc nginx -o=jsonpath='{.spec.clusterIP}')" + ``` + **Using an existing service** + ```bash + export DEST_IP="$(kubectl get svc -o=jsonpath='{.spec.clusterIP}')" + ``` + +1. Deploy the proxy pod + + ```bash + make proxy + # If not using an auth key, authenticate by grabbing the Login URL here: + kubectl logs proxy + ``` + +1. Check if you can to connect to nginx over Tailscale: + + ```bash + curl http://proxy + ``` + + Or, if you have [MagicDNS](https://tailscale.com/kb/1081/magicdns/) disabled: + + ```bash + curl "http://$(tailscale ip -4 proxy)" + ``` \ No newline at end of file diff --git a/docs/k8s/proxy.yaml b/docs/k8s/proxy.yaml new file mode 100644 index 000000000..3ee59f95d --- /dev/null +++ b/docs/k8s/proxy.yaml @@ -0,0 +1,47 @@ +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. +apiVersion: v1 +kind: Pod +metadata: + name: proxy +spec: + serviceAccountName: "{{SA_NAME}}" + initContainers: + # In order to run as a proxy we need to enable IP Forwarding inside + # the container. The `net.ipv4.ip_forward` sysctl is not whitelisted + # in Kubelet by default. + - name: sysctler + image: busybox + securityContext: + privileged: true + command: ["/bin/sh"] + args: + - -c + - sysctl -w net.ipv4.ip_forward=1 + resources: + requests: + cpu: 1m + memory: 1Mi + containers: + - name: tailscale + imagePullPolicy: Always + image: "{{IMAGE_TAG}}" + env: + # Store the state in a k8s secret + - name: KUBE_SECRET + value: "{{KUBE_SECRET}}" + - name: USERSPACE + value: "false" + - name: AUTH_KEY + valueFrom: + secretKeyRef: + name: tailscale-auth + key: AUTH_KEY + optional: true + - name: DEST_IP + value: "{{DEST_IP}}" + securityContext: + capabilities: + add: + - NET_ADMIN diff --git a/docs/k8s/role.yaml b/docs/k8s/role.yaml index f31060aa6..cc79ae4be 100644 --- a/docs/k8s/role.yaml +++ b/docs/k8s/role.yaml @@ -1,10 +1,16 @@ +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - namespace: default name: tailscale rules: - apiGroups: [""] # "" indicates the core API group - resourceNames: ["tailscale"] resources: ["secrets"] - verbs: ["create", "get", "update"] + # Create can not be restricted to a resource name. + verbs: ["create"] +- apiGroups: [""] # "" indicates the core API group + resourceNames: ["{{KUBE_SECRET}}"] + resources: ["secrets"] + verbs: ["get", "update"] diff --git a/docs/k8s/rolebinding.yaml b/docs/k8s/rolebinding.yaml index e979d1c1a..1f3bb3cd7 100644 --- a/docs/k8s/rolebinding.yaml +++ b/docs/k8s/rolebinding.yaml @@ -1,11 +1,13 @@ +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - namespace: default name: tailscale subjects: - kind: ServiceAccount - name: tailscale + name: "{{SA_NAME}}" roleRef: kind: Role name: tailscale diff --git a/docs/k8s/run.sh b/docs/k8s/run.sh new file mode 100755 index 000000000..dfe8d4363 --- /dev/null +++ b/docs/k8s/run.sh @@ -0,0 +1,59 @@ +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +#! /bin/sh + +export PATH=$PATH:/tailscale/bin + +AUTH_KEY="${AUTH_KEY:-}" +ROUTES="${ROUTES:-}" +DEST_IP="${DEST_IP:-}" +EXTRA_ARGS="${EXTRA_ARGS:-}" +USERSPACE="${USERSPACE:-true}" +KUBE_SECRET="${KUBE_SECRET:-tailscale}" + +set -e + +TAILSCALED_ARGS="--state=kube:${KUBE_SECRET} --socket=/tmp/tailscaled.sock" + +if [[ "${USERSPACE}" == "true" ]]; then + if [[ ! -z "${DEST_IP}" ]]; then + echo "IP forwarding is not supported in userspace mode" + exit 1 + fi + TAILSCALED_ARGS="${TAILSCALED_ARGS} --tun=userspace-networking" +else + if [[ ! -d /dev/net ]]; then + mkdir -p /dev/net + fi + + if [[ ! -c /dev/net/tun ]]; then + mknod /dev/net/tun c 10 200 + fi +fi + +echo "Starting tailscaled" +tailscaled ${TAILSCALED_ARGS} & +PID=$! + +UP_ARGS="--accept-dns=false" +if [[ ! -z "${ROUTES}" ]]; then + UP_ARGS="--advertise-routes=${ROUTES} ${UP_ARGS}" +fi +if [[ ! -z "${AUTH_KEY}" ]]; then + UP_ARGS="--authkey=${AUTH_KEY} ${UP_ARGS}" +fi +if [[ ! -z "${EXTRA_ARGS}" ]]; then + UP_ARGS="${UP_ARGS} ${EXTRA_ARGS:-}" +fi + +echo "Running tailscale up" +tailscale --socket=/tmp/tailscaled.sock up ${UP_ARGS} + +if [[ ! -z "${DEST_IP}" ]]; then + echo "Adding iptables rule for DNAT" + iptables -t nat -I PREROUTING -d "$(tailscale ip -4)" -j DNAT --to-destination "${DEST_IP}" +fi + +wait ${PID} \ No newline at end of file diff --git a/docs/k8s/sa.yaml b/docs/k8s/sa.yaml index ffa54fad8..5bfa2dbfc 100644 --- a/docs/k8s/sa.yaml +++ b/docs/k8s/sa.yaml @@ -1,5 +1,7 @@ +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. apiVersion: v1 kind: ServiceAccount metadata: - name: tailscale - namespace: default + name: {{SA_NAME}} diff --git a/docs/k8s/sidecar.yaml b/docs/k8s/sidecar.yaml new file mode 100644 index 000000000..98e60c22c --- /dev/null +++ b/docs/k8s/sidecar.yaml @@ -0,0 +1,31 @@ +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + serviceAccountName: "{{SA_NAME}}" + containers: + - name: nginx + image: nginx + - name: ts-sidecar + imagePullPolicy: Always + image: "{{IMAGE_TAG}}" + env: + # Store the state in a k8s secret + - name: KUBE_SECRET + value: "{{KUBE_SECRET}}" + - name: USERSPACE + value: "false" + - name: AUTH_KEY + valueFrom: + secretKeyRef: + name: tailscale-auth + key: AUTH_KEY + optional: true + securityContext: + capabilities: + add: + - NET_ADMIN diff --git a/docs/k8s/userspace-sidecar.yaml b/docs/k8s/userspace-sidecar.yaml new file mode 100644 index 000000000..8b1ebed7d --- /dev/null +++ b/docs/k8s/userspace-sidecar.yaml @@ -0,0 +1,30 @@ +# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + serviceAccountName: "{{SA_NAME}}" + containers: + - name: nginx + image: nginx + - name: ts-sidecar + imagePullPolicy: Always + image: "{{IMAGE_TAG}}" + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + env: + # Store the state in a k8s secret + - name: KUBE_SECRET + value: "{{KUBE_SECRET}}" + - name: USERSPACE + value: "true" + - name: AUTH_KEY + valueFrom: + secretKeyRef: + name: tailscale-auth + key: AUTH_KEY + optional: true