docs/k8s: add instructions on how to run as a sidecar or a proxy.

Signed-off-by: Maisem Ali <maisem@tailscale.com>

build_docker.sh

@@ -25,4 +25,4 @@ docker build \
   --build-arg VERSION_LONG=$VERSION_LONG \
   --build-arg VERSION_SHORT=$VERSION_SHORT \
   --build-arg VERSION_GIT_HASH=$VERSION_GIT_HASH \
-  -t tailscale:tailscale .
+  -t tailscale:$VERSION_SHORT -t tailscale:latest .

docs/k8s/Dockerfile

@@ -0,0 +1,7 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM tailscale:latest
+COPY run.sh /run.sh
+CMD "/run.sh"

docs/k8s/Makefile

@@ -0,0 +1,34 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+ifndef IMAGE_TAG
+  $(error "IMAGE_TAG is not set")
+endif
+
+ROUTES ?= ""
+SA_NAME ?= tailscale
+KUBE_SECRET ?= tailscale
+
+build:
+	@docker build . -t $(IMAGE_TAG)
+
+push: build
+	@docker push $(IMAGE_TAG)
+
+rbac:
+	@sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" role.yaml | kubectl apply -f -
+	@sed -e "s;{{SA_NAME}};$(SA_NAME);g" rolebinding.yaml | kubectl apply -f -
+	@sed -e "s;{{SA_NAME}};$(SA_NAME);g" sa.yaml | kubectl apply -f -
+
+sidecar:
+	@kubectl delete -f sidecar.yaml --ignore-not-found --grace-period=0
+	@sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" sidecar.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | kubectl create -f-
+
+userspace-sidecar:
+	@kubectl delete -f userspace-sidecar.yaml --ignore-not-found --grace-period=0
+	@sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" userspace-sidecar.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | kubectl create -f-
+
+proxy:
+	@kubectl delete -f proxy.yaml --ignore-not-found --grace-period=0
+	@sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" proxy.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | sed -e "s;{{DEST_IP}};$(DEST_IP);g" | kubectl create -f-

docs/k8s/README.md

@@ -1,20 +1,110 @@
-# Using Kubernetes Secrets as the state store for Tailscale
-Tailscale supports using Kubernetes Secrets as the state store, however there is some configuration required in order for it to work.
+# Overview
+There are quite a few ways of running Tailscale inside a Kubernetes Cluster, some of the common ones are covered in this doc.
+## Instructions
+### Setup
+1. (Optional) Create the following secret which will automate login.<br>
+   You will need to get an [auth key](https://tailscale.com/kb/1085/auth-keys/) from [Tailscale Admin Console](https://login.tailscale.com/admin/authkeys).<br>
+   If you don't provide the key, you can still authenticate using the url in the logs.
 
-**Note: this only works if `tailscaled` runs inside a pod in the cluster.**
+   ```yaml
+   apiVersion: v1
+   kind: Secret
+   metadata:
+     name: tailscale-auth
+   stringData:
+     AUTH_KEY: tskey-...
+   ```
+
+1. Build and push the container
+
+   ```bash
+   export IMAGE_TAG=tailscale-k8s:latest
+   make push
+   ```
+
+1. Tailscale (v1.16+) supports storing state inside a Kubernetes Secret.
+
+   Configure RBAC to allow the Tailscale pod to read/write the `tailscale` secret.
+   ```bash
+   export SA_NAME=tailscale
+   export KUBE_SECRET=tailscale
+   make rbac
+   ```
+
+### Sample Sidecar
+Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivty between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
+
+1. Create and login to the sample nginx pod with a Tailscale sidecar
+
+   ```bash
+   make sidecar
+   # If not using an auth key, authenticate by grabbing the Login URL here:
+   kubectl logs nginx ts-sidecar
+   ```
+
+1. Check if you can to connect to nginx over Tailscale:
+
+   ```bash
+   curl http://nginx
+   ```
+   Or, if you have [MagicDNS](https://tailscale.com/kb/1081/magicdns/) disabled:
+   ```bash
+   curl "http://$(tailscale ip -4 nginx)"
+   ```
+
+#### Userspace Sidecar
+You can also run the sidecar in userspace mode. The obvious benefit is reducing the amount of permissions Tailscale needs to run, the downside is that for outbound connectivity from the pod to the Tailnet you would need to use either the [SOCKS proxy](https://tailscale.com/kb/1112/userspace-networking) or HTTP proxy.
 
-1. Create a service account for Tailscale (optional)
+1. Create and login to the sample nginx pod with a Tailscale sidecar
+
+   ```bash
+   make userspace-sidecar
+   # If not using an auth key, authenticate by grabbing the Login URL here:
+   kubectl logs nginx ts-sidecar
    ```
-   kubectl create -f sa.yaml
+
+1. Check if you can to connect to nginx over Tailscale:
+
+   ```bash
+   curl http://nginx
+   ```
+   Or, if you have [MagicDNS](https://tailscale.com/kb/1081/magicdns/) disabled:
+   ```bash
+   curl "http://$(tailscale ip -4 nginx)"
    ```
 
-1. Create role and role bindings for the service account
+### Sample Proxy
+Running a Tailscale proxy allows you to provide inbound connectivity to a Kubernetes Service.
+
+1. Provide the `ClusterIP` of the service you want to reach by either:
+
+   **Creating a new deployment**
+   ```bash
+   kubectl create deployment nginx --image nginx
+   kubectl expose deployment nginx --port 80
+   export DEST_IP="$(kubectl get svc nginx -o=jsonpath='{.spec.clusterIP}')"
    ```
-   kubectl create -f role.yaml
-   kubectl create -f rolebinding.yaml
+   **Using an existing service**
+   ```bash
+   export DEST_IP="$(kubectl get svc <SVC_NAME> -o=jsonpath='{.spec.clusterIP}')"
    ```
 
-1. Launch `tailscaled` with a Kubernetes Secret as the state store.
+1. Deploy the proxy pod
+
+   ```bash
+   make proxy
+   # If not using an auth key, authenticate by grabbing the Login URL here:
+   kubectl logs proxy
    ```
-   tailscaled --state=kube:tailscale
+
+1. Check if you can to connect to nginx over Tailscale:
+
+   ```bash
+   curl http://proxy
+   ```
+
+   Or, if you have [MagicDNS](https://tailscale.com/kb/1081/magicdns/) disabled:
+
+   ```bash
+   curl "http://$(tailscale ip -4 proxy)"
    ```

docs/k8s/proxy.yaml

@@ -0,0 +1,47 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: proxy
+spec:
+  serviceAccountName: "{{SA_NAME}}"
+  initContainers:
+    # In order to run as a proxy we need to enable IP Forwarding inside
+    # the container. The `net.ipv4.ip_forward` sysctl is not whitelisted
+    # in Kubelet by default.
+  - name: sysctler
+    image: busybox
+    securityContext:
+      privileged: true
+    command: ["/bin/sh"]
+    args:
+      - -c
+      - sysctl -w net.ipv4.ip_forward=1
+    resources:
+      requests:
+        cpu: 1m
+        memory: 1Mi
+  containers:
+  - name: tailscale
+    imagePullPolicy: Always
+    image: "{{IMAGE_TAG}}"
+    env:
+    # Store the state in a k8s secret
+    - name: KUBE_SECRET
+      value: "{{KUBE_SECRET}}"
+    - name: USERSPACE
+      value: "false"
+    - name: AUTH_KEY
+      valueFrom:
+        secretKeyRef:
+          name: tailscale-auth
+          key: AUTH_KEY
+          optional: true
+    - name: DEST_IP
+      value: "{{DEST_IP}}"
+    securityContext:
+      capabilities:
+        add:
+        - NET_ADMIN

docs/k8s/role.yaml

@@ -1,10 +1,16 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  namespace: default
   name: tailscale
 rules:
 - apiGroups: [""] # "" indicates the core API group
-  resourceNames: ["tailscale"]
   resources: ["secrets"]
-  verbs: ["create", "get", "update"]
+  # Create can not be restricted to a resource name.
+  verbs: ["create"]
+- apiGroups: [""] # "" indicates the core API group
+  resourceNames: ["{{KUBE_SECRET}}"]
+  resources: ["secrets"]
+  verbs: ["get", "update"]

docs/k8s/rolebinding.yaml

@@ -1,11 +1,13 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  namespace: default
   name: tailscale
 subjects:
 - kind: ServiceAccount
-  name: tailscale
+  name: "{{SA_NAME}}"
 roleRef:
   kind: Role
   name: tailscale

docs/k8s/run.sh

@@ -0,0 +1,59 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+#! /bin/sh
+
+export PATH=$PATH:/tailscale/bin
+
+AUTH_KEY="${AUTH_KEY:-}"
+ROUTES="${ROUTES:-}"
+DEST_IP="${DEST_IP:-}"
+EXTRA_ARGS="${EXTRA_ARGS:-}"
+USERSPACE="${USERSPACE:-true}"
+KUBE_SECRET="${KUBE_SECRET:-tailscale}"
+
+set -e
+
+TAILSCALED_ARGS="--state=kube:${KUBE_SECRET} --socket=/tmp/tailscaled.sock"
+
+if [[ "${USERSPACE}" == "true" ]]; then
+  if [[ ! -z "${DEST_IP}" ]]; then
+    echo "IP forwarding is not supported in userspace mode"
+    exit 1
+  fi
+  TAILSCALED_ARGS="${TAILSCALED_ARGS} --tun=userspace-networking"
+else
+  if [[ ! -d /dev/net ]]; then
+    mkdir -p /dev/net
+  fi
+
+  if [[ ! -c /dev/net/tun ]]; then
+    mknod /dev/net/tun c 10 200
+  fi
+fi
+
+echo "Starting tailscaled"
+tailscaled ${TAILSCALED_ARGS} &
+PID=$!
+
+UP_ARGS="--accept-dns=false"
+if [[ ! -z "${ROUTES}" ]]; then
+  UP_ARGS="--advertise-routes=${ROUTES} ${UP_ARGS}"
+fi
+if [[ ! -z "${AUTH_KEY}" ]]; then
+  UP_ARGS="--authkey=${AUTH_KEY} ${UP_ARGS}"
+fi
+if [[ ! -z "${EXTRA_ARGS}" ]]; then
+  UP_ARGS="${UP_ARGS} ${EXTRA_ARGS:-}"
+fi
+
+echo "Running tailscale up"
+tailscale --socket=/tmp/tailscaled.sock up ${UP_ARGS}
+
+if [[ ! -z "${DEST_IP}" ]]; then
+  echo "Adding iptables rule for DNAT"
+  iptables -t nat -I PREROUTING -d "$(tailscale ip -4)" -j DNAT --to-destination "${DEST_IP}"
+fi
+
+wait ${PID}

docs/k8s/sa.yaml

@@ -1,5 +1,7 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: tailscale
-  namespace: default
+  name: {{SA_NAME}}

docs/k8s/sidecar.yaml

@@ -0,0 +1,31 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+spec:
+  serviceAccountName: "{{SA_NAME}}"
+  containers:
+  - name: nginx
+    image: nginx
+  - name: ts-sidecar
+    imagePullPolicy: Always
+    image: "{{IMAGE_TAG}}"
+    env:
+      # Store the state in a k8s secret
+    - name: KUBE_SECRET
+      value: "{{KUBE_SECRET}}"
+    - name: USERSPACE
+      value: "false"
+    - name: AUTH_KEY
+      valueFrom:
+        secretKeyRef:
+          name: tailscale-auth
+          key: AUTH_KEY
+          optional: true
+    securityContext:
+      capabilities:
+        add:
+        - NET_ADMIN

docs/k8s/userspace-sidecar.yaml

@@ -0,0 +1,30 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+spec:
+  serviceAccountName: "{{SA_NAME}}"
+  containers:
+  - name: nginx
+    image: nginx
+  - name: ts-sidecar
+    imagePullPolicy: Always
+    image: "{{IMAGE_TAG}}"
+    securityContext:
+      runAsUser: 1000
+      runAsGroup: 1000
+    env:
+      # Store the state in a k8s secret
+    - name: KUBE_SECRET
+      value: "{{KUBE_SECRET}}"
+    - name: USERSPACE
+      value: "true"
+    - name: AUTH_KEY
+      valueFrom:
+        secretKeyRef:
+          name: tailscale-auth
+          key: AUTH_KEY
+          optional: true