WIP: BYO TLS certs for Ingress

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml

@@ -30,3 +30,25 @@ roleRef:
   kind: Role
   name: proxies
   apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: proxies-all-secrets
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: proxies-all-secrets
+subjects:
+- kind: ServiceAccount
+  name: proxies
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: proxies-all-secrets
+  apiGroup: rbac.authorization.k8s.io

cmd/k8s-operator/ingress.go

@@ -129,9 +129,15 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
 	}
 
+	useProvidedCerts := ing.GetAnnotations()["tailscale.com/use-provided-certs"] == "true"
 	// magic443 is a fake hostname that we can use to tell containerboot to swap
 	// out with the real hostname once it's known.
 	const magic443 = "${TS_CERT_DOMAIN}:443"
+	var proxyHost = magic443
+	if useProvidedCerts {
+		proxyHost = fmt.Sprintf("%s:443", ing.Spec.TLS[0].Hosts[0]) // TODO: validate that host is set
+	}
+
 	sc := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{
 			443: {
@@ -139,18 +145,33 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 			},
 		},
 		Web: map[ipn.HostPort]*ipn.WebServerConfig{
-			magic443: {
+			ipn.HostPort(proxyHost): {
 				Handlers: map[string]*ipn.HTTPHandler{},
 			},
 		},
 	}
-	if opt.Bool(ing.Annotations[AnnotationFunnel]).EqualBool(true) {
+	if useProvidedCerts {
+		logger.Info("use provided certs")
+		// get the external certs annotation
+		secretName := ing.Spec.TLS[0].SecretName
+		if secretName == "" {
+			return fmt.Errorf("MagicDNS disabled, but Ingress does not specify an alternative TLS certs Secret")
+		}
+		// TODO: here maybe verify that the Secret exists?
+		sc.KubeSecretCertStore = &ipn.KubeSecretCertStore{
+			Name:      secretName,
+			Namespace: ing.Namespace, // what if default?
+		}
+	} else {
+		logger.Info("don't use provided certs")
+	}
+	if opt.Bool(ing.Annotations[AnnotationFunnel]).EqualBool(true) && !useProvidedCerts {
 		sc.AllowFunnel = map[ipn.HostPort]bool{
 			magic443: true,
 		}
 	}
 
-	web := sc.Web[magic443]
+	web := sc.Web[ipn.HostPort(proxyHost)]
 	addIngressBackend := func(b *networkingv1.IngressBackend, path string) {
 		if b == nil {
 			return
@@ -194,7 +215,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	addIngressBackend(ing.Spec.DefaultBackend, "/")
 
 	var tlsHost string // hostname or FQDN or empty
-	if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 {
+	if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 && !useProvidedCerts {
 		tlsHost = ing.Spec.TLS[0].Hosts[0]
 	}
 	for _, rule := range ing.Spec.Rules {

ipn/ipn_clone.go

@@ -96,6 +96,7 @@ var _ServeConfigCloneNeedsRegeneration = ServeConfig(struct {
 	AllowFunnel map[HostPort]bool
 	Foreground  map[string]*ServeConfig
 	ETag        string
+	KubeSecretCertStore *KubeSecretCertStore
 }{})
 
 // Clone makes a deep copy of TCPPortHandler.

ipn/ipn_view.go

@@ -185,6 +185,10 @@ func (v ServeConfigView) AllowFunnel() views.Map[HostPort, bool] {
 	return views.MapOf(v.ж.AllowFunnel)
 }
 
+func (v ServeConfigView) KubeSecretCert() *KubeSecretCertStore {
+	return v.ж.KubeSecretCertStore
+}
+
 func (v ServeConfigView) Foreground() views.MapFn[string, *ServeConfig, ServeConfigView] {
 	return views.MapFnOf(v.ж.Foreground, func(t *ServeConfig) ServeConfigView {
 		return t.View()
@@ -199,6 +203,7 @@ var _ServeConfigViewNeedsRegeneration = ServeConfig(struct {
 	AllowFunnel map[HostPort]bool
 	Foreground  map[string]*ServeConfig
 	ETag        string
+	KubeSecretCertStore *KubeSecretCertStore
 }{})
 
 // View returns a readonly view of TCPPortHandler.

ipn/ipnlocal/serve.go

@@ -28,6 +28,7 @@ import (
 
 	"golang.org/x/net/http2"
 	"tailscale.com/ipn"
+	"tailscale.com/kube"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/netutil"
 	"tailscale.com/syncs"
@@ -875,6 +876,31 @@ func (b *LocalBackend) getTLSServeCertForPort(port uint16) func(hi *tls.ClientHe
 
 		ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 		defer cancel()
+		if kubeSecret := b.serveConfig.KubeSecretCert(); kubeSecret != nil {
+			// TODO: initiate kube client once, maybe cache the certs somewhere too
+			c, err := kube.New()
+			if err != nil {
+				return nil, fmt.Errorf("error initalizing kube client: %v", err)
+			}
+			c.SetNS(kubeSecret.Namespace)
+			secret, err := c.GetSecret(ctx, kubeSecret.Name)
+			if err != nil {
+				return nil, fmt.Errorf("error getting certs secret: %v", err)
+			}
+			certBytes, ok := secret.Data["tls.crt"]
+			if !ok {
+				return nil, fmt.Errorf("secret does not contain tls.crt")
+			}
+			keyBytes, ok := secret.Data["tls.key"]
+			if !ok {
+				return nil, fmt.Errorf("secret data does not contain tls.key")
+			}
+			cert, err := tls.X509KeyPair(certBytes, keyBytes)
+			if err != nil {
+				return nil, fmt.Errorf("error creating TLS key pair: %w", err)
+			}
+			return &cert, nil
+		}
 		pair, err := b.GetCertPEM(ctx, hi.ServerName)
 		if err != nil {
 			return nil, err

ipn/serve.go

@@ -50,6 +50,13 @@ type ServeConfig struct {
 	// GetServeConfig request and is translated to an If-Match header
 	// during a SetServeConfig request.
 	ETag string `json:"-"`
+	// Reference to a Kubernetes Secret that contains TLS certs for serve
+	KubeSecretCertStore *KubeSecretCertStore `json:",omitempty"`
+}
+
+type KubeSecretCertStore struct {
+	Name      string `json:",omitempty"`
+	Namespace string `json:",omitempty"`
 }
 
 // HostPort is an SNI name and port number, joined by a colon.

kube/client.go

@@ -91,6 +91,11 @@ func (c *Client) SetURL(url string) {
 	c.url = url
 }
 
+// SetNS sets the ns to use to get Secret. This is a temp fix- TODO: do it differently.
+func (c *Client) SetNS(ns string) {
+	c.ns = ns
+}
+
 // SetDialer sets the dialer to use when establishing a connection
 // to the Kubernetes API server.
 func (c *Client) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {