k8s-operator,ipn: add new ProxyGroup Type peer-relay

Change-Id: Ia5f5bab08c4ca234714c993cd8742fb3f22b1d0b
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -148,6 +148,7 @@ spec:
                     - egress
                     - ingress
                     - kube-apiserver
+                    - peer-relay
                   x-kubernetes-validations:
                     - rule: self == oldSelf
                       message: ProxyGroup type is immutable

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -3050,6 +3050,7 @@ spec:
                                     - egress
                                     - ingress
                                     - kube-apiserver
+                                    - peer-relay
                                 type: string
                                 x-kubernetes-validations:
                                     - message: ProxyGroup type is immutable

cmd/k8s-operator/proxygroup.go

@@ -98,6 +98,7 @@ type ProxyGroupReconciler struct {
 	egressProxyGroups    set.Slice[types.UID] // for egress proxygroups gauge
 	ingressProxyGroups   set.Slice[types.UID] // for ingress proxygroups gauge
 	apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge
+	peerRelayProxyGroups set.Slice[types.UID] // for proxygroups configured as a peer relay
 }
 
 func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
@@ -1010,10 +1011,13 @@ func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGr
 		r.ingressProxyGroups.Add(pg.UID)
 	case tsapi.ProxyGroupTypeKubernetesAPIServer:
 		r.apiServerProxyGroups.Add(pg.UID)
+	case tsapi.ProxyGroupTypePeerRelay:
+		r.peerRelayProxyGroups.Add(pg.UID)
 	}
 	gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
 	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
 	gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len()))
+	// gaugePeerRelayProxyGroupResources.Set(int64(r.peerRelayProxyGroups.Len()))
 }
 
 // ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the
@@ -1026,10 +1030,13 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro
 		r.ingressProxyGroups.Remove(pg.UID)
 	case tsapi.ProxyGroupTypeKubernetesAPIServer:
 		r.apiServerProxyGroups.Remove(pg.UID)
+	case tsapi.ProxyGroupTypePeerRelay:
+		r.peerRelayProxyGroups.Remove(pg.UID)
 	}
 	gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
 	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
 	gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len()))
+	// gaugePeerRelayProxyGroupResources.Set(int64(r.peerRelayProxyGroups.Len()))
 }
 
 func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string, loginServer string) (tailscaledConfigs, error) {
@@ -1055,6 +1062,10 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a
 		conf.StaticEndpoints = staticEndpoints
 	}
 
+	if pg.Spec.Type == tsapi.ProxyGroupTypePeerRelay {
+		conf.RelayServerPort = ptr.To(7777)
+	}
+
 	return map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha{
 		pgMinCapabilityVersion: *conf,
 	}, nil

cmd/k8s-operator/proxygroup_specs.go

@@ -119,16 +119,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 			})
 		}
 
-		volumes = append(volumes, corev1.Volume{
+		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress || pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
-			Name: proxyConfigVolName,
+			volumes = append(volumes, corev1.Volume{
-			VolumeSource: corev1.VolumeSource{
+				Name: proxyConfigVolName,
-				ConfigMap: &corev1.ConfigMapVolumeSource{
+				VolumeSource: corev1.VolumeSource{
-					LocalObjectReference: corev1.LocalObjectReference{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
-						Name: proxyConfigVolName,
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: proxyConfigVolName,
+						},
 					},
 				},
-			},
+			})
-		})
+		}
 
 		return volumes
 	}()
@@ -150,11 +152,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 			})
 		}
 
-		mounts = append(mounts, corev1.VolumeMount{
+		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress || pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
-			Name:      proxyConfigVolName,
+			mounts = append(mounts, corev1.VolumeMount{
-			MountPath: "/etc/proxies",
+				Name:      proxyConfigVolName,
-			ReadOnly:  true,
+				MountPath: "/etc/proxies",
-		})
+				ReadOnly:  true,
+			})
+		}
 
 		return mounts
 	}()
@@ -198,7 +202,8 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 			})
 		}
 
-		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+		switch pg.Spec.Type {
+		case tsapi.ProxyGroupTypeEgress:
 			envs = append(envs,
 				// TODO(irbekrm): in 1.80 we deprecated TS_EGRESS_SERVICES_CONFIG_PATH in favour of
 				// TS_EGRESS_PROXIES_CONFIG_PATH. Remove it in 1.84.
@@ -218,7 +223,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 					Name:  "TS_ENABLE_HEALTH_CHECK",
 					Value: "true",
 				})
-		} else { // ingress
+		case tsapi.ProxyGroupTypeIngress:
 			envs = append(envs, corev1.EnvVar{
 				Name:  "TS_INTERNAL_APP",
 				Value: kubetypes.AppProxyGroupIngress,

ipn/conf.go

@@ -50,6 +50,8 @@ type ConfigVAlpha struct {
 	// should advertise amongst its wireguard endpoints.
 	StaticEndpoints []netip.AddrPort `json:",omitempty"`
 
+	RelayServerPort *int `json:",omitempty"` // if set, the port to listen on for peer relay connections
+
 	// TODO(bradfitz,maisem): future something like:
 	// Profile map[string]*Config // keyed by alice@gmail.com, corp.com (TailnetSID)
 }
@@ -155,5 +157,7 @@ func (c *ConfigVAlpha) ToPrefs() (MaskedPrefs, error) {
 	if c.AdvertiseServices != nil {
 		mp.AdvertiseServices = c.AdvertiseServices
 	}
+	mp.RelayServerPortSet = true
+	mp.RelayServerPort = c.RelayServerPort
 	return mp, nil
 }

k8s-operator/api.md

@@ -716,7 +716,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.<br />Type is immutable once a ProxyGroup is created. |  | Enum: [egress ingress kube-apiserver] <br />Type: string <br /> |
+| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.<br />Type is immutable once a ProxyGroup is created. |  | Enum: [egress ingress kube-apiserver peer-relay] <br />Type: string <br /> |
 | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. |  | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
 | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  | Minimum: 0 <br /> |
 | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. |  | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |
@@ -749,7 +749,7 @@ _Underlying type:_ _string_
 
 
 _Validation:_
-- Enum: [egress ingress kube-apiserver]
+- Enum: [egress ingress kube-apiserver peer-relay]
 - Type: string
 
 _Appears in:_

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -150,13 +150,14 @@ type TailnetDevice struct {
 }
 
 // +kubebuilder:validation:Type=string
-// +kubebuilder:validation:Enum=egress;ingress;kube-apiserver
+// +kubebuilder:validation:Enum=egress;ingress;kube-apiserver;peer-relay
 type ProxyGroupType string
 
 const (
 	ProxyGroupTypeEgress              ProxyGroupType = "egress"
 	ProxyGroupTypeIngress             ProxyGroupType = "ingress"
 	ProxyGroupTypeKubernetesAPIServer ProxyGroupType = "kube-apiserver"
+	ProxyGroupTypePeerRelay           ProxyGroupType = "peer-relay"
 )
 
 // +kubebuilder:validation:Type=string