cmd/k8s-operator: cleanup runReconciler signature (#11993)

Updates#cleanup

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/generate/main.go

@@ -38,10 +38,10 @@ func main() {
 	}
 	repoRoot := "../../"
 	switch os.Args[1] {
-	case "helmcrd": // insert CRD to Helm templates behind a installCRDs=true conditional check
+	case "helmcrd": // insert CRDs to Helm templates behind a installCRDs=true conditional check
-		log.Print("Adding Connector CRD to Helm templates")
+		log.Print("Adding CRDs to Helm templates")
 		if err := generate("./"); err != nil {
-			log.Fatalf("error adding Connector CRD to Helm templates: %v", err)
+			log.Fatalf("error adding CRDs to Helm templates: %v", err)
 		}
 		return
 	case "staticmanifests": // generate static manifests from Helm templates (including the CRD)

cmd/k8s-operator/operator.go

@@ -66,6 +66,7 @@ func main() {
 		priorityClassName     = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags                  = defaultEnv("PROXY_TAGS", "tag:k8s")
 		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
+		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
 
 	var opts []kzap.Opts
@@ -94,9 +95,19 @@ func main() {
 	defer s.Close()
 	restConfig := config.GetConfigOrDie()
 	maybeLaunchAPIServerProxy(zlog, restConfig, s, mode)
-	// TODO (irbekrm): gather the reconciler options into an opts struct
+	rOpts := reconcilerOpts{
-	// rather than passing a million of them in one by one.
+		log:                           zlog,
-	runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode)
+		tsServer:                      s,
+		tsClient:                      tsClient,
+		tailscaleNamespace:            tsNamespace,
+		restConfig:                    restConfig,
+		proxyImage:                    image,
+		proxyPriorityClassName:        priorityClassName,
+		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
+		proxyTags:                     tags,
+		proxyFirewallMode:             tsFirewallMode,
+	}
+	runReconcilers(rOpts)
 }
 
 // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
@@ -204,11 +215,8 @@ waitOnline:
 
 // runReconcilers starts the controller-runtime manager and registers the
 // ServiceReconciler. It blocks forever.
-func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string) {
+func runReconcilers(opts reconcilerOpts) {
-	var (
+	startlog := opts.log.Named("startReconcilers")
-		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
-	)
-	startlog := zlog.Named("startReconcilers")
 	// For secrets and statefulsets, we only get permission to touch the objects
 	// in the controller's own namespace. This cannot be expressed by
 	// .Watches(...) below, instead you have to add a per-type field selector to
@@ -216,7 +224,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 	// implicitly filter what parts of the world the builder code gets to see at
 	// all.
 	nsFilter := cache.ByObject{
-		Field: client.InNamespace(tsNamespace).AsSelector(),
+		Field: client.InNamespace(opts.tailscaleNamespace).AsSelector(),
 	}
 	mgrOpts := manager.Options{
 		// TODO (irbekrm): stricter filtering what we watch/cache/call
@@ -234,7 +242,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		},
 		Scheme: tsapi.GlobalScheme,
 	}
-	mgr, err := manager.New(restConfig, mgrOpts)
+	mgr, err := manager.New(opts.restConfig, mgrOpts)
 	if err != nil {
 		startlog.Fatalf("could not create manager: %v", err)
 	}
@@ -248,13 +256,13 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 	eventRecorder := mgr.GetEventRecorderFor("tailscale-operator")
 	ssr := &tailscaleSTSReconciler{
 		Client:                 mgr.GetClient(),
-		tsnetServer:            s,
+		tsnetServer:            opts.tsServer,
-		tsClient:               tsClient,
+		tsClient:               opts.tsClient,
-		defaultTags:            strings.Split(tags, ","),
+		defaultTags:            strings.Split(opts.proxyTags, ","),
-		operatorNamespace:      tsNamespace,
+		operatorNamespace:      opts.tailscaleNamespace,
-		proxyImage:             image,
+		proxyImage:             opts.proxyImage,
-		proxyPriorityClassName: priorityClassName,
+		proxyPriorityClassName: opts.proxyPriorityClassName,
-		tsFirewallMode:         tsFirewallMode,
+		tsFirewallMode:         opts.proxyFirewallMode,
 	}
 	err = builder.
 		ControllerManagedBy(mgr).
@@ -266,10 +274,10 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		Complete(&ServiceReconciler{
 			ssr:                   ssr,
 			Client:                mgr.GetClient(),
-			logger:                zlog.Named("service-reconciler"),
+			logger:                opts.log.Named("service-reconciler"),
-			isDefaultLoadBalancer: isDefaultLoadBalancer,
+			isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer,
 			recorder:              eventRecorder,
-			tsNamespace:           tsNamespace,
+			tsNamespace:           opts.tailscaleNamespace,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -291,7 +299,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 			ssr:      ssr,
 			recorder: eventRecorder,
 			Client:   mgr.GetClient(),
-			logger:   zlog.Named("ingress-reconciler"),
+			logger:   opts.log.Named("ingress-reconciler"),
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -310,14 +318,14 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 			ssr:      ssr,
 			recorder: eventRecorder,
 			Client:   mgr.GetClient(),
-			logger:   zlog.Named("connector-reconciler"),
+			logger:   opts.log.Named("connector-reconciler"),
 			clock:    tstime.DefaultClock{},
 		})
 	if err != nil {
 		startlog.Fatalf("could not create connector reconciler: %v", err)
 	}
 	// TODO (irbekrm): switch to metadata-only watches for resources whose
-	// spec we don't need to inspect to reduce memory consumption
+	// spec we don't need to inspect to reduce memory consumption.
 	// https://github.com/kubernetes-sigs/controller-runtime/issues/1159
 	nameserverFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("nameserver"))
 	err = builder.ControllerManagedBy(mgr).
@@ -328,10 +336,9 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		Watches(&corev1.ServiceAccount{}, nameserverFilter).
 		Complete(&NameserverReconciler{
 			recorder:    eventRecorder,
-			tsNamespace: tsNamespace,
+			tsNamespace: opts.tailscaleNamespace,
-
 			Client:      mgr.GetClient(),
-			logger: zlog.Named("nameserver-reconciler"),
+			logger:      opts.log.Named("nameserver-reconciler"),
 			clock:       tstime.DefaultClock{},
 		})
 	if err != nil {
@@ -342,7 +349,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		Complete(&ProxyClassReconciler{
 			Client:   mgr.GetClient(),
 			recorder: eventRecorder,
-			logger:   zlog.Named("proxyclass-reconciler"),
+			logger:   opts.log.Named("proxyclass-reconciler"),
 			clock:    tstime.DefaultClock{},
 		})
 	if err != nil {
@@ -355,12 +362,12 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 	dnsRREpsOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerEndpointSliceHandler)
 	// On DNSConfig changes, reconcile all headless Services for
 	// ingress/egress proxies in operator namespace.
-	dnsRRDNSConfigOpts := handler.EnqueueRequestsFromMapFunc(enqueueAllIngressEgressProxySvcsInNS(tsNamespace, mgr.GetClient(), logger))
+	dnsRRDNSConfigOpts := handler.EnqueueRequestsFromMapFunc(enqueueAllIngressEgressProxySvcsInNS(opts.tailscaleNamespace, mgr.GetClient(), logger))
 	// On Service events, if it is an ingress/egress proxy headless Service, reconcile it.
 	dnsRRServiceOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerServiceHandler)
 	// On Ingress events, if it is a tailscale Ingress or if tailscale is the default ingress controller, reconcile the proxy
 	// headless Service.
-	dnsRRIngressOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerIngressHandler(tsNamespace, isDefaultLoadBalancer, mgr.GetClient(), logger))
+	dnsRRIngressOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerIngressHandler(opts.tailscaleNamespace, opts.proxyActAsDefaultLoadBalancer, mgr.GetClient(), logger))
 	err = builder.ControllerManagedBy(mgr).
 		Named("dns-records-reconciler").
 		Watches(&corev1.Service{}, dnsRRServiceOpts).
@@ -369,9 +376,9 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		Watches(&tsapi.DNSConfig{}, dnsRRDNSConfigOpts).
 		Complete(&dnsRecordsReconciler{
 			Client:                mgr.GetClient(),
-			tsNamespace:           tsNamespace,
+			tsNamespace:           opts.tailscaleNamespace,
-			logger:                zlog.Named("dns-records-reconciler"),
+			logger:                opts.log.Named("dns-records-reconciler"),
-			isDefaultLoadBalancer: isDefaultLoadBalancer,
+			isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create DNS records reconciler: %v", err)
@@ -382,6 +389,42 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 	}
 }
 
+type reconcilerOpts struct {
+	log                *zap.SugaredLogger
+	tsServer           *tsnet.Server
+	tsClient           *tailscale.Client
+	tailscaleNamespace string       // namespace in which operator resources will be deployed
+	restConfig         *rest.Config // config for connecting to the kube API server
+	proxyImage         string       // <proxy-image-repo>:<proxy-image-tag>
+	// proxyPriorityClassName isPriorityClass to be set for proxy Pods. This
+	// is a legacy mechanism for cluster resource configuration options -
+	// going forward use ProxyClass.
+	// https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass
+	proxyPriorityClassName string
+	// proxyTags are ACL tags to tag proxy auth keys. Multiple tags should
+	// be provided as a string with comma-separated tag values. Proxy tags
+	// default to tag:k8s.
+	// https://tailscale.com/kb/1085/auth-keys
+	proxyTags string
+	// proxyActAsDefaultLoadBalancer determines whether this operator
+	// instance should act as the default ingress controller when looking at
+	// Ingress resources with unset ingress.spec.ingressClassName.
+	// TODO (irbekrm): this setting does not respect the default
+	// IngressClass.
+	// https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class
+	// We should fix that and preferably integrate with that mechanism as
+	// well - perhaps make the operator itself create the default
+	// IngressClass if this is set to true.
+	proxyActAsDefaultLoadBalancer bool
+	// proxyFirewallMode determines whether non-userspace proxies should use
+	// iptables or nftables for firewall configuration. Accepted values are
+	// iptables, nftables and auto. If set to auto, proxy will automatically
+	// determine which mode is supported for a given host (prefer nftables).
+	// Auto is usually the best choice, unless you want to explicitly set
+	// specific mode for debugging purposes.
+	proxyFirewallMode string
+}
+
 // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each
 // ingress/egress proxy headless Service found in the provided namespace.
 func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {