add ability to give tls certificates to metrics server

apis/config/v1beta1/configuration_types.go

@@ -169,6 +169,12 @@ type ControllerMetrics struct {
 	// metrics will be reported.
 	// +optional
 	EnableClusterQueueResources bool `json:"enableClusterQueueResources,omitempty"`
+
+	// CertDir is the directory that contains the server key and certificate.
+	// The server key and certificate
+	// must be named tls.key and tls.crt, respectively.
+	// +optional
+	CertDir string `json:"certDir,omitempty"`
 }
 
 // ControllerHealth defines the health configs.

cmd/kueue/main.go

@@ -18,10 +18,12 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"flag"
 	"net/http"
 	"os"
+	"path/filepath"
 
 	zaplog "go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
@@ -39,6 +41,7 @@ import (
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -153,6 +156,26 @@ func main() {
 		SecureServing:  true,
 		FilterProvider: filters.WithAuthenticationAndAuthorization,
 	}
+
+	if len(cfg.Metrics.CertDir) > 0 {
+		metricsCertPath := cfg.Metrics.CertDir
+		setupLog.Info("Initializing metrics certificate watcher using provided certificates",
+			"metrics-cert-path", metricsCertPath)
+
+		var err error
+		metricsCertWatcher, err := certwatcher.New(
+			filepath.Join(metricsCertPath, "tls.crt"),
+			filepath.Join(metricsCertPath, "tls.key"),
+		)
+		if err != nil {
+			setupLog.Error(err, "to initialize metrics certificate watcher", "error", err)
+			os.Exit(1)
+		}
+
+		metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+			config.GetCertificate = metricsCertWatcher.GetCertificate
+		})
+	}
 	options.Metrics = metricsServerOptions
 
 	metrics.Register()

config/components/certmanager/certificate-metrics.yaml

@@ -0,0 +1,17 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: metrics-certs  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  dnsNames:
+  # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+  # replacements in the config/default/kustomization.yaml file.
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: metrics-server-cert

config/components/certmanager/kustomization.yaml

@@ -1,5 +1,7 @@
 resources:
 - certificate.yaml
+- certificate-metrics.yaml
+
 
 configurations:
 - kustomizeconfig.yaml

config/components/manager/controller_manager_config.yaml

@@ -4,6 +4,8 @@ health:
   healthProbeBindAddress: :8081
 metrics:
   bindAddress: :8443
+# Uncommented if you are using certs with prometheus
+#  certDir: "/tmp/k8s-metrics-server/metrics-certs"
 # enableClusterQueueResources: true
 webhook:
   port: 9443

config/components/prometheus/kustomization.yaml

@@ -1,3 +1,11 @@
 resources:
 - monitor.yaml
 - role.yaml
+# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
+# to securely reference certificates created and managed by cert-manager.
+# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
+# to mount the "metrics-server-cert" secret in the Manager Deployment.
+# patches:
+# - path: monitor_tls_patch.yaml
+#   target:
+#     kind: ServiceMonitor

config/components/prometheus/monitor_tls_patch.yaml

@@ -0,0 +1,19 @@
+# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
+# using certificates managed by cert-manager
+- op: replace
+  path: /spec/endpoints/0/tlsConfig
+  value:
+    # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+    serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc
+    insecureSkipVerify: false
+    ca:
+      secret:
+        name: metrics-server-cert
+        key: ca.crt
+    cert:
+      secret:
+        name: metrics-server-cert
+        key: tls.crt
+    keySecret:
+      name: metrics-server-cert
+      key: tls.key

config/default/cert_metrics_manager_patch.yaml

@@ -0,0 +1,25 @@
+# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs.
+
+# Add the volumeMount for the metrics-server certs
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-metrics-server/metrics-certs
+    name: metrics-certs
+    readOnly: true
+
+# Add the metrics-server certs volume configuration
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert
+      optional: false
+      items:
+        - key: ca.crt
+          path: ca.crt
+        - key: tls.crt
+          path: tls.crt
+        - key: tls.key
+          path: tls.key

config/default/kustomization.yaml

@@ -29,7 +29,7 @@ resources:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 # - ../components/certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../components/prometheus
+# - ../components/prometheus
 # [METRICS] Expose the controller manager metrics service.
 - metrics_service.yaml
 
@@ -52,8 +52,15 @@ patches:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
 # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection
-# - path: mutating_webhookcainjection_patch.yaml
-# - path: validating_webhookcainjection_patch.yaml
+- path: mutating_webhookcainjection_patch.yaml
+- path: validating_webhookcainjection_patch.yaml
+
+# Uncomment the patches line if you enable Metrics and CertManager
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+# - path: cert_metrics_manager_patch.yaml
+#  target:
+#    kind: Deployment
 
 
 # the following config is for teaching kustomize how to do var substitution