feature reload-on-delete implemented, test cases enhanced

Author: deschmih

README.md

@@ -317,6 +317,7 @@ helm uninstall {{RELEASE_NAME}} -n {{NAMESPACE}}
 | `reloader.ignoreSecrets`          | To ignore secrets. Valid value are either `true` or `false`. Either `ignoreSecrets` or `ignoreConfigMaps` can be ignored, not both at the same time | boolean     | `false`   |
 | `reloader.ignoreConfigMaps`       | To ignore configMaps. Valid value are either `true` or `false`                                                                                      | boolean     | `false`   |
 | `reloader.reloadOnCreate`         | Enable reload on create events. Valid value are either `true` or `false`                                                                            | boolean     | `false`   |
+| `reloader.reloadOnDelete`         | Enable reload on delete events. Valid value are either `true` or `false`                                                                            | boolean     | `false`   |
 | `reloader.syncAfterRestart`       | Enable sync after Reloader restarts for **Add** events, works only when reloadOnCreate is `true`. Valid value are either `true` or `false`          | boolean     | `false`   |
 | `reloader.reloadStrategy`         | Strategy to trigger resource restart, set to either `default`, `env-vars` or `annotations`                                                          | enumeration | `default` |
 | `reloader.ignoreNamespaces`       | List of comma separated namespaces to ignore, if multiple are provided, they are combined with the AND operator                                     | string      | `""`      |
@@ -377,11 +378,15 @@ helm uninstall {{RELEASE_NAME}} -n {{NAMESPACE}}
   1. Configmaps/secrets being added to the cache will cause Reloader to perform a rolling update of the associated workload
   1. When applications are deployed for the first time, Reloader will perform a rolling update of the associated workload
   1. If you are running Reloader in HA mode all workloads will have a rolling update performed when a new leader is elected
+- `reloadOnDelete` controls how Reloader handles secrets being deleted. If `reloadOnDelete` is set to true:
+    1. Configmaps/secrets being deleted will cause Reloader to perform a rolling update of the associated workload
 - `serviceMonitor` will be removed in future releases of Reloader in favour of Pod monitor
 - If `reloadOnCreate` is set to false:
   1. Updates to configmaps/secrets that occur while there is no leader will not be picked up by the new leader until a subsequent update of the configmap/secret occurs
   1. In the worst case the window in which there can be no leader is 15s as this is the LeaseDuration
-- By default, `reloadOnCreate` and `syncAfterRestart` are both set to false. Both need to be enabled explicitly
+- If `reloadOnDelete` is set to false:
+    1. Deleting of configmaps/secrets has no effect to pods that references these resources.
+- By default, `reloadOnCreate`, `reloadOnDelete` and `syncAfterRestart` are all set to false. All need to be enabled explicitly
 
 ## Help
 

deployments/kubernetes/chart/reloader/templates/deployment.yaml

@@ -171,7 +171,7 @@ spec:
           {{- . | toYaml | nindent 10 }}
           {{- end }}
       {{- end }}
-      {{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll)}}
+      {{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (eq .Values.reloader.reloadOnDelete true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll)}}
         args:
           {{- if .Values.reloader.logFormat }}
           - "--log-format={{ .Values.reloader.logFormat }}"
@@ -231,6 +231,9 @@ spec:
           {{- if eq .Values.reloader.reloadOnCreate true }}
           - "--reload-on-create={{ .Values.reloader.reloadOnCreate }}"
           {{- end }}
+          {{- if eq .Values.reloader.reloadOnDelete true }}
+          - "--reload-on-delete={{ .Values.reloader.reloadOnDelete }}"
+          {{- end }}
           {{- if eq .Values.reloader.syncAfterRestart true }}
           - "--sync-after-restart={{ .Values.reloader.syncAfterRestart }}"
           {{- end }}

deployments/kubernetes/chart/reloader/values.yaml

@@ -18,6 +18,7 @@ reloader:
   ignoreSecrets: false
   ignoreConfigMaps: false
   reloadOnCreate: false
+  reloadOnDelete: false
   syncAfterRestart: false
   reloadStrategy: default # Set to default, env-vars or annotations
   ignoreNamespaces: "" # Comma separated list of namespaces to ignore

internal/pkg/cmd/reloader.go

@@ -51,6 +51,7 @@ func NewReloaderCommand() *cobra.Command {
 	cmd.PersistentFlags().StringVar(&options.IsArgoRollouts, "is-Argo-Rollouts", "false", "Add support for argo rollouts")
 	cmd.PersistentFlags().StringVar(&options.ReloadStrategy, constants.ReloadStrategyFlag, constants.EnvVarsReloadStrategy, "Specifies the desired reload strategy")
 	cmd.PersistentFlags().StringVar(&options.ReloadOnCreate, "reload-on-create", "false", "Add support to watch create events")
+	cmd.PersistentFlags().StringVar(&options.ReloadOnDelete, "reload-on-delete", "false", "Add support to watch delete events")
 	cmd.PersistentFlags().BoolVar(&options.EnableHA, "enable-ha", false, "Adds support for running multiple replicas via leadership election")
 	cmd.PersistentFlags().BoolVar(&options.SyncAfterRestart, "sync-after-restart", false, "Sync add events after reloader restarts")
 

internal/pkg/controller/controller.go

@@ -178,13 +178,22 @@ func (c *Controller) Update(old interface{}, new interface{}) {
 
 // Delete function to add an object to the queue in case of deleting a resource
 func (c *Controller) Delete(old interface{}) {
+
+	if options.ReloadOnDelete == "true" {
+		if !c.resourceInIgnoredNamespace(old) && c.resourceInSelectedNamespaces(old) && secretControllerInitialized && configmapControllerInitialized {
+			c.queue.Add(handler.ResourceDeleteHandler{
+				Resource:   old,
+				Collectors: c.collectors,
+				Recorder:   c.recorder,
+			})
+		}
+	}
+
 	switch object := old.(type) {
 	case *v1.Namespace:
 		c.removeSelectedNamespaceFromCache(*object)
 		return
 	}
-
-	// Todo: Any future delete event can be handled here
 }
 
 // Run function for controller which handles the queue

internal/pkg/handler/create.go

@@ -27,7 +27,7 @@ func (r ResourceCreatedHandler) Handle() error {
 			return sendUpgradeWebhook(config, options.WebhookUrl)
 		}
 		// process resource based on its type
-		return doRollingUpgrade(config, r.Collectors, r.Recorder)
+		return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
 	}
 	return nil
 }

internal/pkg/handler/delete.go

@@ -0,0 +1,92 @@
+package handler
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/stakater/Reloader/internal/pkg/callbacks"
+	"github.com/stakater/Reloader/internal/pkg/constants"
+	"github.com/stakater/Reloader/internal/pkg/metrics"
+	"github.com/stakater/Reloader/internal/pkg/options"
+	"github.com/stakater/Reloader/internal/pkg/testutil"
+	"github.com/stakater/Reloader/internal/pkg/util"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+)
+
+// ResourceDeleteHandler contains new objects
+type ResourceDeleteHandler struct {
+	Resource   interface{}
+	Collectors metrics.Collectors
+	Recorder   record.EventRecorder
+}
+
+// Handle processes resources being deleted
+func (r ResourceDeleteHandler) Handle() error {
+	if r.Resource == nil {
+		logrus.Errorf("Resource delete handler received nil resource")
+	} else {
+		config, _ := r.GetConfig()
+		// Send webhook
+		if options.WebhookUrl != "" {
+			return sendUpgradeWebhook(config, options.WebhookUrl)
+		}
+		// process resource based on its type
+		return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy)
+	}
+	return nil
+}
+
+// GetConfig gets configurations containing SHA, annotations, namespace and resource name
+func (r ResourceDeleteHandler) GetConfig() (util.Config, string) {
+	var oldSHAData string
+	var config util.Config
+	if _, ok := r.Resource.(*v1.ConfigMap); ok {
+		config = util.GetConfigmapConfig(r.Resource.(*v1.ConfigMap))
+	} else if _, ok := r.Resource.(*v1.Secret); ok {
+		config = util.GetSecretConfig(r.Resource.(*v1.Secret))
+	} else {
+		logrus.Warnf("Invalid resource: Resource should be 'Secret' or 'Configmap' but found, %v", r.Resource)
+	}
+	return config, oldSHAData
+}
+
+func invokeDeleteStrategy(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
+	if options.ReloadStrategy == constants.AnnotationsReloadStrategy {
+		return removePodAnnotations(upgradeFuncs, item, config, autoReload)
+	}
+
+	return removeContainerEnvVars(upgradeFuncs, item, config, autoReload)
+}
+
+func removePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
+	config.SHAValue = testutil.GetSHAfromEmptyData()
+	return updatePodAnnotations(upgradeFuncs, item, config, autoReload)
+}
+
+func removeContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
+	envVar := getEnvVarName(config.ResourceName, config.Type)
+	container := getContainerUsingResource(upgradeFuncs, item, config, autoReload)
+
+	if container == nil {
+		return constants.NoContainerFound
+	}
+
+	//remove if env var exists
+	containers := upgradeFuncs.ContainersFunc(item)
+	for i := range containers {
+		envs := containers[i].Env
+		index := -1
+		for j := range envs {
+			if envs[j].Name == envVar {
+				index = j
+				break
+			}
+		}
+		if index != -1 {
+			containers[i].Env = append(containers[i].Env[:index], containers[i].Env[index+1:]...)
+			return constants.Updated
+		}
+	}
+
+	return constants.NotUpdated
+}

internal/pkg/handler/update.go

@@ -29,7 +29,7 @@ func (r ResourceUpdatedHandler) Handle() error {
 				return sendUpgradeWebhook(config, options.WebhookUrl)
 			}
 			// process resource based on its type
-			return doRollingUpgrade(config, r.Collectors, r.Recorder)
+			return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
 		}
 	}
 	return nil

internal/pkg/handler/upgrade.go

@@ -142,35 +142,35 @@ func sendWebhook(url string) (string, []error) {
 	return buffer.String(), nil
 }
 
-func doRollingUpgrade(config util.Config, collectors metrics.Collectors, recorder record.EventRecorder) error {
+func doRollingUpgrade(config util.Config, collectors metrics.Collectors, recorder record.EventRecorder, invoke invokeStrategy) error {
 	clients := kube.GetClients()
 
-	err := rollingUpgrade(clients, config, GetDeploymentRollingUpgradeFuncs(), collectors, recorder)
+	err := rollingUpgrade(clients, config, GetDeploymentRollingUpgradeFuncs(), collectors, recorder, invoke)
 	if err != nil {
 		return err
 	}
-	err = rollingUpgrade(clients, config, GetCronJobCreateJobFuncs(), collectors, recorder)
+	err = rollingUpgrade(clients, config, GetCronJobCreateJobFuncs(), collectors, recorder, invoke)
 	if err != nil {
 		return err
 	}
-	err = rollingUpgrade(clients, config, GetDaemonSetRollingUpgradeFuncs(), collectors, recorder)
+	err = rollingUpgrade(clients, config, GetDaemonSetRollingUpgradeFuncs(), collectors, recorder, invoke)
 	if err != nil {
 		return err
 	}
-	err = rollingUpgrade(clients, config, GetStatefulSetRollingUpgradeFuncs(), collectors, recorder)
+	err = rollingUpgrade(clients, config, GetStatefulSetRollingUpgradeFuncs(), collectors, recorder, invoke)
 	if err != nil {
 		return err
 	}
 
 	if kube.IsOpenshift {
-		err = rollingUpgrade(clients, config, GetDeploymentConfigRollingUpgradeFuncs(), collectors, recorder)
+		err = rollingUpgrade(clients, config, GetDeploymentConfigRollingUpgradeFuncs(), collectors, recorder, invoke)
 		if err != nil {
 			return err
 		}
 	}
 
 	if options.IsArgoRollouts == "true" {
-		err = rollingUpgrade(clients, config, GetArgoRolloutRollingUpgradeFuncs(), collectors, recorder)
+		err = rollingUpgrade(clients, config, GetArgoRolloutRollingUpgradeFuncs(), collectors, recorder, invoke)
 		if err != nil {
 			return err
 		}
@@ -179,17 +179,17 @@ func doRollingUpgrade(config util.Config, collectors metrics.Collectors, recorde
 	return nil
 }
 
-func rollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder) error {
+func rollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error {
 
-	err := PerformRollingUpgrade(clients, config, upgradeFuncs, collectors, recorder)
+	err := PerformAction(clients, config, upgradeFuncs, collectors, recorder, strategy)
 	if err != nil {
 		logrus.Errorf("Rolling upgrade for '%s' failed with error = %v", config.ResourceName, err)
 	}
 	return err
 }
 
-// PerformRollingUpgrade upgrades the deployment if there is any change in configmap or secret data
-func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder) error {
+// PerformAction invokes the deployment if there is any change in configmap or secret data
+func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error {
 	items := upgradeFuncs.ItemsFunc(clients, config.Namespace)
 
 	for _, i := range items {
@@ -210,7 +210,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 		reloaderEnabled, _ := strconv.ParseBool(reloaderEnabledValue)
 		typedAutoAnnotationEnabled, _ := strconv.ParseBool(typedAutoAnnotationEnabledValue)
 		if reloaderEnabled || typedAutoAnnotationEnabled || reloaderEnabledValue == "" && typedAutoAnnotationEnabledValue == "" && options.AutoReloadAll {
-			result = invokeReloadStrategy(upgradeFuncs, i, config, true)
+			result = strategy(upgradeFuncs, i, config, true)
 		}
 
 		if result != constants.Updated && annotationValue != "" {
@@ -219,7 +219,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 				value = strings.TrimSpace(value)
 				re := regexp.MustCompile("^" + value + "$")
 				if re.Match([]byte(config.ResourceName)) {
-					result = invokeReloadStrategy(upgradeFuncs, i, config, false)
+					result = strategy(upgradeFuncs, i, config, false)
 					if result == constants.Updated {
 						break
 					}
@@ -230,7 +230,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 		if result != constants.Updated && searchAnnotationValue == "true" {
 			matchAnnotationValue := config.ResourceAnnotations[options.SearchMatchAnnotation]
 			if matchAnnotationValue == "true" {
-				result = invokeReloadStrategy(upgradeFuncs, i, config, true)
+				result = strategy(upgradeFuncs, i, config, true)
 			}
 		}
 
@@ -380,6 +380,8 @@ func getContainerUsingResource(upgradeFuncs callbacks.RollingUpgradeFuncs, item
 	return container
 }
 
+type invokeStrategy func(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result
+
 func invokeReloadStrategy(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
 	if options.ReloadStrategy == constants.AnnotationsReloadStrategy {
 		return updatePodAnnotations(upgradeFuncs, item, config, autoReload)
@@ -416,6 +418,13 @@ func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runti
 	return constants.Updated
 }
 
+func getReloaderAnnotationKey() string {
+	return fmt.Sprintf("%s/%s",
+		constants.ReloaderAnnotationPrefix,
+		constants.LastReloadedFromAnnotation,
+	)
+}
+
 func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, error) {
 	if target == nil {
 		return nil, errors.New("target is required")
@@ -426,10 +435,7 @@ func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, er
 	// Intentionally only storing the last item in order to keep
 	// the generated annotations as small as possible.
 	annotations := make(map[string]string)
-	lastReloadedResourceName := fmt.Sprintf("%s/%s",
-		constants.ReloaderAnnotationPrefix,
-		constants.LastReloadedFromAnnotation,
-	)
+	lastReloadedResourceName := getReloaderAnnotationKey()
 
 	lastReloadedResource, err := json.Marshal(target)
 	if err != nil {
@@ -440,9 +446,13 @@ func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, er
 	return annotations, nil
 }
 
+func getEnvVarName(resourceName string, typeName string) string {
+	return constants.EnvVarPrefix + util.ConvertToEnvVarName(resourceName) + "_" + typeName
+}
+
 func updateContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
 	var result constants.Result
-	envVar := constants.EnvVarPrefix + util.ConvertToEnvVarName(config.ResourceName) + "_" + config.Type
+	envVar := getEnvVarName(config.ResourceName, config.Type)
 	container := getContainerUsingResource(upgradeFuncs, item, config, autoReload)
 
 	if container == nil {