diff --git a/api/operator/v1beta1/vmauth_types_test.go b/api/operator/v1beta1/vmauth_types_test.go index 06bafaf33..79d901d04 100644 --- a/api/operator/v1beta1/vmauth_types_test.go +++ b/api/operator/v1beta1/vmauth_types_test.go @@ -26,13 +26,13 @@ func TestVMAuthValidate(t *testing.T) { // invalid ingress f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail spec: ingress: - tlsHosts: + tlsHosts: - host-1 - host-2`, wantErr: `spec.ingress.tlsSecretName cannot be empty with non-empty spec.ingress.tlsHosts`, @@ -41,7 +41,7 @@ spec: // both configSecret and external config is defined at the same time f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail @@ -57,13 +57,13 @@ spec: // incorrect unauthorized access config, missing backends" f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail spec: unauthorizedUserAccessSpec: - default_url: + default_url: - http://url-1`, wantErr: "incorrect cr.spec.UnauthorizedUserAccess syntax: at least one of `url_map`, `url_prefix` or `targetRefs` must be defined", }) @@ -71,7 +71,7 @@ spec: // incorrect unauthorized access config, bad metric_labels syntax f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail @@ -80,7 +80,7 @@ spec: metric_labels: 124124asff: 12fsaf url_prefix: http://some-dst - default_url: + default_url: - http://url-1`, wantErr: `incorrect cr.spec.UnauthorizedUserAccess syntax: incorrect metricLabelName="124124asff", must match pattern="^[a-zA-Z_:.][a-zA-Z0-9_:.]*$"`, }) @@ -88,7 +88,7 @@ spec: // incorrect unauthorized access config url_map" f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail @@ -100,7 +100,7 @@ spec: - url_prefix: http://some-url src_paths: ["/path-1"] - url_prefix: http://some-url-2 - default_url: + default_url: - http://url-1`, wantErr: `incorrect cr.spec.UnauthorizedUserAccess syntax: incorrect url_map at idx=1: incorrect url_map config at least of one src_paths,src_hosts,src_query_args or src_headers must be defined`, }) @@ -108,12 +108,12 @@ spec: // both unauthorizedUserAccessSpec and UnauthorizedUserAccess defined f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail spec: - unauthorizedAccessConfig: + unauthorizedAccessConfig: - url_prefix: http://some-url src_paths: ["/path-1"] - url_prefix: http://some-url-2 @@ -124,7 +124,7 @@ spec: url_map: - url_prefix: http://some-url src_paths: ["/path-1"] - default_url: + default_url: - http://url-1`, wantErr: "at most one option can be used `spec.unauthorizedAccessConfig` or `spec.unauthorizedUserAccessSpec`, got both", }) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 53ce0a760..04c35aacd 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,8 @@ aliases: * FEATURE: [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth): previously VMAuth could read configuration only from predefined locations; now VMAuth supports arbitrary filesystem access configuration, allowing users to reference required files directly and reducing configuration workarounds. See [#899](https://github.com/VictoriaMetrics/operator/issues/899). +* FEATURE: [vmoperator](https://docs.victoriametrics.com/operator/): added `VM_COMMON_LABELS` and `VM_COMMON_ANNOTATIONS` environment variables to apply common labels/annotations to all Kubernetes resources managed by the operator. These cannot override labels/annotations already set by the operator or via `spec.managedMetadata`. This also ensures HTTPRoutes and PVCs include ManagedMetadata labels and annotations + * BUGFIX: [converter](https://docs.victoriametrics.com/operator/integrations/prometheus/#objects-conversion): disable all prometheus controllers if CRD group was not found. See [#2838](https://github.com/VictoriaMetrics/helm-charts/issues/2838). ## [v0.69.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.69.0) diff --git a/docs/env.md b/docs/env.md index 4d9a4ef9e..2ca70bf3b 100644 --- a/docs/env.md +++ b/docs/env.md @@ -267,3 +267,5 @@ | VM_WAIT_READY_INTERVAL: `5s` #
Defines poll interval for VM CRs | | VM_FORCERESYNCINTERVAL: `60s` #
configures force resync interval for VMAgent, VMAlert, VMAlertmanager and VMAuth. | | VM_ENABLESTRICTSECURITY: `false` #
EnableStrictSecurity will add default `securityContext` to pods and containers created by operator Default PodSecurityContext include: 1. RunAsNonRoot: true 2. RunAsUser/RunAsGroup/FSGroup: 65534 '65534' refers to 'nobody' in all the used default images like alpine, busybox. If you're using customize image, please make sure '65534' is a valid uid in there or specify SecurityContext. 3. FSGroupChangePolicy: &onRootMismatch If KubeVersion>=1.20, use `FSGroupChangePolicy="onRootMismatch"` to skip the recursive permission change when the root of the volume already has the correct permissions 4. SeccompProfile: type: RuntimeDefault Use `RuntimeDefault` seccomp profile by default, which is defined by the container runtime, instead of using the Unconfined (seccomp disabled) mode. Default container SecurityContext include: 1. AllowPrivilegeEscalation: false 2. ReadOnlyRootFilesystem: true 3. Capabilities: drop: - all turn off `EnableStrictSecurity` by default, see https://github.com/VictoriaMetrics/operator/issues/749 for details | +| VM_COMMON_LABELS: `-` #
CommonLabels are added to every Kubernetes resource created by the operator. They cannot override labels already set by the operator or via spec.managedMetadata. Format: key=value,key2=value2 | +| VM_COMMON_ANNOTATIONS: `-` #
CommonAnnotations are added to every Kubernetes resource created by the operator. They cannot override annotations already set by the operator or via spec.managedMetadata. Format: key=value,key2=value2 | diff --git a/internal/config/config.go b/internal/config/config.go index caa6f2608..b7db7c0df 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -623,6 +623,15 @@ type BaseOperatorConf struct { // - all // turn off `EnableStrictSecurity` by default, see https://github.com/VictoriaMetrics/operator/issues/749 for details EnableStrictSecurity bool `default:"false" env:"VM_ENABLESTRICTSECURITY"` + + // CommonLabels are added to every Kubernetes resource created by the operator. + // They cannot override labels already set by the operator or via spec.managedMetadata. + // Format: key=value,key2=value2 + CommonLabels map[string]string `default:"" env:"VM_COMMON_LABELS" envSeparator:"," envKeyValSeparator:"="` + // CommonAnnotations are added to every Kubernetes resource created by the operator. + // They cannot override annotations already set by the operator or via spec.managedMetadata. + // Format: key=value,key2=value2 + CommonAnnotations map[string]string `default:"" env:"VM_COMMON_ANNOTATIONS" envSeparator:"," envKeyValSeparator:"="` } // ResyncAfterDuration returns requeue duration for object period reconcile diff --git a/internal/controller/operator/factory/build/daemonset.go b/internal/controller/operator/factory/build/daemonset.go index df1d0d425..d5b282a20 100644 --- a/internal/controller/operator/factory/build/daemonset.go +++ b/internal/controller/operator/factory/build/daemonset.go @@ -2,34 +2,14 @@ package build import ( appsv1 "k8s.io/api/apps/v1" - "k8s.io/utils/ptr" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" ) -// DeploymentAddCommonParams adds common params for all deployments +// DaemonSetAddCommonParams adds common params for all deployments func DaemonSetAddCommonParams(dst *appsv1.DaemonSet, params *vmv1beta1.CommonAppsParams) { - dst.Spec.Template.Spec.Affinity = params.Affinity - dst.Spec.Template.Spec.Tolerations = params.Tolerations - dst.Spec.Template.Spec.SchedulerName = params.SchedulerName - dst.Spec.Template.Spec.RuntimeClassName = params.RuntimeClassName - dst.Spec.Template.Spec.HostAliases = params.HostAliases - if len(params.HostAliasesUnderScore) > 0 { - dst.Spec.Template.Spec.HostAliases = params.HostAliasesUnderScore - } - dst.Spec.Template.Spec.PriorityClassName = params.PriorityClassName - dst.Spec.Template.Spec.HostNetwork = params.HostNetwork - dst.Spec.Template.Spec.DNSPolicy = params.DNSPolicy - dst.Spec.Template.Spec.DNSConfig = params.DNSConfig - dst.Spec.Template.Spec.NodeSelector = params.NodeSelector + PodTemplateAddCommonParams(&dst.Spec.Template, params) dst.Spec.Template.Spec.SecurityContext = addStrictSecuritySettingsWithRootToPod(params) - dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds - dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints - dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets - dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates dst.Spec.MinReadySeconds = params.MinReadySeconds dst.Spec.RevisionHistoryLimit = params.RevisionHistoryLimitCount - if params.DisableAutomountServiceAccountToken { - dst.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(false) - } } diff --git a/internal/controller/operator/factory/build/defaults.go b/internal/controller/operator/factory/build/defaults.go index cc70e86ad..9150e9a0e 100644 --- a/internal/controller/operator/factory/build/defaults.go +++ b/internal/controller/operator/factory/build/defaults.go @@ -7,6 +7,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" @@ -48,11 +49,19 @@ func AddDefaults(scheme *runtime.Scheme) { scheme.AddTypeDefaultingFunc(&vmv1.VMAnomaly{}, addVMAnomalyDefaults) scheme.AddTypeDefaultingFunc(&vmv1beta1.VMServiceScrape{}, addVMServiceScrapeDefaults) scheme.AddTypeDefaultingFunc(&vmv1alpha1.VMDistributed{}, addVMDistributedDefaults) + + scheme.AddTypeDefaultingFunc(&appsv1.DaemonSet{}, addDefaultMetadata) + scheme.AddTypeDefaultingFunc(&corev1.ConfigMap{}, addDefaultMetadata) + scheme.AddTypeDefaultingFunc(&corev1.Namespace{}, addDefaultMetadata) + scheme.AddTypeDefaultingFunc(&corev1.PersistentVolumeClaim{}, addDefaultMetadata) + scheme.AddTypeDefaultingFunc(&corev1.Secret{}, addDefaultMetadata) } func addVMDistributedDefaults(objI any) { cr := objI.(*vmv1alpha1.VMDistributed) + addDefaultMetadata(cr) + if cr.Spec.ZoneCommon.ReadyTimeout == nil { cr.Spec.ZoneCommon.ReadyTimeout = &metav1.Duration{ Duration: 5 * time.Minute, @@ -81,6 +90,8 @@ func addVMDistributedDefaults(objI any) { func addStatefulsetDefaults(objI any) { obj := objI.(*appsv1.StatefulSet) + addDefaultMetadata(obj) + // special case for vm operator defaults if obj.Spec.UpdateStrategy.Type == "" { obj.Spec.UpdateStrategy.Type = appsv1.OnDeleteStatefulSetStrategyType @@ -118,6 +129,9 @@ func addStatefulsetDefaults(objI any) { // https://github.com/kubernetes/kubernetes/blob/master/pkg/apis/apps/v1/defaults.go func addDeploymentDefaults(objI any) { obj := objI.(*appsv1.Deployment) + + addDefaultMetadata(obj) + // Set DeploymentSpec.Replicas to 1 if it is not set. if obj.Spec.Replicas == nil { obj.Spec.Replicas = new(int32) @@ -158,6 +172,9 @@ func addDeploymentDefaults(objI any) { // https://github.com/kubernetes/kubernetes/blob/master/pkg/apis/core/v1/defaults.go func addServiceDefaults(objI any) { obj := objI.(*corev1.Service) + + addDefaultMetadata(obj) + if obj.Spec.SessionAffinity == "" { obj.Spec.SessionAffinity = corev1.ServiceAffinityNone } @@ -216,6 +233,8 @@ func addVMAuthDefaults(objI any) { cr := objI.(*vmv1beta1.VMAuth) c := getCfg() + addDefaultMetadata(cr) + if cr.Spec.ConfigSecret != "" { // Removed if later with ConfigSecret field later cr.Spec.SecretRef = &corev1.SecretKeySelector{ @@ -238,6 +257,8 @@ func addVMAlertDefaults(objI any) { cr := objI.(*vmv1beta1.VMAlert) c := getCfg() + addDefaultMetadata(cr) + cv := config.ApplicationDefaults(c.VMAlert) cp := commonParams{ tag: cr.Spec.ComponentVersion, @@ -253,6 +274,7 @@ func addVMAlertDefaults(objI any) { func addVMAgentDefaults(objI any) { cr := objI.(*vmv1beta1.VMAgent) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VMAgent) cp := commonParams{ @@ -269,6 +291,7 @@ func addVMAgentDefaults(objI any) { func addVLAgentDefaults(objI any) { cr := objI.(*vmv1.VLAgent) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VLAgent) cp := commonParams{ @@ -281,6 +304,7 @@ func addVLAgentDefaults(objI any) { func addVMSingleDefaults(objI any) { cr := objI.(*vmv1beta1.VMSingle) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VMSingle) cp := commonParams{ tag: cr.Spec.ComponentVersion, @@ -302,6 +326,7 @@ func addVMSingleDefaults(objI any) { func addVLogsDefaults(objI any) { cr := objI.(*vmv1beta1.VLogs) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VLogs) cp := commonParams{tag: cr.Spec.ComponentVersion} addDefaultsToCommonParams(&cr.Spec.CommonAppsParams, &cp, &cv) @@ -310,6 +335,8 @@ func addVLogsDefaults(objI any) { func addVMAnomalyDefaults(objI any) { cr := objI.(*vmv1.VMAnomaly) + addDefaultMetadata(cr) + // vmanomaly takes up to 2 minutes to start if cr.Spec.LivenessProbe == nil { cr.Spec.LivenessProbe = &corev1.Probe{ @@ -344,6 +371,7 @@ func addVMAnomalyDefaults(objI any) { func addVLSingleDefaults(objI any) { cr := objI.(*vmv1.VLSingle) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VLSingle) cp := commonParams{ tag: cr.Spec.ComponentVersion, @@ -355,6 +383,7 @@ func addVLSingleDefaults(objI any) { func addVTSingleDefaults(objI any) { cr := objI.(*vmv1.VTSingle) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VTSingle) cp := commonParams{tag: cr.Spec.ComponentVersion} addDefaultsToCommonParams(&cr.Spec.CommonAppsParams, &cp, &cv) @@ -363,6 +392,7 @@ func addVTSingleDefaults(objI any) { func addVMAlertmanagerDefaults(objI any) { cr := objI.(*vmv1beta1.VMAlertmanager) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VMAlertmanager) if cr.Spec.ClusterDomainName == "" { cr.Spec.ClusterDomainName = c.ClusterDomainName @@ -401,6 +431,7 @@ func addRequestsLoadBalancerDefaults(lb *vmv1beta1.VMAuthLoadBalancer, cp *commo func addVMClusterDefaults(objI any) { cr := objI.(*vmv1beta1.VMCluster) c := getCfg() + addDefaultMetadata(cr) if cr.Spec.ClusterDomainName == "" { cr.Spec.ClusterDomainName = c.ClusterDomainName } @@ -454,6 +485,20 @@ func addVMClusterDefaults(objI any) { } } +func addDefaultMetadata(objI any) { + cfg := config.MustGetBaseConfig() + obj, ok := objI.(metav1.Object) + if !ok { + return + } + if len(cfg.CommonLabels) > 0 { + obj.SetLabels(labels.Merge(cfg.CommonLabels, obj.GetLabels())) + } + if len(cfg.CommonAnnotations) > 0 { + obj.SetAnnotations(labels.Merge(cfg.CommonAnnotations, obj.GetAnnotations())) + } +} + func addDefaultsToCommonParams(common *vmv1beta1.CommonAppsParams, cp *commonParams, appDefaults *config.ApplicationDefaults) { c := getCfg() if common.Image.Repository == "" { @@ -524,6 +569,7 @@ func addDefaultsToVMBackup(cr *vmv1beta1.VMBackup, useDefaultResources bool, app return } c := getCfg() + addDefaultMetadata(cr) if cr.Image.Repository == "" { cr.Image.Repository = appDefaults.Image @@ -549,6 +595,7 @@ func addVMServiceScrapeDefaults(objI any) { if cr == nil { return } + addDefaultMetadata(cr) c := getCfg() if cr.Spec.DiscoveryRole == "" && c.VMServiceScrape.EnforceEndpointSlices { cr.Spec.DiscoveryRole = "endpointslice" @@ -563,6 +610,7 @@ const ( func addVTClusterDefaults(objI any) { cr := objI.(*vmv1.VTCluster) c := getCfg() + addDefaultMetadata(cr) cp := commonParams{ useStrictSecurity: cr.Spec.UseStrictSecurity, tag: cr.Spec.ClusterVersion, @@ -606,6 +654,7 @@ func addVTClusterDefaults(objI any) { func addVLClusterDefaults(objI any) { cr := objI.(*vmv1.VLCluster) c := getCfg() + addDefaultMetadata(cr) cp := commonParams{ useStrictSecurity: cr.Spec.UseStrictSecurity, tag: cr.Spec.ClusterVersion, diff --git a/internal/controller/operator/factory/build/defaults_test.go b/internal/controller/operator/factory/build/defaults_test.go index 86203d29b..cea3727a6 100644 --- a/internal/controller/operator/factory/build/defaults_test.go +++ b/internal/controller/operator/factory/build/defaults_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" @@ -549,3 +551,42 @@ func TestClusterComponentVersionDefaults(t *testing.T) { }) } } + +func TestAddDefaultMetadata(t *testing.T) { + cfg := config.MustGetBaseConfig() + defaultCfg := *cfg + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + + cfg.CommonLabels = map[string]string{ + "common-label": "common-value", + "existing-label": "should-not-overwrite", + } + cfg.CommonAnnotations = map[string]string{ + "common-annotation": "common-value", + "existing-annotation": "should-not-overwrite", + } + + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "existing-label": "existing-value", + }, + Annotations: map[string]string{ + "existing-annotation": "existing-value", + }, + }, + } + + addDefaultMetadata(obj) + + assert.Equal(t, map[string]string{ + "common-label": "common-value", + "existing-label": "existing-value", + }, obj.Labels) + assert.Equal(t, map[string]string{ + "common-annotation": "common-value", + "existing-annotation": "existing-value", + }, obj.Annotations) +} diff --git a/internal/controller/operator/factory/build/deployment.go b/internal/controller/operator/factory/build/deployment.go index 5b38cc2ad..5a7e0de9c 100644 --- a/internal/controller/operator/factory/build/deployment.go +++ b/internal/controller/operator/factory/build/deployment.go @@ -2,35 +2,14 @@ package build import ( appsv1 "k8s.io/api/apps/v1" - "k8s.io/utils/ptr" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" ) // DeploymentAddCommonParams adds common params for all deployments func DeploymentAddCommonParams(dst *appsv1.Deployment, params *vmv1beta1.CommonAppsParams) { - dst.Spec.Template.Spec.Affinity = params.Affinity - dst.Spec.Template.Spec.Tolerations = params.Tolerations - dst.Spec.Template.Spec.SchedulerName = params.SchedulerName - dst.Spec.Template.Spec.RuntimeClassName = params.RuntimeClassName - dst.Spec.Template.Spec.HostAliases = params.HostAliases - if len(params.HostAliasesUnderScore) > 0 { - dst.Spec.Template.Spec.HostAliases = params.HostAliasesUnderScore - } - dst.Spec.Template.Spec.PriorityClassName = params.PriorityClassName - dst.Spec.Template.Spec.HostNetwork = params.HostNetwork - dst.Spec.Template.Spec.DNSPolicy = params.DNSPolicy - dst.Spec.Template.Spec.DNSConfig = params.DNSConfig - dst.Spec.Template.Spec.NodeSelector = params.NodeSelector - dst.Spec.Template.Spec.SecurityContext = addStrictSecuritySettingsToPod(params) - dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds - dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints - dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets - dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates + PodTemplateAddCommonParams(&dst.Spec.Template, params) dst.Spec.MinReadySeconds = params.MinReadySeconds dst.Spec.Replicas = params.ReplicaCount dst.Spec.RevisionHistoryLimit = params.RevisionHistoryLimitCount - if params.DisableAutomountServiceAccountToken { - dst.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(false) - } } diff --git a/internal/controller/operator/factory/build/httproute.go b/internal/controller/operator/factory/build/httproute.go index a5143e849..3f53f595c 100644 --- a/internal/controller/operator/factory/build/httproute.go +++ b/internal/controller/operator/factory/build/httproute.go @@ -16,7 +16,7 @@ import ( // HTTPRoute creates HTTPRoute object func HTTPRoute(cr builderOpts, port string, httpRoute *vmv1beta1.EmbeddedHTTPRoute) (*gwapiv1.HTTPRoute, error) { - lbls := labels.Merge(httpRoute.Labels, cr.SelectorLabels()) + lbls := labels.Merge(labels.Merge(cr.FinalLabels(), httpRoute.Labels), cr.SelectorLabels()) spec := gwapiv1.HTTPRouteSpec{ CommonRouteSpec: gwapiv1.CommonRouteSpec{ @@ -35,7 +35,7 @@ func HTTPRoute(cr builderOpts, port string, httpRoute *vmv1beta1.EmbeddedHTTPRou Name: cr.PrefixedName(), Namespace: cr.GetNamespace(), Labels: lbls, - Annotations: httpRoute.Annotations, + Annotations: labels.Merge(cr.FinalAnnotations(), httpRoute.Annotations), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, Spec: spec, diff --git a/internal/controller/operator/factory/build/podtemplate.go b/internal/controller/operator/factory/build/podtemplate.go new file mode 100644 index 000000000..9a6f7065d --- /dev/null +++ b/internal/controller/operator/factory/build/podtemplate.go @@ -0,0 +1,43 @@ +package build + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/ptr" + + vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" +) + +// PodTemplateAddCommonParams populates common pod-level fields on a PodTemplateSpec +func PodTemplateAddCommonParams(dst *corev1.PodTemplateSpec, params *vmv1beta1.CommonAppsParams) { + dst.Spec.Affinity = params.Affinity + dst.Spec.Tolerations = params.Tolerations + dst.Spec.SchedulerName = params.SchedulerName + dst.Spec.RuntimeClassName = params.RuntimeClassName + dst.Spec.HostAliases = params.HostAliases + if len(params.HostAliasesUnderScore) > 0 { + dst.Spec.HostAliases = params.HostAliasesUnderScore + } + dst.Spec.PriorityClassName = params.PriorityClassName + dst.Spec.HostNetwork = params.HostNetwork + dst.Spec.DNSPolicy = params.DNSPolicy + dst.Spec.DNSConfig = params.DNSConfig + dst.Spec.NodeSelector = params.NodeSelector + dst.Spec.SecurityContext = addStrictSecuritySettingsToPod(params) + dst.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds + dst.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints + dst.Spec.ImagePullSecrets = params.ImagePullSecrets + dst.Spec.ReadinessGates = params.ReadinessGates + if params.DisableAutomountServiceAccountToken { + dst.Spec.AutomountServiceAccountToken = ptr.To(false) + } + + cfg := config.MustGetBaseConfig() + if len(cfg.CommonAnnotations) > 0 { + dst.Annotations = labels.Merge(cfg.CommonAnnotations, dst.Annotations) + } + if len(cfg.CommonLabels) > 0 { + dst.Labels = labels.Merge(cfg.CommonLabels, dst.Labels) + } +} diff --git a/internal/controller/operator/factory/build/podtemplate_test.go b/internal/controller/operator/factory/build/podtemplate_test.go new file mode 100644 index 000000000..2f57047e7 --- /dev/null +++ b/internal/controller/operator/factory/build/podtemplate_test.go @@ -0,0 +1,440 @@ +package build + +import ( + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" +) + +func TestPodTemplateParams(t *testing.T) { + f := func(params *vmv1beta1.CommonAppsParams, want corev1.PodSpec) { + t.Helper() + var got corev1.PodTemplateSpec + PodTemplateAddCommonParams(&got, params) + assert.Equal(t, want, got.Spec) + } + + affinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + {MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "zone", Operator: corev1.NodeSelectorOpIn, Values: []string{"us-east"}}}}, + }, + }, + }, + } + tolerations := []corev1.Toleration{{Key: "dedicated", Value: "monitoring", Effect: corev1.TaintEffectNoSchedule}} + + // affinity and tolerations + f( + &vmv1beta1.CommonAppsParams{ + Affinity: affinity, + Tolerations: tolerations, + }, + corev1.PodSpec{ + Affinity: affinity, + Tolerations: tolerations, + }, + ) + + // HostAliases set + f( + &vmv1beta1.CommonAppsParams{ + HostAliases: []corev1.HostAlias{ + {IP: "1.2.3.4", Hostnames: []string{"my.host"}}, + }, + }, + corev1.PodSpec{ + HostAliases: []corev1.HostAlias{ + {IP: "1.2.3.4", Hostnames: []string{"my.host"}}, + }, + }, + ) + + // HostAliasesUnderScore takes precedence over HostAliases + f( + &vmv1beta1.CommonAppsParams{ + HostAliases: []corev1.HostAlias{ + {IP: "1.2.3.4", Hostnames: []string{"old.host"}}, + }, + HostAliasesUnderScore: []corev1.HostAlias{ + {IP: "5.6.7.8", Hostnames: []string{"new.host"}}, + }, + }, + corev1.PodSpec{ + HostAliases: []corev1.HostAlias{ + {IP: "5.6.7.8", Hostnames: []string{"new.host"}}, + }, + }, + ) + + // DisableAutomountServiceAccountToken=true + f( + &vmv1beta1.CommonAppsParams{ + DisableAutomountServiceAccountToken: true, + }, + corev1.PodSpec{ + AutomountServiceAccountToken: ptr.To(false), + }, + ) + + // DisableAutomountServiceAccountToken=false + f( + &vmv1beta1.CommonAppsParams{ + DisableAutomountServiceAccountToken: false, + }, + corev1.PodSpec{}, + ) + + // scheduler, runtime class, priority class, node selector + f( + &vmv1beta1.CommonAppsParams{ + SchedulerName: "custom-scheduler", + RuntimeClassName: ptr.To("gvisor"), + PriorityClassName: "high-priority", + NodeSelector: map[string]string{"disktype": "ssd"}, + }, + corev1.PodSpec{ + SchedulerName: "custom-scheduler", + RuntimeClassName: ptr.To("gvisor"), + PriorityClassName: "high-priority", + NodeSelector: map[string]string{"disktype": "ssd"}, + }, + ) + + // DNS settings + f( + &vmv1beta1.CommonAppsParams{ + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + DNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{"8.8.8.8"}, + }, + }, + corev1.PodSpec{ + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + DNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{"8.8.8.8"}, + }, + }, + ) + + // image pull secrets and readiness gates + f( + &vmv1beta1.CommonAppsParams{ + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "registry-secret"}}, + ReadinessGates: []corev1.PodReadinessGate{{ConditionType: "example.com/ready"}}, + }, + corev1.PodSpec{ + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "registry-secret"}}, + ReadinessGates: []corev1.PodReadinessGate{{ConditionType: "example.com/ready"}}, + }, + ) + + // topology spread constraints + f( + &vmv1beta1.CommonAppsParams{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.DoNotSchedule}, + }, + }, + corev1.PodSpec{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.DoNotSchedule}, + }, + }, + ) + + // termination grace period + f( + &vmv1beta1.CommonAppsParams{ + TerminationGracePeriodSeconds: ptr.To(int64(30)), + }, + corev1.PodSpec{ + TerminationGracePeriodSeconds: ptr.To(int64(30)), + }, + ) + + // host network + f( + &vmv1beta1.CommonAppsParams{ + HostNetwork: true, + }, + corev1.PodSpec{ + HostNetwork: true, + }, + ) +} + +func TestDeploymentAddCommonParams(t *testing.T) { + f := func(params *vmv1beta1.CommonAppsParams, want appsv1.DeploymentSpec) { + t.Helper() + var got appsv1.Deployment + DeploymentAddCommonParams(&got, params) + assert.Equal(t, want, got.Spec) + } + + // replica count, min ready seconds, revision history + f( + &vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(3)), + MinReadySeconds: 10, + RevisionHistoryLimitCount: ptr.To(int32(5)), + }, + appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(3)), + MinReadySeconds: 10, + RevisionHistoryLimit: ptr.To(int32(5)), + }, + ) + + // pod-level fields are propagated + f( + &vmv1beta1.CommonAppsParams{ + NodeSelector: map[string]string{"env": "prod"}, + ReplicaCount: ptr.To(int32(1)), + }, + appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{"env": "prod"}, + }, + }, + }, + ) +} + +func TestStatefulSetAddCommonParams(t *testing.T) { + f := func(params *vmv1beta1.CommonAppsParams, want appsv1.StatefulSetSpec) { + t.Helper() + var got appsv1.StatefulSet + StatefulSetAddCommonParams(&got, params) + assert.Equal(t, want, got.Spec) + } + + // replica count, min ready seconds, revision history + f( + &vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(2)), + MinReadySeconds: 5, + RevisionHistoryLimitCount: ptr.To(int32(3)), + }, + appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(2)), + MinReadySeconds: 5, + RevisionHistoryLimit: ptr.To(int32(3)), + }, + ) + + // pod-level fields are propagated + f( + &vmv1beta1.CommonAppsParams{ + SchedulerName: "my-scheduler", + ReplicaCount: ptr.To(int32(1)), + }, + appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + SchedulerName: "my-scheduler", + }, + }, + }, + ) +} + +func TestDaemonSetAddCommonParams(t *testing.T) { + f := func(params *vmv1beta1.CommonAppsParams, want appsv1.DaemonSetSpec) { + t.Helper() + var got appsv1.DaemonSet + DaemonSetAddCommonParams(&got, params) + assert.Equal(t, want, got.Spec) + } + + // min ready seconds and revision history + f( + &vmv1beta1.CommonAppsParams{ + MinReadySeconds: 15, + RevisionHistoryLimitCount: ptr.To(int32(2)), + }, + appsv1.DaemonSetSpec{ + MinReadySeconds: 15, + RevisionHistoryLimit: ptr.To(int32(2)), + }, + ) + + // pod-level fields propagated + f( + &vmv1beta1.CommonAppsParams{ + NodeSelector: map[string]string{"role": "worker"}, + }, + appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{"role": "worker"}, + }, + }, + }, + ) + + // root-aware security context + f( + &vmv1beta1.CommonAppsParams{ + UseStrictSecurity: ptr.To(true), + }, + appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + SecurityContext: addStrictSecuritySettingsWithRootToPod(&vmv1beta1.CommonAppsParams{ + UseStrictSecurity: ptr.To(true), + }), + }, + }, + }, + ) +} + +func TestPodTemplateAddCommonParams_MergesCommonLabels(t *testing.T) { + cfg := config.MustGetBaseConfig() + defaultCfg := *cfg + defer func() { *config.MustGetBaseConfig() = defaultCfg }() + + f := func(commonLabels map[string]string, initialLabels map[string]string, wantLabels map[string]string) { + t.Helper() + cfg.CommonAnnotations = nil + cfg.CommonLabels = commonLabels + + got := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: initialLabels, + }, + } + + PodTemplateAddCommonParams(&got, &vmv1beta1.CommonAppsParams{}) + + if got.Labels == nil { + got.Labels = map[string]string{} + } + + assert.Equal(t, wantLabels, got.Labels) + } + + // existing + commons + f( + map[string]string{ + "common-label": "common-value", + "existing-label": "should-not-overwrite", + }, + map[string]string{ + "existing-label": "existing-value", + }, + map[string]string{ + "common-label": "common-value", + "existing-label": "existing-value", + }, + ) + + // don't let common rewrite selector-like labels + f( + map[string]string{ + "app.kubernetes.io/name": "common-app", + "app": "common", + }, + map[string]string{ + "app.kubernetes.io/name": "my-app", + "app": "my-app", + }, + map[string]string{ + "app.kubernetes.io/name": "my-app", + "app": "my-app", + }, + ) + + // common selector-like labels added when missing + f( + map[string]string{ + "app.kubernetes.io/name": "common-app", + "app": "common", + }, + map[string]string{}, + map[string]string{ + "app.kubernetes.io/name": "common-app", + "app": "common", + }, + ) + + // mix of selector and regular labels + f( + map[string]string{ + "app.kubernetes.io/name": "common-app", + "common-label": "common-value", + }, + map[string]string{ + "common-label": "existing-value", + }, + map[string]string{ + "app.kubernetes.io/name": "common-app", + "common-label": "existing-value", + }, + ) +} + +func TestPodTemplateAddCommonParams_MergesCommonAnnotations(t *testing.T) { + cfg := config.MustGetBaseConfig() + defaultCfg := *cfg + defer func() { *config.MustGetBaseConfig() = defaultCfg }() + + f := func(commonAnnotations map[string]string, initialAnnotations map[string]string, wantAnnotations map[string]string) { + t.Helper() + cfg.CommonLabels = nil + cfg.CommonAnnotations = commonAnnotations + + got := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: initialAnnotations, + }, + } + + PodTemplateAddCommonParams(&got, &vmv1beta1.CommonAppsParams{}) + + if got.Annotations == nil { + got.Annotations = map[string]string{} + } + + assert.Equal(t, wantAnnotations, got.Annotations) + } + + // existing + commons + f( + map[string]string{ + "common-annotation": "common-value", + "existing-annotation": "should-not-overwrite", + }, + map[string]string{ + "existing-annotation": "existing-value", + }, + map[string]string{ + "common-annotation": "common-value", + "existing-annotation": "existing-value", + }, + ) + + // add annotations when missing + f( + map[string]string{ + "common-annotation": "common-value", + }, + map[string]string{}, + map[string]string{ + "common-annotation": "common-value", + }, + ) +} diff --git a/internal/controller/operator/factory/build/statefulset.go b/internal/controller/operator/factory/build/statefulset.go index a7b19574e..d180fea40 100644 --- a/internal/controller/operator/factory/build/statefulset.go +++ b/internal/controller/operator/factory/build/statefulset.go @@ -2,35 +2,14 @@ package build import ( appsv1 "k8s.io/api/apps/v1" - "k8s.io/utils/ptr" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" ) // StatefulSetAddCommonParams adds common params to given statefulset func StatefulSetAddCommonParams(dst *appsv1.StatefulSet, params *vmv1beta1.CommonAppsParams) { - dst.Spec.Template.Spec.Affinity = params.Affinity - dst.Spec.Template.Spec.Tolerations = params.Tolerations - dst.Spec.Template.Spec.SchedulerName = params.SchedulerName - dst.Spec.Template.Spec.RuntimeClassName = params.RuntimeClassName - dst.Spec.Template.Spec.HostAliases = params.HostAliases - if len(params.HostAliasesUnderScore) > 0 { - dst.Spec.Template.Spec.HostAliases = params.HostAliasesUnderScore - } - dst.Spec.Template.Spec.PriorityClassName = params.PriorityClassName - dst.Spec.Template.Spec.HostNetwork = params.HostNetwork - dst.Spec.Template.Spec.DNSPolicy = params.DNSPolicy - dst.Spec.Template.Spec.DNSConfig = params.DNSConfig - dst.Spec.Template.Spec.NodeSelector = params.NodeSelector - dst.Spec.Template.Spec.SecurityContext = addStrictSecuritySettingsToPod(params) - dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds - dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints - dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets - dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates + PodTemplateAddCommonParams(&dst.Spec.Template, params) dst.Spec.MinReadySeconds = params.MinReadySeconds dst.Spec.Replicas = params.ReplicaCount dst.Spec.RevisionHistoryLimit = params.RevisionHistoryLimitCount - if params.DisableAutomountServiceAccountToken { - dst.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(false) - } } diff --git a/internal/controller/operator/factory/reconcile/secret.go b/internal/controller/operator/factory/reconcile/secret.go index 3dbbacf20..1e9fcbb09 100644 --- a/internal/controller/operator/factory/reconcile/secret.go +++ b/internal/controller/operator/factory/reconcile/secret.go @@ -18,6 +18,7 @@ import ( func Secret(ctx context.Context, rclient client.Client, newObj *corev1.Secret, prevMeta *metav1.ObjectMeta, owner *metav1.OwnerReference) error { nsn := types.NamespacedName{Name: newObj.Name, Namespace: newObj.Namespace} removeFinalizer := true + rclient.Scheme().Default(newObj) return retryOnConflict(func() error { var existingObj corev1.Secret if err := rclient.Get(ctx, nsn, &existingObj); err != nil { diff --git a/internal/controller/operator/factory/reconcile/service.go b/internal/controller/operator/factory/reconcile/service.go index 0f5c766e4..d3ddbc502 100644 --- a/internal/controller/operator/factory/reconcile/service.go +++ b/internal/controller/operator/factory/reconcile/service.go @@ -23,6 +23,7 @@ import ( // its users responsibility to define it correctly. func Service(ctx context.Context, rclient client.Client, newObj, prevObj *corev1.Service, owner *metav1.OwnerReference) error { svcForReconcile := newObj.DeepCopy() + rclient.Scheme().Default(svcForReconcile) return retryOnConflict(func() error { return reconcileService(ctx, rclient, svcForReconcile, prevObj, owner) }) diff --git a/internal/controller/operator/factory/vlagent/vlagent_test.go b/internal/controller/operator/factory/vlagent/vlagent_test.go index 6589a5f49..b917b210f 100644 --- a/internal/controller/operator/factory/vlagent/vlagent_test.go +++ b/internal/controller/operator/factory/vlagent/vlagent_test.go @@ -18,6 +18,7 @@ import ( vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -25,6 +26,7 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1.VLAgent + cfgMutator func(c *config.BaseOperatorConf) validate func(set *appsv1.StatefulSet) predefinedObjects []runtime.Object } @@ -32,6 +34,14 @@ func TestCreateOrUpdate(t *testing.T) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) ctx := context.TODO() + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } build.AddDefaults(fclient.Scheme()) fclient.Scheme().Default(o.cr) assert.NoError(t, CreateOrUpdate(ctx, o.cr, fclient)) @@ -294,6 +304,71 @@ func TestCreateOrUpdate(t *testing.T) { }, }, }) + + // managed metadata + f(opts{ + cr: &vmv1.VLAgent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-agent", + Namespace: "default", + }, + Spec: vmv1.VLAgentSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{ + "env": "prod", + }, + Annotations: map[string]string{ + "controller": "true", + }, + }, + RemoteWrite: []vmv1.VLAgentRemoteWriteSpec{ + {URL: "http://remote-write"}, + }, + }, + }, + validate: func(set *appsv1.StatefulSet) { + assert.Equal(t, set.Labels, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlagent", + "app.kubernetes.io/instance": "example-agent", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + assert.Equal(t, set.Annotations, map[string]string{ + "controller": "true", + }) + }, + }) + + // common labels + f(opts{ + cr: &vmv1.VLAgent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-agent", + Namespace: "default", + }, + }, + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{ + "env": "prod", + } + c.CommonAnnotations = map[string]string{ + "controller": "true", + } + }, + validate: func(set *appsv1.StatefulSet) { + assert.Equal(t, set.Labels, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlagent", + "app.kubernetes.io/instance": "example-agent", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + assert.Equal(t, set.Annotations, map[string]string{ + "controller": "true", + }) + }, + }) } func TestBuildRemoteWriteArgs(t *testing.T) { diff --git a/internal/controller/operator/factory/vlcluster/vlcluster_test.go b/internal/controller/operator/factory/vlcluster/vlcluster_test.go index 2d4a0a762..449ed560a 100644 --- a/internal/controller/operator/factory/vlcluster/vlcluster_test.go +++ b/internal/controller/operator/factory/vlcluster/vlcluster_test.go @@ -91,35 +91,86 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { // ensure SA created var sa corev1.ServiceAccount - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.GetServiceAccountName(), Namespace: cr.Namespace}, &sa)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.GetServiceAccountName(), Namespace: cr.Namespace}, &sa)) assert.Nil(t, sa.Annotations) - assert.Equal(t, sa.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentRoot)) + assert.Equal(t, sa.Labels, map[string]string{ + "app.kubernetes.io/name": "vlcluster", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check insert var dep appsv1.Deployment - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt := dep.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9481", "-internalselect.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"}) assert.Nil(t, dep.Annotations) - assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentInsert)) + assert.Equal(t, dep.Labels, map[string]string{ + "app.kubernetes.io/name": "vlinsert", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check select - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt = dep.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9471", "-internalinsert.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"}) assert.Nil(t, dep.Annotations) - assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentSelect)) + assert.Equal(t, dep.Labels, map[string]string{ + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check storage var sts appsv1.StatefulSet - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt = sts.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9491", "-storageDataPath=/vlstorage-data"}) assert.Nil(t, sts.Annotations) - assert.Equal(t, sts.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentStorage)) + assert.Equal(t, sts.Labels, map[string]string{ + "app.kubernetes.io/name": "vlstorage", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + + // check services + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlinsert", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlstorage", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -144,7 +195,7 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { // check storage var sts appsv1.StatefulSet - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt := sts.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-httpListenAddr=:9491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vlstorage-data"}) @@ -180,7 +231,7 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { // check select var d appsv1.Deployment - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &d)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &d)) assert.Len(t, d.Spec.Template.Spec.Containers, 1) cnt := d.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{ @@ -258,9 +309,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(vmv1beta1.ClusterComponentInsert), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vlinsert", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -322,9 +379,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -384,9 +447,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vlstorage", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -465,9 +534,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vlinsert", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1000", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -553,6 +628,113 @@ func TestCreateOrUpdate(t *testing.T) { assert.True(t, k8serrors.IsNotFound(err)) }, }) + + // managed metadata + f(opts{ + cr: &vmv1.VLCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VLClusterSpec{ + VLSelect: &vmv1.VLSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VLInsert: &vmv1.VLInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VLStorage: &vmv1.VLStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vlselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, set.Annotations, map[string]string{"controller": "true"}) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1.VLCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VLClusterSpec{ + VLSelect: &vmv1.VLSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VLInsert: &vmv1.VLInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VLStorage: &vmv1.VLStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vlselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, set.Annotations, map[string]string{"controller": "true"}) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }}) } func TestCreateOrUpdate_Paused(t *testing.T) { diff --git a/internal/controller/operator/factory/vlsingle/vlsingle.go b/internal/controller/operator/factory/vlsingle/vlsingle.go index fb66a348e..3900a2962 100644 --- a/internal/controller/operator/factory/vlsingle/vlsingle.go +++ b/internal/controller/operator/factory/vlsingle/vlsingle.go @@ -48,8 +48,8 @@ func newPVC(r *vmv1.VLSingle) *corev1.PersistentVolumeClaim { ObjectMeta: metav1.ObjectMeta{ Name: r.PrefixedName(), Namespace: r.Namespace, - Labels: labels.Merge(r.Spec.StorageMetadata.Labels, r.SelectorLabels()), - Annotations: r.Spec.StorageMetadata.Annotations, + Labels: labels.Merge(labels.Merge(r.FinalLabels(), r.Spec.StorageMetadata.Labels), r.SelectorLabels()), + Annotations: labels.Merge(r.FinalAnnotations(), r.Spec.StorageMetadata.Annotations), OwnerReferences: []metav1.OwnerReference{r.AsOwner()}, }, Spec: *r.Spec.Storage, diff --git a/internal/controller/operator/factory/vlsingle/vlsingle_test.go b/internal/controller/operator/factory/vlsingle/vlsingle_test.go index ceb684b71..d66b14168 100644 --- a/internal/controller/operator/factory/vlsingle/vlsingle_test.go +++ b/internal/controller/operator/factory/vlsingle/vlsingle_test.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" @@ -23,8 +24,8 @@ import ( func TestCreateOrUpdateVLSingle(t *testing.T) { type opts struct { cr *vmv1.VLSingle - c *config.BaseOperatorConf - want *appsv1.Deployment + cfgMutator func(c *config.BaseOperatorConf) + validate func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) wantErr bool predefinedObjects []runtime.Object } @@ -32,17 +33,30 @@ func TestCreateOrUpdateVLSingle(t *testing.T) { f := func(o opts) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) - err := CreateOrUpdate(context.TODO(), fclient, o.cr) + build.AddDefaults(fclient.Scheme()) + fclient.Scheme().Default(o.cr) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } + ctx := context.TODO() + err := CreateOrUpdate(ctx, fclient, o.cr) if o.wantErr { assert.Error(t, err) } else { assert.NoError(t, err) } + if o.validate != nil { + o.validate(ctx, fclient, o.cr) + } } // base gen f(opts{ - c: config.MustGetBaseConfig(), cr: &vmv1.VLSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -70,12 +84,16 @@ func TestCreateOrUpdateVLSingle(t *testing.T) { }, k8stools.NewReadyDeployment("vlsingle-base", "default"), }, - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vlsingle-base", Namespace: "default"}}, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vlsingle-base", got.Name) + assert.Equal(t, "default", got.Namespace) + }, }) // base with specific port f(opts{ - c: config.MustGetBaseConfig(), cr: &vmv1.VLSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -105,12 +123,16 @@ func TestCreateOrUpdateVLSingle(t *testing.T) { }, k8stools.NewReadyDeployment("vlsingle-base", "default"), }, - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vlsingle-base", Namespace: "default"}}, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vlsingle-base", got.Name) + assert.Equal(t, "default", got.Namespace) + }, }) // with syslog tls config f(opts{ - c: config.MustGetBaseConfig(), cr: &vmv1.VLSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -174,9 +196,66 @@ func TestCreateOrUpdateVLSingle(t *testing.T) { }, k8stools.NewReadyDeployment("vlsingle-base", "default"), }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vlsingle-base", got.Name) + assert.Equal(t, "default", got.Namespace) + }, + }) - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vlsingle-base", Namespace: "default"}}, + // managed metadata + f(opts{ + cr: &vmv1.VLSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VLSingleSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, got.Labels, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + assert.Equal(t, got.Annotations, map[string]string{"controller": "true"}) + }, }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1.VLSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, got.Labels, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + assert.Equal(t, got.Annotations, map[string]string{"controller": "true"}) + }}) } func TestCreateOrUpdateVLSingle_Paused(t *testing.T) { @@ -206,9 +285,8 @@ func TestCreateOrUpdateVLSingleService(t *testing.T) { type opts struct { cr *vmv1.VLSingle c *config.BaseOperatorConf - want *corev1.Service + validate func(*corev1.Service) wantErr bool - wantPortsLen int predefinedObjects []runtime.Object } f := func(o opts) { @@ -228,8 +306,9 @@ func TestCreateOrUpdateVLSingleService(t *testing.T) { Namespace: svc.Namespace, } assert.NoError(t, fclient.Get(ctx, nsn, &got)) - assert.Equal(t, got.Name, o.want.Name) - assert.Len(t, got.Spec.Ports, o.wantPortsLen) + if o.validate != nil { + o.validate(&got) + } } // base service test @@ -241,13 +320,18 @@ func TestCreateOrUpdateVLSingleService(t *testing.T) { Namespace: "default", }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vlsingle-logs-1", - Namespace: "default", - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vlsingle-logs-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + assert.Len(t, svc.Spec.Ports, 1) + + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlsingle", + "app.kubernetes.io/instance": "logs-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, - wantPortsLen: 1, }) // with extra service nodePort @@ -267,13 +351,18 @@ func TestCreateOrUpdateVLSingleService(t *testing.T) { }, }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vlsingle-logs-1", - Namespace: "default", - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vlsingle-logs-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + assert.Len(t, svc.Spec.Ports, 1) + // verify labels exist and include core operator metadata + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlsingle", + "app.kubernetes.io/instance": "logs-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, - wantPortsLen: 1, predefinedObjects: []runtime.Object{ &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/controller/operator/factory/vmagent/vmagent_test.go b/internal/controller/operator/factory/vmagent/vmagent_test.go index 0a17cc33f..cb5f5b42f 100644 --- a/internal/controller/operator/factory/vmagent/vmagent_test.go +++ b/internal/controller/operator/factory/vmagent/vmagent_test.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -28,6 +29,7 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1beta1.VMAgent + cfgMutator func(*config.BaseOperatorConf) validate func(ctx context.Context, client client.Client, cr *vmv1beta1.VMAgent) wantErr bool predefinedObjects []runtime.Object @@ -37,6 +39,14 @@ func TestCreateOrUpdate(t *testing.T) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) ctx := context.TODO() + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } build.AddDefaults(fclient.Scheme()) fclient.Scheme().Default(o.cr) err := CreateOrUpdate(ctx, o.cr, fclient) @@ -812,6 +822,67 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, ds.Spec.UpdateStrategy.RollingUpdate) }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMAgent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAgentSpec{ + RemoteWrite: []vmv1beta1.VMAgentRemoteWriteSpec{ + {URL: "http://remote-write"}, + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, client client.Client, cr *vmv1beta1.VMAgent) { + var set appsv1.Deployment + assert.NoError(t, client.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmagent-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMAgent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAgentSpec{ + RemoteWrite: []vmv1beta1.VMAgentRemoteWriteSpec{ + {URL: "http://remote-write"}, + }, + }, + }, + validate: func(ctx context.Context, client client.Client, cr *vmv1beta1.VMAgent) { + var set appsv1.Deployment + assert.NoError(t, client.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmagent-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + }}) } func TestBuildRemoteWriteArgs(t *testing.T) { @@ -1769,10 +1840,14 @@ func TestBuildRemoteWriteArgs(t *testing.T) { func TestCreateOrUpdateService(t *testing.T) { type opts struct { - cr *vmv1beta1.VMAgent - want func(svc *corev1.Service) error - wantAdditionalService func(svc *corev1.Service) error - predefinedObjects []runtime.Object + cr *vmv1beta1.VMAgent + // legacy functional expectations (kept for compatibility) + want func(svc *corev1.Service) + wantAdditionalService func(svc *corev1.Service) + // new preferred validation hooks (used first if present) + validate func(svc *corev1.Service) + validateAdditional func(svc *corev1.Service) + predefinedObjects []runtime.Object } f := func(o opts) { @@ -1787,11 +1862,20 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: svc.Namespace, } assert.NoError(t, cl.Get(ctx, nsn, &got)) - assert.NoError(t, o.want(&got)) - if o.wantAdditionalService != nil { + // prefer new validate hooks; fall back to legacy want functions when validate is not provided + if o.validate != nil { + o.validate(&got) + } else if o.want != nil { + o.want(&got) + } + if o.validateAdditional != nil { + var additionalSvc corev1.Service + assert.NoError(t, cl.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: o.cr.Spec.ServiceSpec.NameOrDefault(o.cr.Name)}, &additionalSvc)) + o.validateAdditional(&additionalSvc) + } else if o.wantAdditionalService != nil { var additionalSvc corev1.Service assert.NoError(t, cl.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: o.cr.Spec.ServiceSpec.NameOrDefault(o.cr.Name)}, &additionalSvc)) - assert.NoError(t, o.wantAdditionalService(&additionalSvc)) + o.wantAdditionalService(&additionalSvc) } } @@ -1803,14 +1887,16 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: "default", }, }, - want: func(svc *corev1.Service) error { - if svc.Name != "vmagent-base" { - return fmt.Errorf("unexpected name for service: %v", svc.Name) - } - if len(svc.Spec.Ports) != 1 { - return fmt.Errorf("unexpected count for service ports: %v", len(svc.Spec.Ports)) - } - return nil + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmagent-base", svc.Name) + assert.Len(t, svc.Spec.Ports, 1) + // ensure operator-managed labels present + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -1850,14 +1936,16 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: func(svc *corev1.Service) error { - if svc.Name != "vmagent-base" { - return fmt.Errorf("unexpected name for service: %v", svc.Name) - } - if len(svc.Spec.Ports) != 3 { - return fmt.Errorf("unexpected count for ports, want 3, got: %v", len(svc.Spec.Ports)) - } - return nil + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmagent-base", svc.Name) + assert.Len(t, svc.Spec.Ports, 3) + + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -1904,26 +1992,22 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: func(svc *corev1.Service) error { - if svc.Name != "vmagent-base" { - return fmt.Errorf("unexpected name for service: %v", svc.Name) - } - if len(svc.Spec.Ports) != 3 { - return fmt.Errorf("unexpected count for ports, want 3, got: %v", len(svc.Spec.Ports)) - } - return nil + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmagent-base", svc.Name) + assert.Len(t, svc.Spec.Ports, 3) + + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, - wantAdditionalService: func(svc *corev1.Service) error { - if len(svc.Spec.Ports) != 1 { - return fmt.Errorf("unexpected count for ports, want 1, got: %v", len(svc.Spec.Ports)) - } - if svc.Spec.Ports[0].NodePort != 8085 { - return fmt.Errorf("unexpected port %v, want 8085", svc.Spec.Ports[0]) - } - if svc.Spec.Ports[0].Protocol != corev1.ProtocolUDP { - return fmt.Errorf("unexpected protocol want udp, got: %v", svc.Spec.Ports[0].Protocol) - } - return nil + validateAdditional: func(svc *corev1.Service) { + // additional service should preserve the explicit port definition + assert.Len(t, svc.Spec.Ports, 1) + assert.Equal(t, int32(8085), svc.Spec.Ports[0].NodePort) + assert.Equal(t, corev1.ProtocolUDP, svc.Spec.Ports[0].Protocol) }, }) } diff --git a/internal/controller/operator/factory/vmalert/vmalert_test.go b/internal/controller/operator/factory/vmalert/vmalert_test.go index 862a11d69..f1bd64e46 100644 --- a/internal/controller/operator/factory/vmalert/vmalert_test.go +++ b/internal/controller/operator/factory/vmalert/vmalert_test.go @@ -2,7 +2,6 @@ package vmalert import ( "context" - "fmt" "strings" "testing" @@ -13,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/config" @@ -23,24 +23,30 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1beta1.VMAlert + cfgMutator func(*config.BaseOperatorConf) cmNames []string predefinedObjects []runtime.Object - validate func(*appsv1.Deployment, *corev1.Secret) + validate func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) } f := func(o opts) { t.Helper() ctx := context.TODO() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } + build.AddDefaults(fclient.Scheme()) + fclient.Scheme().Default(o.cr) err := CreateOrUpdate(ctx, o.cr, fclient, o.cmNames) assert.NoError(t, err) if o.validate != nil { - var generatedDeploment appsv1.Deployment - assert.NoError(t, fclient.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: o.cr.PrefixedName()}, &generatedDeploment)) - var generatedTLSSecret corev1.Secret - tlsSecretName := build.ResourceName(build.TLSAssetsResourceKind, o.cr) - assert.NoError(t, fclient.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: tlsSecretName}, &generatedTLSSecret)) - o.validate(&generatedDeploment, &generatedTLSSecret) + o.validate(ctx, fclient, o.cr) } } @@ -95,7 +101,9 @@ func TestCreateOrUpdate(t *testing.T) { predefinedObjects: []runtime.Object{ k8stools.NewReadyDeployment("vmalert-basic-vmalert", "default"), }, - validate: func(d *appsv1.Deployment, s *corev1.Secret) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) { + var d appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &d)) var foundOk bool for _, cnt := range d.Spec.Template.Spec.Containers { if cnt.Name == "vmalert" { @@ -199,10 +207,27 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vmalert-basic-vmalert", "default"), }, - validate: func(d *appsv1.Deployment, s *corev1.Secret) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) { + var s corev1.Secret + tlsSecretName := build.ResourceName(build.TLSAssetsResourceKind, cr) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: tlsSecretName}, &s)) assert.NotEmpty(t, s.Data["default_configmap_datasource-tls_ca"]) assert.NotEmpty(t, s.Data["default_configmap_datasource-tls_cert"]) assert.NotEmpty(t, s.Data["default_datasource-tls_key"]) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "basic-vmalert", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, s.Labels) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "basic-vmalert", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -370,6 +395,91 @@ func TestCreateOrUpdate(t *testing.T) { k8stools.NewReadyDeployment("vmalert-basic-vmalert", "default"), }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMAlert{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAlertSpec{ + Notifier: &vmv1beta1.VMAlertNotifierSpec{ + URL: "http://notifier", + }, + Datasource: vmv1beta1.VMAlertDatasourceSpec{ + URL: "http://datasource", + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMAlert{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAlertSpec{ + Notifier: &vmv1beta1.VMAlertNotifierSpec{ + URL: "http://notifier", + }, + Datasource: vmv1beta1.VMAlertDatasourceSpec{ + URL: "http://datasource", + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }}) } func TestBuildNotifiers(t *testing.T) { @@ -525,7 +635,7 @@ func TestCreateOrUpdateService(t *testing.T) { type opts struct { cr *vmv1beta1.VMAlert c *config.BaseOperatorConf - want func(svc *corev1.Service) error + validate func(svc *corev1.Service) predefinedObjects []runtime.Object } f := func(o opts) { @@ -540,7 +650,9 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: svc.Namespace, } assert.NoError(t, cl.Get(ctx, nsn, &got)) - assert.NoError(t, o.want(&got)) + if o.validate != nil { + o.validate(&got) + } } // base test @@ -552,11 +664,14 @@ func TestCreateOrUpdateService(t *testing.T) { Name: "base", }, }, - want: func(svc *corev1.Service) error { - if svc.Name != "vmalert-base" { - return fmt.Errorf("unexpected name for vmalert service: %v", svc.Name) - } - return nil + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmalert-base", svc.Name) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) } diff --git a/internal/controller/operator/factory/vmalertmanager/alertmanager_test.go b/internal/controller/operator/factory/vmalertmanager/alertmanager_test.go index ecc59bc48..f3ae41135 100644 --- a/internal/controller/operator/factory/vmalertmanager/alertmanager_test.go +++ b/internal/controller/operator/factory/vmalertmanager/alertmanager_test.go @@ -14,8 +14,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -23,7 +25,8 @@ import ( func TestCreateOrUpdateAlertManager(t *testing.T) { type opts struct { cr *vmv1beta1.VMAlertmanager - validate func(set *appsv1.StatefulSet) + cfgMutator func(*config.BaseOperatorConf) + validate func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) wantErr bool predefinedObjects []runtime.Object } @@ -33,6 +36,14 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { build.AddDefaults(fclient.Scheme()) fclient.Scheme().Default(o.cr) ctx := context.TODO() + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } err := CreateOrUpdateAlertManager(ctx, o.cr, fclient) if o.wantErr { assert.Error(t, err) @@ -40,9 +51,7 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { assert.NoError(t, err) } if o.validate != nil { - var got appsv1.StatefulSet - assert.NoError(t, fclient.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: o.cr.PrefixedName()}, &got)) - o.validate(&got) + o.validate(ctx, fclient, o.cr) } } @@ -63,7 +72,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Equal(t, set.Name, "vmalertmanager-test-am") assert.Equal(t, set.Spec.Template.Spec.Containers[0].Resources, corev1.ResourceRequirements{ Limits: corev1.ResourceList{ @@ -82,6 +93,15 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { "managed-by": "vm-operator", "main": "system", }) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "main": "system", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "test-am", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -103,7 +123,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Len(t, set.Spec.Template.Spec.Containers, 2) vmaContainer := set.Spec.Template.Spec.Containers[0] assert.Equal(t, vmaContainer.Name, "alertmanager") @@ -141,7 +163,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Equal(t, set.Name, "vmalertmanager-test-am") assert.Len(t, set.Spec.Template.Spec.Volumes, 4) templatesVolume := set.Spec.Template.Spec.Volumes[2] @@ -179,7 +203,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Len(t, set.Spec.Template.Spec.Containers, 2) vmaContainer := set.Spec.Template.Spec.Containers[0] @@ -214,7 +240,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Len(t, set.Spec.Template.Spec.Containers, 2) vmaContainer := set.Spec.Template.Spec.Containers[0] @@ -232,6 +260,77 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, "unexpected cluster peer arguments found") }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMAlertmanager{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAlertmanagerSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMAlertmanager{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }}) } func Test_createDefaultAMConfig(t *testing.T) { @@ -263,7 +362,7 @@ func Test_createDefaultAMConfig(t *testing.T) { } var amcfgs vmv1beta1.VMAlertmanagerConfigList - assert.Nil(t, fclient.List(ctx, &amcfgs)) + assert.NoError(t, fclient.List(ctx, &amcfgs)) for _, amcfg := range amcfgs.Items { assert.Equal(t, amcfg.Status.UpdateStatus, vmv1beta1.UpdateStatusOperational) } diff --git a/internal/controller/operator/factory/vmalertmanager/config_test.go b/internal/controller/operator/factory/vmalertmanager/config_test.go index 4bf799739..b8e83b09f 100644 --- a/internal/controller/operator/factory/vmalertmanager/config_test.go +++ b/internal/controller/operator/factory/vmalertmanager/config_test.go @@ -2105,7 +2105,7 @@ func Test_UpdateDefaultAMConfig(t *testing.T) { }, }, }) - assert.Nil(t, os.Setenv("WATCH_NAMESPACE", "default")) + assert.NoError(t, os.Setenv("WATCH_NAMESPACE", "default")) } func TestBuildWebConfig(t *testing.T) { diff --git a/internal/controller/operator/factory/vmanomaly/statefulset_test.go b/internal/controller/operator/factory/vmanomaly/statefulset_test.go index cd455e77d..dcfa233ce 100644 --- a/internal/controller/operator/factory/vmanomaly/statefulset_test.go +++ b/internal/controller/operator/factory/vmanomaly/statefulset_test.go @@ -18,6 +18,7 @@ import ( vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -25,6 +26,7 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1.VMAnomaly + cfgMutator func(*config.BaseOperatorConf) validate func(sts *appsv1.StatefulSet, idx int) wantErr bool predefinedObjects []runtime.Object @@ -35,6 +37,14 @@ func TestCreateOrUpdate(t *testing.T) { fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) build.AddDefaults(fclient.Scheme()) fclient.Scheme().Default(o.cr) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } err := CreateOrUpdate(ctx, o.cr, fclient) if o.wantErr { assert.Error(t, err) @@ -299,6 +309,126 @@ schedulers: assert.Equal(t, set.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels["shard-num"], strconv.Itoa(idx)) }, }) + + // managed metadata + f(opts{ + cr: &vmv1.VMAnomaly{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-anomaly", + Namespace: "monitoring", + Annotations: map[string]string{"not": "touch"}, + Labels: map[string]string{"main": "system"}, + }, + Spec: vmv1.VMAnomalySpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + License: &vmv1beta1.License{ + Key: ptr.To("test"), + }, + ConfigRawYaml: ` +reader: + queries: + query_alias2: + expr: vm_metric +writer: + datasource_url: "http://test.com" +models: + model_univariate_1: + class: 'zscore' + z_threshold: 2.5 + queries: ['query_alias2'] +schedulers: + scheduler_periodic_1m: + class: "scheduler.periodic.PeriodicScheduler" + infer_every: 1m + fit_every: 2m + fit_window: 3h +`, + Reader: &vmv1.VMAnomalyReadersSpec{ + DatasourceURL: "http://test.com", + SamplingPeriod: "1m", + }, + Writer: &vmv1.VMAnomalyWritersSpec{ + DatasourceURL: "http://write.endpoint", + }, + }, + }, + validate: func(set *appsv1.StatefulSet, _ int) { + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmanomaly", + "app.kubernetes.io/instance": "test-anomaly", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + }, + }) + + // common labels + f(opts{ + cr: &vmv1.VMAnomaly{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-anomaly", + Namespace: "monitoring", + Annotations: map[string]string{"not": "touch"}, + Labels: map[string]string{"main": "system"}, + }, + Spec: vmv1.VMAnomalySpec{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + License: &vmv1beta1.License{ + Key: ptr.To("test"), + }, + ConfigRawYaml: ` +reader: + queries: + query_alias2: + expr: vm_metric +writer: + datasource_url: "http://test.com" +models: + model_univariate_1: + class: 'zscore' + z_threshold: 2.5 + queries: ['query_alias2'] +schedulers: + scheduler_periodic_1m: + class: "scheduler.periodic.PeriodicScheduler" + infer_every: 1m + fit_every: 2m + fit_window: 3h +`, + Reader: &vmv1.VMAnomalyReadersSpec{ + DatasourceURL: "http://test.com", + SamplingPeriod: "1m", + }, + Writer: &vmv1.VMAnomalyWritersSpec{ + DatasourceURL: "http://write.endpoint", + }, + }, + }, + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + validate: func(set *appsv1.StatefulSet, _ int) { + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmanomaly", + "app.kubernetes.io/instance": "test-anomaly", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + }, + }) } func Test_createDefaultConfig(t *testing.T) { diff --git a/internal/controller/operator/factory/vmauth/vmauth.go b/internal/controller/operator/factory/vmauth/vmauth.go index e6d5144e7..1175a66d3 100644 --- a/internal/controller/operator/factory/vmauth/vmauth.go +++ b/internal/controller/operator/factory/vmauth/vmauth.go @@ -528,13 +528,13 @@ func buildIngressConfig(cr *vmv1beta1.VMAuth) *networkingv1.Ingress { // add user defined routes. spec.Rules = append(spec.Rules, cr.Spec.Ingress.ExtraRules...) spec.TLS = append(spec.TLS, cr.Spec.Ingress.ExtraTLS...) - lbls := labels.Merge(cr.Spec.Ingress.Labels, cr.SelectorLabels()) + lbls := labels.Merge(labels.Merge(cr.FinalLabels(), cr.Spec.Ingress.Labels), cr.SelectorLabels()) return &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: cr.PrefixedName(), Namespace: cr.Namespace, Labels: lbls, - Annotations: cr.Spec.Ingress.Annotations, + Annotations: labels.Merge(cr.FinalAnnotations(), cr.Spec.Ingress.Annotations), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, Spec: spec, diff --git a/internal/controller/operator/factory/vmauth/vmauth_test.go b/internal/controller/operator/factory/vmauth/vmauth_test.go index 42bdb791b..573ed4194 100644 --- a/internal/controller/operator/factory/vmauth/vmauth_test.go +++ b/internal/controller/operator/factory/vmauth/vmauth_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" + appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -90,6 +91,23 @@ func TestCreateOrUpdate(t *testing.T) { }, }, }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAuth) { + wantLabels := map[string]string{ + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + } + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, wantLabels, svc.Labels) + var secret corev1.Secret + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.ConfigSecretName()}, &secret)) + assert.Equal(t, wantLabels, secret.Labels) + var httpRoute gwapiv1.HTTPRoute + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &httpRoute)) + assert.Equal(t, wantLabels, httpRoute.Labels) + }, }) // simple-remove-httproute @@ -255,9 +273,14 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -330,9 +353,14 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1000", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -411,6 +439,95 @@ func TestCreateOrUpdate(t *testing.T) { assert.True(t, k8serrors.IsNotFound(err)) }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMAuth{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAuthSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAuth) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmauth-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + var secret corev1.Secret + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.ConfigSecretName()}, &secret)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, secret.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMAuth{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAuth) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmauth-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + var secret corev1.Secret + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.ConfigSecretName()}, &secret)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, secret.Labels) + }}) } func TestMakeSpecForAuthOk(t *testing.T) { diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_test.go b/internal/controller/operator/factory/vmcluster/vmcluster_test.go index 8451db9e4..71b687086 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster_test.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster_test.go @@ -431,9 +431,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, nsn, &vpaGot)) vpaExpected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: selectName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: selectName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/part-of": "vmcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -697,6 +703,113 @@ func TestCreateOrUpdate(t *testing.T) { assert.True(t, k8serrors.IsNotFound(err)) }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMClusterSpec{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMClusterSpec{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, svc.Labels) + }}) } func TestCreatOrUpdateClusterServices(t *testing.T) { @@ -775,6 +888,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true `) // with vmbackup and additional service ports @@ -820,6 +935,7 @@ objectmeta: spec: ports: - name: web-rpc + protocol: TCP port: 8011 targetport: intval: 8011 @@ -850,6 +966,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true `) @@ -893,6 +1011,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true `) // with native and extra service @@ -938,6 +1058,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true `) f(vmv1beta1.ClusterComponentInsert, &vmv1beta1.VMCluster{ @@ -984,6 +1106,8 @@ spec: managed-by: vm-operator clusterip: "" type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster `) // transit to headless f(vmv1beta1.ClusterComponentInsert, &vmv1beta1.VMCluster{ @@ -1043,6 +1167,8 @@ spec: managed-by: vm-operator clusterip: "None" type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster `, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "vminsert-test", @@ -1127,6 +1253,10 @@ spec: app.kubernetes.io/name: vminsert managed-by: vm-operator type: LoadBalancer + sessionaffinity: None + externaltrafficpolicy: Cluster + allocateloadbalancernodeports: true + internaltrafficpolicy: Cluster loadbalancerclass: service.k8s.aws/nlb `, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -1216,6 +1346,8 @@ spec: app.kubernetes.io/name: vminsert managed-by: vm-operator type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster clusterip: "None" loadbalancerclass: service.k8s.aws/nlb `, &corev1.Service{ diff --git a/internal/controller/operator/factory/vmsingle/vmsingle.go b/internal/controller/operator/factory/vmsingle/vmsingle.go index d7163b330..8ad7e6d4c 100644 --- a/internal/controller/operator/factory/vmsingle/vmsingle.go +++ b/internal/controller/operator/factory/vmsingle/vmsingle.go @@ -58,8 +58,8 @@ func makePvc(cr *vmv1beta1.VMSingle) *corev1.PersistentVolumeClaim { ObjectMeta: metav1.ObjectMeta{ Name: cr.PrefixedName(), Namespace: cr.Namespace, - Labels: labels.Merge(cr.Spec.StorageMetadata.Labels, cr.SelectorLabels()), - Annotations: cr.Spec.StorageMetadata.Annotations, + Labels: labels.Merge(labels.Merge(cr.FinalLabels(), cr.Spec.StorageMetadata.Labels), cr.SelectorLabels()), + Annotations: labels.Merge(cr.FinalAnnotations(), cr.Spec.StorageMetadata.Annotations), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, Spec: *cr.Spec.Storage, diff --git a/internal/controller/operator/factory/vmsingle/vmsingle_test.go b/internal/controller/operator/factory/vmsingle/vmsingle_test.go index 4e681431d..2e59b7db9 100644 --- a/internal/controller/operator/factory/vmsingle/vmsingle_test.go +++ b/internal/controller/operator/factory/vmsingle/vmsingle_test.go @@ -12,8 +12,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -21,14 +23,29 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1beta1.VMSingle - want *appsv1.Deployment + cfgMutator func(*config.BaseOperatorConf) + validate func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) predefinedObjects []runtime.Object } f := func(o opts) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) - assert.NoError(t, CreateOrUpdate(context.TODO(), o.cr, fclient)) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } + build.AddDefaults(fclient.Scheme()) + fclient.Scheme().Default(o.cr) + ctx := context.TODO() + assert.NoError(t, CreateOrUpdate(ctx, o.cr, fclient)) + if o.validate != nil { + o.validate(ctx, fclient, o.cr) + } } // base-vmsingle-gen @@ -50,7 +67,11 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vmsingle-vmsingle-base", "default"), }, - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vmsingle-vmsingle-base", Namespace: "default"}}, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vmsingle-vmsingle-base", got.Name) + }, }) // base-vmsingle-with-ports @@ -78,20 +99,107 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vmsingle-vmsingle-base", "default"), }, - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vmsingle-vmsingle-base", Namespace: "default"}}, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vmsingle-vmsingle-base", got.Name) + }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMSingleSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, got.Annotations) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, got.Annotations) + }}) + + // common labels cannot overwrite standard labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "hacked", + "app.kubernetes.io/instance": "hacked", + "app.kubernetes.io/component": "hacked", + "managed-by": "hacked", + } + }, + cr: &vmv1beta1.VMSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + }}) } func TestCreateOrUpdateService(t *testing.T) { type opts struct { cr *vmv1beta1.VMSingle - want *corev1.Service + validate func(*corev1.Service) predefinedObjects []runtime.Object } f := func(o opts) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) + build.AddDefaults(fclient.Scheme()) ctx := context.TODO() assert.NoError(t, createOrUpdateService(ctx, fclient, o.cr, nil)) svc := build.Service(o.cr, o.cr.Spec.Port, nil) @@ -101,8 +209,11 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: svc.Namespace, } assert.NoError(t, fclient.Get(ctx, nsn, &got)) - assert.Equal(t, got.Name, o.want.Name) - assert.Equal(t, got.Spec.Ports, o.want.Spec.Ports) + if o.validate != nil { + o.validate(&got) + } else { + assert.Equal(t, got.Name, svc.Name) + } } // base service test @@ -113,25 +224,30 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: "default", }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vmsingle-single-1", - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, { - Name: "http-alias", - Port: 8428, - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmsingle-single-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + + expectedPorts := []corev1.ServicePort{ + { + Name: "http", + Protocol: "TCP", + TargetPort: intstr.FromInt32(0), }, - }, + { + Name: "http-alias", + Port: 8428, + Protocol: "TCP", + TargetPort: intstr.FromInt32(8428), + }, + } + assert.Equal(t, expectedPorts, svc.Spec.Ports) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "single-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -151,60 +267,27 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vmsingle-single-1", - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, { - Name: "http-alias", - Protocol: "TCP", - Port: 8428, - TargetPort: intstr.FromString(""), - }, { - Name: "graphite-tcp", - Protocol: "TCP", - Port: 8053, - TargetPort: intstr.FromInt(8053), - }, { - Name: "graphite-udp", - Protocol: "UDP", - Port: 8053, - TargetPort: intstr.FromInt(8053), - }, { - Name: "influx-tcp", - Protocol: "TCP", - Port: 8051, - TargetPort: intstr.FromInt(8051), - }, { - Name: "influx-udp", - Protocol: "UDP", - Port: 8051, - TargetPort: intstr.FromInt(8051), - }, { - Name: "opentsdb-tcp", - Protocol: "TCP", - Port: 8054, - TargetPort: intstr.FromInt(8054), - }, { - Name: "opentsdb-udp", - Protocol: "UDP", - Port: 8054, - TargetPort: intstr.FromInt(8054), - }, { - Name: "opentsdb-http", - Protocol: "TCP", - Port: 8052, - TargetPort: intstr.FromInt(8052), - }, - }, - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmsingle-single-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + // sanity-check ports count and a couple of representative ports + assert.Len(t, svc.Spec.Ports, 9) + // check graphite tcp present + foundGraphite := false + for _, p := range svc.Spec.Ports { + if p.Name == "graphite-tcp" && p.Port == 8053 { + foundGraphite = true + break + } + } + assert.True(t, foundGraphite) + + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "single-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -224,25 +307,16 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vmsingle-single-1", - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, { - Name: "http-alias", - Port: 8428, - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, - }, - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmsingle-single-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + assert.Len(t, svc.Spec.Ports, 2) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "single-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, predefinedObjects: []runtime.Object{ &corev1.Service{ diff --git a/internal/controller/operator/factory/vtcluster/cluster_test.go b/internal/controller/operator/factory/vtcluster/cluster_test.go index ecd9030de..f87c25a34 100644 --- a/internal/controller/operator/factory/vtcluster/cluster_test.go +++ b/internal/controller/operator/factory/vtcluster/cluster_test.go @@ -91,35 +91,86 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTCluster) { // ensure SA created var sa corev1.ServiceAccount - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.GetServiceAccountName(), Namespace: cr.Namespace}, &sa)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.GetServiceAccountName(), Namespace: cr.Namespace}, &sa)) assert.Nil(t, sa.Annotations) - assert.Equal(t, sa.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentRoot)) + assert.Equal(t, sa.Labels, map[string]string{ + "app.kubernetes.io/name": "vtcluster", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check insert var dep appsv1.Deployment - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt := dep.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:10481", "-internalselect.disable=true", "-storageNode=vtstorage-base-0.vtstorage-base.default:10491,vtstorage-base-1.vtstorage-base.default:10491"}) assert.Nil(t, dep.Annotations) - assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentInsert)) + assert.Equal(t, dep.Labels, map[string]string{ + "app.kubernetes.io/name": "vtinsert", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check select - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt = dep.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:10471", "-internalinsert.disable=true", "-storageNode=vtstorage-base-0.vtstorage-base.default:10491,vtstorage-base-1.vtstorage-base.default:10491"}) assert.Nil(t, dep.Annotations) - assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentSelect)) + assert.Equal(t, dep.Labels, map[string]string{ + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check storage var sts appsv1.StatefulSet - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt = sts.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:10491", "-storageDataPath=/vtstorage-data"}) assert.Nil(t, sts.Annotations) - assert.Equal(t, sts.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentStorage)) + assert.Equal(t, sts.Labels, map[string]string{ + "app.kubernetes.io/name": "vtstorage", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + + // check services + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtinsert", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtstorage", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -144,7 +195,7 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTCluster) { // check storage var sts appsv1.StatefulSet - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt := sts.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-httpListenAddr=:10491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vtstorage-data"}) @@ -213,9 +264,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(vmv1beta1.ClusterComponentInsert), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vtinsert", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -277,9 +334,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -339,9 +402,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vtstorage", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -420,9 +489,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vtinsert", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1000", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -508,4 +583,111 @@ func TestCreateOrUpdate(t *testing.T) { assert.True(t, k8serrors.IsNotFound(err)) }, }) + // managed metadata + f(opts{ + cr: &vmv1.VTCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VTClusterSpec{ + Select: &vmv1.VTSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + Insert: &vmv1.VTInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + Storage: &vmv1.VTStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTCluster) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vtselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vtcluster", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1.VTCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VTClusterSpec{ + Select: &vmv1.VTSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + Insert: &vmv1.VTInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + Storage: &vmv1.VTStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTCluster) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vtselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vtcluster", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) } diff --git a/internal/controller/operator/factory/vtsingle/vtsingle.go b/internal/controller/operator/factory/vtsingle/vtsingle.go index b3d690c4e..9860f4e72 100644 --- a/internal/controller/operator/factory/vtsingle/vtsingle.go +++ b/internal/controller/operator/factory/vtsingle/vtsingle.go @@ -48,8 +48,8 @@ func newPVC(r *vmv1.VTSingle) *corev1.PersistentVolumeClaim { ObjectMeta: metav1.ObjectMeta{ Name: r.PrefixedName(), Namespace: r.Namespace, - Labels: labels.Merge(r.Spec.StorageMetadata.Labels, r.SelectorLabels()), - Annotations: r.Spec.StorageMetadata.Annotations, + Labels: labels.Merge(labels.Merge(r.FinalLabels(), r.Spec.StorageMetadata.Labels), r.SelectorLabels()), + Annotations: labels.Merge(r.FinalAnnotations(), r.Spec.StorageMetadata.Annotations), OwnerReferences: []metav1.OwnerReference{r.AsOwner()}, }, Spec: *r.Spec.Storage, diff --git a/internal/controller/operator/factory/vtsingle/vtsingle_test.go b/internal/controller/operator/factory/vtsingle/vtsingle_test.go index ad44ff663..97322275b 100644 --- a/internal/controller/operator/factory/vtsingle/vtsingle_test.go +++ b/internal/controller/operator/factory/vtsingle/vtsingle_test.go @@ -5,11 +5,13 @@ import ( "testing" "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" @@ -21,25 +23,39 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1.VTSingle - c *config.BaseOperatorConf + cfgMutator func(*config.BaseOperatorConf) + validate func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) wantErr bool predefinedObjects []runtime.Object } - f := func(opts opts) { + f := func(o opts) { t.Helper() - fclient := k8stools.GetTestClientWithObjects(opts.predefinedObjects) - err := CreateOrUpdate(context.TODO(), fclient, opts.cr) - if opts.wantErr { + fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) + build.AddDefaults(fclient.Scheme()) + fclient.Scheme().Default(o.cr) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } + ctx := context.TODO() + err := CreateOrUpdate(ctx, fclient, o.cr) + if o.wantErr { assert.Error(t, err) } else { assert.NoError(t, err) } + if o.validate != nil { + o.validate(ctx, fclient, o.cr) + } } // base gen - o := opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -67,12 +83,10 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vtsingle-base", "default"), }, - } - f(o) + }) // base with specific port - o = opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -101,12 +115,10 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vtsingle-base", "default"), }, - } - f(o) + }) // with syslog tls config - o = opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -135,17 +147,68 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vtsingle-base", "default"), }, - } - f(o) + }) + + // managed metadata + f(opts{ + cr: &vmv1.VTSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VTSingleSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, got.Annotations) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1.VTSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, got.Annotations) + }, + }) } func TestCreateOrUpdateService(t *testing.T) { type opts struct { cr *vmv1.VTSingle - c *config.BaseOperatorConf - want *corev1.Service + validate func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) wantErr bool - wantPortsLen int predefinedObjects []runtime.Object } @@ -158,39 +221,36 @@ func TestCreateOrUpdateService(t *testing.T) { return } assert.NoError(t, err) - svc := build.Service(o.cr, o.cr.Spec.Port, nil) - var got corev1.Service - nsn := types.NamespacedName{ - Name: svc.Name, - Namespace: svc.Namespace, + if o.validate != nil { + o.validate(ctx, fclient, o.cr) } - assert.NoError(t, fclient.Get(ctx, nsn, &got)) - assert.Equal(t, got.Name, o.want.Name) - assert.Len(t, got.Spec.Ports, o.wantPortsLen) } // base service test - o := opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "traces-1", Namespace: "default", }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vtsingle-traces-1", - Namespace: "default", - }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) { + var got corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(), Namespace: cr.Namespace}, &got)) + assert.Equal(t, "vtsingle-traces-1", got.Name) + assert.Equal(t, "default", got.Namespace) + assert.Len(t, got.Spec.Ports, 1) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtsingle", + "app.kubernetes.io/instance": "traces-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) }, - wantPortsLen: 1, - } - f(o) + }) // with extra service nodePort - o = opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "traces-1", @@ -205,13 +265,19 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vtsingle-traces-1", - Namespace: "default", - }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) { + var got corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(), Namespace: cr.Namespace}, &got)) + assert.Equal(t, "vtsingle-traces-1", got.Name) + assert.Equal(t, "default", got.Namespace) + assert.Len(t, got.Spec.Ports, 1) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtsingle", + "app.kubernetes.io/instance": "traces-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) }, - wantPortsLen: 1, predefinedObjects: []runtime.Object{ &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -227,6 +293,5 @@ func TestCreateOrUpdateService(t *testing.T) { Spec: corev1.ServiceSpec{}, }, }, - } - f(o) + }) }