diff --git a/Makefile b/Makefile index 749b3f2524..10836ae3bc 100644 --- a/Makefile +++ b/Makefile @@ -408,6 +408,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-supervisor-runtimesdk.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology/cluster-template-topology-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-autoscaler" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-autoscaler-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-fast-rollout-supervisor.yaml" diff --git a/apis/vmware/v1beta1/vspheremachinetemplate_types.go b/apis/vmware/v1beta1/vspheremachinetemplate_types.go index 6f36e10587..3bfbc2136f 100644 --- a/apis/vmware/v1beta1/vspheremachinetemplate_types.go +++ b/apis/vmware/v1beta1/vspheremachinetemplate_types.go @@ -18,24 +18,44 @@ limitations under the License. package v1beta1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + // VSphereResourceCPU defines Resource type CPU for VSphereMachines. + VSphereResourceCPU corev1.ResourceName = "cpu" + + // VSphereResourceMemory defines Resource type memory for VSphereMachines. + VSphereResourceMemory corev1.ResourceName = "memory" +) + // VSphereMachineTemplateSpec defines the desired state of VSphereMachineTemplate. type VSphereMachineTemplateSpec struct { Template VSphereMachineTemplateResource `json:"template"` } +// VSphereMachineTemplateStatus defines the observed state of VSphereMachineTemplate. +type VSphereMachineTemplateStatus struct { + // Capacity defines the resource capacity for this VSphereMachineTemplate. + // This value is used for autoscaling from zero operations as defined in: + // https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md + // +optional + Capacity corev1.ResourceList `json:"capacity,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:resource:path=vspheremachinetemplates,scope=Namespaced,categories=cluster-api // +kubebuilder:storageversion +// +kubebuilder:subresource:status // VSphereMachineTemplate is the Schema for the vspheremachinetemplates API. type VSphereMachineTemplate struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec VSphereMachineTemplateSpec `json:"spec,omitempty"` + Spec VSphereMachineTemplateSpec `json:"spec,omitempty"` + Status VSphereMachineTemplateStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/vmware/v1beta1/zz_generated.deepcopy.go b/apis/vmware/v1beta1/zz_generated.deepcopy.go index 565bcc8b36..09e4af4637 100644 --- a/apis/vmware/v1beta1/zz_generated.deepcopy.go +++ b/apis/vmware/v1beta1/zz_generated.deepcopy.go @@ -466,6 +466,7 @@ func (in *VSphereMachineTemplate) DeepCopyInto(out *VSphereMachineTemplate) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereMachineTemplate. @@ -550,6 +551,28 @@ func (in *VSphereMachineTemplateSpec) DeepCopy() *VSphereMachineTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VSphereMachineTemplateStatus) DeepCopyInto(out *VSphereMachineTemplateStatus) { + *out = *in + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereMachineTemplateStatus. +func (in *VSphereMachineTemplateStatus) DeepCopy() *VSphereMachineTemplateStatus { + if in == nil { + return nil + } + out := new(VSphereMachineTemplateStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VSphereMachineVolume) DeepCopyInto(out *VSphereMachineVolume) { *out = *in diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0a1a04a20b..6d5635b0aa 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -384,6 +384,14 @@ rules: - get - list - watch +- apiGroups: + - vmoperator.vmware.com + resources: + - virtualmachineclasses + verbs: + - get + - list + - watch - apiGroups: - vmoperator.vmware.com resources: diff --git a/config/supervisor/crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml b/config/supervisor/crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml index b11c86f071..f2c8d4f433 100644 --- a/config/supervisor/crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml +++ b/config/supervisor/crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml @@ -163,6 +163,25 @@ spec: required: - template type: object + status: + description: VSphereMachineTemplateStatus defines the observed state of + VSphereMachineTemplate. + properties: + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Capacity defines the resource capacity for this VSphereMachineTemplate. + This value is used for autoscaling from zero operations as defined in: + https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md + type: object + type: object type: object served: true storage: true + subresources: + status: {} diff --git a/controllers/vmware/test/controllers_test.go b/controllers/vmware/test/controllers_test.go index 44e3298156..17e068daa2 100644 --- a/controllers/vmware/test/controllers_test.go +++ b/controllers/vmware/test/controllers_test.go @@ -44,6 +44,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" "sigs.k8s.io/cluster-api-provider-vsphere/controllers" + "sigs.k8s.io/cluster-api-provider-vsphere/controllers/vmware" vmwarewebhooks "sigs.k8s.io/cluster-api-provider-vsphere/internal/webhooks/vmware" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/constants" capvcontext "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context" @@ -259,6 +260,10 @@ func getManager(cfg *rest.Config, networkProvider string, withWebhooks bool) man return err } } + if err := vmware.AddVSphereMachineTemplateControllerToManager(ctx, controllerCtx, mgr, controllerOpts); err != nil { + return err + } + return controllers.AddMachineControllerToManager(ctx, controllerCtx, mgr, true, controllerOpts) } diff --git a/controllers/vmware/vspheremachinetemplate_controller.go b/controllers/vmware/vspheremachinetemplate_controller.go new file mode 100644 index 0000000000..97114e4529 --- /dev/null +++ b/controllers/vmware/vspheremachinetemplate_controller.go @@ -0,0 +1,121 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmware + +import ( + "context" + + "github.com/pkg/errors" + vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/cluster-api/util/predicates" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" + capvcontext "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context" +) + +// +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vspheremachinetemplates,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vspheremachinetemplates/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineclasses,verbs=get;list;watch + +// AddVSphereMachineTemplateControllerToManager adds the machine template controller to the provided +// manager. +func AddVSphereMachineTemplateControllerToManager(ctx context.Context, controllerManagerContext *capvcontext.ControllerManagerContext, mgr manager.Manager, options controller.Options) error { + r := &vSphereMachineTemplateReconciler{ + Client: controllerManagerContext.Client, + } + + return ctrl.NewControllerManagedBy(mgr). + For(&vmwarev1.VSphereMachineTemplate{}). + WithOptions(options). + Watches( + &vmoprv1.VirtualMachineClass{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueVirtualMachineClassToVSphereMachineTemplateRequests), + ). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), controllerManagerContext.WatchFilterValue)). + Complete(r) +} + +type vSphereMachineTemplateReconciler struct { + Client client.Client +} + +func (r *vSphereMachineTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { + // Fetch VSphereMachineTemplate object + vSphereMachineTemplate := &vmwarev1.VSphereMachineTemplate{} + if err := r.Client.Get(ctx, req.NamespacedName, vSphereMachineTemplate); err != nil { + if apierrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + // Fetch the VirtualMachineClass + vmClass := &vmoprv1.VirtualMachineClass{} + if err := r.Client.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: vSphereMachineTemplate.Spec.Template.Spec.ClassName}, vmClass); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to get VirtualMachineClass %q for VSphereMachineTemplate", vSphereMachineTemplate.Spec.Template.Spec.ClassName) + } + + patchHelper, err := patch.NewHelper(vSphereMachineTemplate, r.Client) + if err != nil { + return reconcile.Result{}, err + } + + if vSphereMachineTemplate.Status.Capacity == nil { + vSphereMachineTemplate.Status.Capacity = corev1.ResourceList{} + } + if vmClass.Spec.Hardware.Cpus > 0 { + vSphereMachineTemplate.Status.Capacity[vmwarev1.VSphereResourceCPU] = *resource.NewQuantity(vmClass.Spec.Hardware.Cpus, resource.DecimalSI) + } + if !vmClass.Spec.Hardware.Memory.IsZero() { + vSphereMachineTemplate.Status.Capacity[vmwarev1.VSphereResourceMemory] = vmClass.Spec.Hardware.Memory + } + + return reconcile.Result{}, patchHelper.Patch(ctx, vSphereMachineTemplate) +} + +// enqueueVirtualMachineClassToVSphereMachineTemplateRequests returns a list of VSphereMachineTemplate reconcile requests +// that use a specific VirtualMachineClass. +func (r *vSphereMachineTemplateReconciler) enqueueVirtualMachineClassToVSphereMachineTemplateRequests(ctx context.Context, virtualMachineClass client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + vSphereMachineTemplates := &vmwarev1.VSphereMachineTemplateList{} + if err := r.Client.List(ctx, vSphereMachineTemplates, client.InNamespace(virtualMachineClass.GetNamespace())); err != nil { + return nil + } + + for _, vSphereMachineTemplate := range vSphereMachineTemplates.Items { + if vSphereMachineTemplate.Spec.Template.Spec.ClassName != virtualMachineClass.GetName() { + continue + } + + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKey{Namespace: vSphereMachineTemplate.Namespace, Name: vSphereMachineTemplate.Name}, + }) + } + + return requests +} diff --git a/controllers/vmware/vspheremachinetemplate_controller_test.go b/controllers/vmware/vspheremachinetemplate_controller_test.go new file mode 100644 index 0000000000..cc7de6534c --- /dev/null +++ b/controllers/vmware/vspheremachinetemplate_controller_test.go @@ -0,0 +1,200 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmware + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" +) + +func Test_vSphereMachineTemplateReconciler_Reconcile(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + scheme := runtime.NewScheme() + g.Expect(corev1.AddToScheme(scheme)).To(Succeed()) + g.Expect(vmwarev1.AddToScheme(scheme)).To(Succeed()) + g.Expect(vmoprv1.AddToScheme(scheme)).To(Succeed()) + + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + }, + } + + tests := []struct { + name string + vSphereMachineTemplate *vmwarev1.VSphereMachineTemplate + objects []client.Object + wantErr string + wantStatus *vmwarev1.VSphereMachineTemplateStatus + }{ + { + name: "object does not exist", + vSphereMachineTemplate: nil, + objects: []client.Object{}, + wantErr: "", + wantStatus: nil, + }, + { + name: "VirtualMachineClass does not exist", + vSphereMachineTemplate: vSphereMachineTemplate(namespace.Name, "no-class", "not-existing-class", nil), + objects: []client.Object{}, + wantErr: "failed to get VirtualMachineClass \"not-existing-class\" for VSphereMachineTemplate", + wantStatus: nil, + }, + { + name: "VirtualMachineClass does exist but has no data", + vSphereMachineTemplate: vSphereMachineTemplate(namespace.Name, "with-class", "vm-class", nil), + objects: []client.Object{ + virtualMachineClass(namespace.Name, "vm-class", nil), + }, + wantErr: "", + wantStatus: &vmwarev1.VSphereMachineTemplateStatus{}, + }, + { + name: "VirtualMachineClass does exist and has cpu and memory set", + vSphereMachineTemplate: vSphereMachineTemplate(namespace.Name, "with-class", "vm-class", nil), + objects: []client.Object{ + virtualMachineClass(namespace.Name, "vm-class", &vmoprv1.VirtualMachineClassHardware{Cpus: 1, Memory: quantity(1024)}), + }, + wantErr: "", + wantStatus: &vmwarev1.VSphereMachineTemplateStatus{ + Capacity: corev1.ResourceList{ + corev1.ResourceCPU: quantity(1), + corev1.ResourceMemory: quantity(1024), + }, + }, + }, + { + name: "VirtualMachineClass got updated to new cpu and memory values", + vSphereMachineTemplate: vSphereMachineTemplate(namespace.Name, "with-class", "vm-class", &vmwarev1.VSphereMachineTemplateStatus{ + Capacity: corev1.ResourceList{ + corev1.ResourceCPU: quantity(1), + corev1.ResourceMemory: quantity(1024), + }, + }), + objects: []client.Object{ + virtualMachineClass(namespace.Name, "vm-class", &vmoprv1.VirtualMachineClassHardware{Cpus: 2, Memory: quantity(2048)}), + }, + wantErr: "", + wantStatus: &vmwarev1.VSphereMachineTemplateStatus{ + Capacity: corev1.ResourceList{ + corev1.ResourceCPU: quantity(2), + corev1.ResourceMemory: quantity(2048), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + fakeClientBuilder := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(append([]client.Object{namespace}, tt.objects...)...) + + vSphereMachineTemplateName := "not-exists" + if tt.vSphereMachineTemplate != nil { + vSphereMachineTemplateName = tt.vSphereMachineTemplate.GetName() + fakeClientBuilder = fakeClientBuilder. + WithObjects(tt.vSphereMachineTemplate). + WithStatusSubresource(tt.vSphereMachineTemplate) + } + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: vSphereMachineTemplateName, + Namespace: namespace.Name, + }, + } + + r := &vSphereMachineTemplateReconciler{ + Client: fakeClientBuilder.Build(), + } + + _, err := r.Reconcile(ctx, req) + if tt.wantErr == "" { + g.Expect(err).ToNot(HaveOccurred()) + } else { + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + } + + if tt.wantStatus != nil { + vSphereMachineTemplate := &vmwarev1.VSphereMachineTemplate{} + g.Expect(r.Client.Get(ctx, req.NamespacedName, vSphereMachineTemplate)).To(Succeed()) + g.Expect(vSphereMachineTemplate.Status).To(BeComparableTo(*tt.wantStatus)) + } + }) + } +} + +func vSphereMachineTemplate(namespace, name, className string, status *vmwarev1.VSphereMachineTemplateStatus) *vmwarev1.VSphereMachineTemplate { + tpl := &vmwarev1.VSphereMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: vmwarev1.VSphereMachineTemplateSpec{ + Template: vmwarev1.VSphereMachineTemplateResource{ + Spec: vmwarev1.VSphereMachineSpec{ + ClassName: className, + }, + }, + }, + } + + if status != nil { + tpl.Status = *status + } + + return tpl +} + +func virtualMachineClass(namespace, name string, hardware *vmoprv1.VirtualMachineClassHardware) *vmoprv1.VirtualMachineClass { + class := &vmoprv1.VirtualMachineClass{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + + if hardware != nil { + class.Spec.Hardware = *hardware + } + + return class +} + +func quantity(i int64) resource.Quantity { + q := resource.NewQuantity(i, resource.DecimalSI) + // Execute q.String to populate the internal s field + _ = q.String() + return *q +} diff --git a/go.mod b/go.mod index 37355e86f0..83adc38931 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module sigs.k8s.io/cluster-api-provider-vsphere go 1.22.0 -replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.8.1 +replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.8.2-0.20240826111923-18d9dd9bd8c3 replace github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels v0.0.0-20240404200847-de75746a9505 diff --git a/go.sum b/go.sum index 061d3f6eaa..0861a17fce 100644 --- a/go.sum +++ b/go.sum @@ -487,8 +487,8 @@ k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6R k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.0 h1:Tc9rS7JJoZ9sl3OpL4842oIk6lH7gWBb0JOmJ0ute7M= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.0/go.mod h1:1ewhL9l1gkPcU/IU/6rFYfikf+7Y5imWv7ARVbBOzNs= -sigs.k8s.io/cluster-api v1.8.1 h1:OA3w1CjCmXXXDL7aY3WDe+seL0mdFVJX1K5mZwqKbDE= -sigs.k8s.io/cluster-api v1.8.1/go.mod h1:pXv5LqLxuIbhGIXykyNKiJh+KrLweSBajVHHitPLyoY= +sigs.k8s.io/cluster-api v1.8.2-0.20240826111923-18d9dd9bd8c3 h1:gZD+Ne56bdSDogSw1SNDk6iLFSX3iWxD8pHBfiVRiI8= +sigs.k8s.io/cluster-api v1.8.2-0.20240826111923-18d9dd9bd8c3/go.mod h1:pXv5LqLxuIbhGIXykyNKiJh+KrLweSBajVHHitPLyoY= sigs.k8s.io/controller-runtime v0.18.5 h1:nTHio/W+Q4aBlQMgbnC5hZb4IjIidyrizMai9P6n4Rk= sigs.k8s.io/controller-runtime v0.18.5/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/main.go b/main.go index 01c1e36116..4da1a7ded6 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" "sigs.k8s.io/cluster-api-provider-vsphere/controllers" + "sigs.k8s.io/cluster-api-provider-vsphere/controllers/vmware" "sigs.k8s.io/cluster-api-provider-vsphere/feature" "sigs.k8s.io/cluster-api-provider-vsphere/internal/webhooks" vmwarewebhooks "sigs.k8s.io/cluster-api-provider-vsphere/internal/webhooks/vmware" @@ -81,6 +82,7 @@ var ( clusterCacheTrackerConcurrency int vSphereClusterConcurrency int vSphereMachineConcurrency int + vSphereMachineTemplateConcurrency int providerServiceAccountConcurrency int serviceDiscoveryConcurrency int vSphereVMConcurrency int @@ -115,6 +117,9 @@ func InitFlags(fs *pflag.FlagSet) { fs.IntVar(&vSphereMachineConcurrency, "vspheremachine-concurrency", 10, "Number of vSphere machines to process simultaneously") + fs.IntVar(&vSphereMachineTemplateConcurrency, "vspheremachinetemplate-concurrency", 10, + "Number of vSphere machine templates to process simultaneously") + fs.IntVar(&providerServiceAccountConcurrency, "providerserviceaccount-concurrency", 10, "Number of provider service accounts to process simultaneously") @@ -399,6 +404,10 @@ func setupSupervisorControllers(ctx context.Context, controllerCtx *capvcontext. return err } + if err := vmware.AddVSphereMachineTemplateControllerToManager(ctx, controllerCtx, mgr, concurrency(vSphereMachineTemplateConcurrency)); err != nil { + return err + } + if err := controllers.AddServiceAccountProviderControllerToManager(ctx, controllerCtx, mgr, tracker, concurrency(providerServiceAccountConcurrency)); err != nil { return err } diff --git a/test/e2e/autoscaler_test.go b/test/e2e/autoscaler_test.go new file mode 100644 index 0000000000..e99908e8c0 --- /dev/null +++ b/test/e2e/autoscaler_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + "k8s.io/utils/ptr" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" +) + +var _ = Describe("When using the autoscaler with Cluster API using ClusterClass and scale to zero [supervisor] [ClusterClass]", func() { + const specName = "autoscaler" // aligned to CAPI + Setup(specName, func(testSpecificSettingsGetter func() testSettings) { + capi_e2e.AutoscalerSpec(ctx, func() capi_e2e.AutoscalerSpecInput { + return capi_e2e.AutoscalerSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: testSpecificSettingsGetter().ClusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode("topology-autoscaler")), + PostNamespaceCreated: func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) { + if testMode == GovmomiTestMode { + // This test is only implemented for supervisor + Skip("This test is only implemented for supervisor") + } + testSpecificSettingsGetter().PostNamespaceCreatedFunc(managementClusterProxy, workloadClusterNamespace) + }, + InfrastructureAPIGroup: "vmware.infrastructure.cluster.x-k8s.io", + InfrastructureMachineTemplateKind: "vspheremachinetemplates", + AutoscalerVersion: "v1.30.0", + ScaleToAndFromZero: true, + // We have no connectivity from the workload cluster to the kind management cluster in CI so we + // can't deploy the autoscaler to the workload cluster. + InstallOnManagementCluster: true, + } + }) + }) +}) diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index b8b88ee97f..6d6706f200 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -156,6 +156,7 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-quick-start.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-quick-start-runtimesdk.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-supervisor.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-autoscaler-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-runtimesdk-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml" @@ -258,6 +259,7 @@ variables: KUBERNETES_VERSION_LATEST_CI: "ci/latest-1.32" CPI_IMAGE_K8S_VERSION: "v1.31.0" CNI: "./data/cni/calico/calico.yaml" + AUTOSCALER_WORKLOAD: "./data/autoscaler/autoscaler-to-management-workload.yaml" EXP_CLUSTER_RESOURCE_SET: "true" EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" CONTROL_PLANE_MACHINE_COUNT: 1 @@ -272,11 +274,15 @@ variables: VSPHERE_STORAGE_POLICY: "Cluster API vSphere Storage Policy" VSPHERE_STORAGE_CLASS: "test-storageclass" VSPHERE_MACHINE_CLASS_NAME: "test-machine-class" - VSPHERE_MACHINE_CLASS_CPU: "4" - VSPHERE_MACHINE_CLASS_MEMORY: "8Gi" + # CI runs vm-operator v1.8.6 on vCenter 8, setting CPU and Memory does not work and defaults to 2 CPUs / 2Gi memory. + # Changing the value results in VM's getting created but not setting the correct CPU or memory size. + VSPHERE_MACHINE_CLASS_CPU: "2" + VSPHERE_MACHINE_CLASS_MEMORY: "2Gi" VSPHERE_MACHINE_CLASS_NAME_CONFORMANCE: "test-machine-class-conformance" - VSPHERE_MACHINE_CLASS_CPU_CONFORMANCE: "8" - VSPHERE_MACHINE_CLASS_MEMORY_CONFORMANCE: "8Gi" + # CI runs vm-operator v1.8.6 on vCenter 8, setting CPU and Memory does not work and defaults to 2 CPUs / 2Gi memory. + # Changing the value results in VM's getting created but not setting the correct CPU or memory size. + VSPHERE_MACHINE_CLASS_CPU_CONFORMANCE: "2" + VSPHERE_MACHINE_CLASS_MEMORY_CONFORMANCE: "2Gi" VSPHERE_CONTENT_LIBRARY: "capv" VSPHERE_CONTENT_LIBRARY_ITEMS: "ubuntu-2204-kube-v1.28.0,ubuntu-2204-kube-v1.29.0,ubuntu-2204-kube-v1.30.0,ubuntu-2404-kube-v1.31.0" VSPHERE_IMAGE_NAME: "ubuntu-2404-kube-v1.31.0" @@ -302,6 +308,7 @@ variables: SERVICE_ACCOUNTS_CM_NAME: "service-accounts-cm" intervals: + default/wait-autoscaler: ["5m", "10s"] default/wait-controllers: ["5m", "10s"] default/wait-cluster: ["5m", "10s"] default/wait-control-plane: ["10m", "10s"] diff --git a/test/e2e/data/autoscaler/autoscaler-to-management-workload.yaml b/test/e2e/data/autoscaler/autoscaler-to-management-workload.yaml new file mode 100644 index 0000000000..16f1e09c57 --- /dev/null +++ b/test/e2e/data/autoscaler/autoscaler-to-management-workload.yaml @@ -0,0 +1,89 @@ +# This yaml deploys the autoscaler on a workload cluster and configures it to match +# against the corresponding Cluster API cluster which is defined into the management cluster. +--- +# Specify kubeconfig for management cluster +apiVersion: v1 +kind: Secret +metadata: + name: kubeconfig-management-cluster + namespace: ${CLUSTER_NAMESPACE} +stringData: + kubeconfig: | + apiVersion: v1 + kind: Config + clusters: + - name: management-cluster + cluster: + certificate-authority-data: ${MANAGEMENT_CLUSTER_CA} + server: ${MANAGEMENT_CLUSTER_ADDRESS} + contexts: + - name: management-context + context: + cluster: management-cluster + namespace: ${CLUSTER_NAMESPACE} + user: cluster-autoscaler-sa + current-context: management-context + users: + - name: cluster-autoscaler-sa + user: + token: "${MANAGEMENT_CLUSTER_TOKEN}" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: ${CLUSTER_NAMESPACE} + labels: + app: cluster-autoscaler +spec: + selector: + matchLabels: + app: cluster-autoscaler + replicas: 1 + template: + metadata: + labels: + app: cluster-autoscaler + spec: + containers: + - image: registry.k8s.io/autoscaling/cluster-autoscaler:${AUTOSCALER_VERSION} + name: cluster-autoscaler + command: + - /cluster-autoscaler + args: + - --cloud-provider=clusterapi + # Specify kubeconfig for management cluster + - --cloud-config=/management-cluster/kubeconfig + # Specify kubeconfig for workload cluster + - --kubeconfig=/workload-cluster/value + # Limit cluster autoscaler to only match against resources belonging to a single Cluster API cluster + - --node-group-auto-discovery=clusterapi:namespace=${CLUSTER_NAMESPACE},clusterName=${CLUSTER_NAME} + # Set a short scale down unneeded time, so we don't have to wait too long during e2e testing + - --scale-down-unneeded-time=1m + # Set a short scale down delay after add time, so we don't have to wait too long during e2e testing + - --scale-down-delay-after-add=1m + # Set a short scale down delay after delete time, so we don't have to wait too long during e2e testing + - --scale-down-delay-after-delete=1m + # Set a short scale down delay after failure time, so we don't have to wait too long during e2e testing + - --scale-down-delay-after-failure=1m + # Set a max nodes limit as safeguard so that the test does not scale up unbounded. + # Note: The E2E test should only go up to 4 (assuming it starts with a min node group size of 2). + # Using 6 for additional some buffer and to allow different starting min node group sizes. + - --max-nodes-total=6 + volumeMounts: + - name: kubeconfig-management-cluster + mountPath: /management-cluster + readOnly: true + - name: kubeconfig-workload-cluster + mountPath: /workload-cluster + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: kubeconfig-management-cluster + secret: + secretName: kubeconfig-management-cluster + optional: false + - name: kubeconfig-workload-cluster + secret: + secretName: ${CLUSTER_NAME}-kubeconfig + optional: false diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-autoscaler/cluster-autoscaler.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-autoscaler/cluster-autoscaler.yaml new file mode 100644 index 0000000000..b25f6d952e --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-autoscaler/cluster-autoscaler.yaml @@ -0,0 +1,9 @@ +- op: replace + path: /spec/topology/workers/machineDeployments/0/metadata + value: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "2" +- op: remove + path: /spec/topology/workers/machineDeployments/0/replicas + diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-autoscaler/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-autoscaler/kustomization.yaml new file mode 100644 index 0000000000..8efca71355 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-autoscaler/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../topology +patches: + - target: + kind: Cluster + path: ./cluster-autoscaler.yaml diff --git a/test/go.mod b/test/go.mod index d782e7a697..0b17344db6 100644 --- a/test/go.mod +++ b/test/go.mod @@ -2,9 +2,9 @@ module sigs.k8s.io/cluster-api-provider-vsphere/test go 1.22.0 -replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.8.1 +replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.8.2-0.20240826111923-18d9dd9bd8c3 -replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.8.1 +replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.8.2-0.20240826111923-18d9dd9bd8c3 replace sigs.k8s.io/cluster-api-provider-vsphere => ../ @@ -41,7 +41,7 @@ require ( ) require ( - github.com/BurntSushi/toml v1.0.0 // indirect + github.com/BurntSushi/toml v1.4.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect @@ -50,7 +50,7 @@ require ( github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/adrg/xdg v0.5.0 // indirect - github.com/alessio/shellescape v1.4.1 // indirect + github.com/alessio/shellescape v1.4.2 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -171,6 +171,6 @@ require ( k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/kind v0.23.0 // indirect + sigs.k8s.io/kind v0.24.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/test/go.sum b/test/go.sum index 94b5d40ceb..7962b1a6f9 100644 --- a/test/go.sum +++ b/test/go.sum @@ -1,7 +1,7 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= -github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -21,8 +21,8 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY= github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4= -github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= -github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= +github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -524,16 +524,16 @@ k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6R k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.0 h1:Tc9rS7JJoZ9sl3OpL4842oIk6lH7gWBb0JOmJ0ute7M= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.0/go.mod h1:1ewhL9l1gkPcU/IU/6rFYfikf+7Y5imWv7ARVbBOzNs= -sigs.k8s.io/cluster-api v1.8.1 h1:OA3w1CjCmXXXDL7aY3WDe+seL0mdFVJX1K5mZwqKbDE= -sigs.k8s.io/cluster-api v1.8.1/go.mod h1:pXv5LqLxuIbhGIXykyNKiJh+KrLweSBajVHHitPLyoY= -sigs.k8s.io/cluster-api/test v1.8.1 h1:f8T5mPVwNxxFYhdWARJcbtf3hZIqAzSmD/L2WdiEgmI= -sigs.k8s.io/cluster-api/test v1.8.1/go.mod h1:tOLF0saxjx08hoQfXOR484cK+sMmusSGhh1209klXF8= +sigs.k8s.io/cluster-api v1.8.2-0.20240826111923-18d9dd9bd8c3 h1:gZD+Ne56bdSDogSw1SNDk6iLFSX3iWxD8pHBfiVRiI8= +sigs.k8s.io/cluster-api v1.8.2-0.20240826111923-18d9dd9bd8c3/go.mod h1:pXv5LqLxuIbhGIXykyNKiJh+KrLweSBajVHHitPLyoY= +sigs.k8s.io/cluster-api/test v1.8.2-0.20240826111923-18d9dd9bd8c3 h1:+UTa+9HgarY0t+hVDXheAl2OjSAEuGJTktGXMYkckIQ= +sigs.k8s.io/cluster-api/test v1.8.2-0.20240826111923-18d9dd9bd8c3/go.mod h1:odnzMkDndCRPCWdwl0CRofyZyY857wN34bUih1MLKIc= sigs.k8s.io/controller-runtime v0.18.5 h1:nTHio/W+Q4aBlQMgbnC5hZb4IjIidyrizMai9P6n4Rk= sigs.k8s.io/controller-runtime v0.18.5/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kind v0.23.0 h1:8fyDGWbWTeCcCTwA04v4Nfr45KKxbSPH1WO9K+jVrBg= -sigs.k8s.io/kind v0.23.0/go.mod h1:ZQ1iZuJLh3T+O8fzhdi3VWcFTzsdXtNv2ppsHc8JQ7s= +sigs.k8s.io/kind v0.24.0 h1:g4y4eu0qa+SCeKESLpESgMmVFBebL0BDa6f777OIWrg= +sigs.k8s.io/kind v0.24.0/go.mod h1:t7ueEpzPYJvHA8aeLtI52rtFftNgUYUaCwvxjk7phfw= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=