From d79df7c6f3dd1316832c2e37d4ff494f4e46d558 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 5 Sep 2024 16:41:56 +0200 Subject: [PATCH] Implement etcd snapshot restore skeleton Signed-off-by: Danil-Grigorev --- ...er-turtles-exp-etcdrestore-components.yaml | 92 +---- .../api/v1alpha1/etcdmachinesnapshot_types.go | 24 +- .../etcdmachinesnapshotconfig_types.go | 2 +- .../api/v1alpha1/etcdsnapshotrestore_types.go | 49 ++- .../api/v1alpha1/zz_generated.deepcopy.go | 79 ++-- ...s-capi.cattle.io_etcdmachinesnapshots.yaml | 11 +- ...s-capi.cattle.io_etcdsnapshotrestores.yaml | 82 +---- exp/etcdrestore/config/rbac/role.yaml | 12 + .../etcdmachinesnapshot_controller.go | 24 +- .../etcdsnapshotrestore_controller.go | 347 ++++++++++++++++-- .../etcdsnapshotsync_controller.go | 7 +- exp/etcdrestore/controllers/planner.go | 59 ++- .../snapshotters/rke2snapshotter.go | 5 +- exp/etcdrestore/main.go | 10 +- .../webhooks/etcdmachinesnapshot.go | 8 +- .../webhooks/etcdsnapshotrestore.go | 6 +- 16 files changed, 525 insertions(+), 292 deletions(-) diff --git a/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml b/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml index 709be9a7..88a5711d 100644 --- a/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml +++ b/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml @@ -11,8 +11,8 @@ metadata: spec: group: turtles-capi.cattle.io names: - kind: EtcdMachineSnapshot - listKind: EtcdMachineSnapshotList + kind: ETCDMachineSnapshot + listKind: ETCDMachineSnapshotList plural: etcdmachinesnapshots singular: etcdmachinesnapshot scope: Namespaced @@ -20,7 +20,7 @@ spec: - name: v1alpha1 schema: openAPIV3Schema: - description: EtcdMachineSnapshot is the Schema for the EtcdMachineSnapshot + description: ETCDMachineSnapshot is the Schema for the ETCDMachineSnapshot API. properties: apiVersion: @@ -41,7 +41,7 @@ spec: metadata: type: object spec: - description: EtcdMachineSnapshotSpec defines the desired state of EtcdMachineSnapshot + description: ETCDMachineSnapshotSpec defines the desired state of EtcdMachineSnapshot properties: clusterName: type: string @@ -60,6 +60,9 @@ spec: - machineName - manual type: object + x-kubernetes-validations: + - message: ETCD snapshot location can't be empty. + rule: size(self.location)>0 status: default: {} description: EtcdSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore @@ -133,8 +136,8 @@ metadata: spec: group: turtles-capi.cattle.io names: - kind: EtcdSnapshotRestore - listKind: EtcdSnapshotRestoreList + kind: ETCDSnapshotRestore + listKind: ETCDSnapshotRestoreList plural: etcdsnapshotrestores singular: etcdsnapshotrestore scope: Namespaced @@ -142,7 +145,7 @@ spec: - name: v1alpha1 schema: openAPIV3Schema: - description: EtcdSnapshotRestore is the schema for the EtcdSnapshotRestore + description: ETCDSnapshotRestore is the schema for the ETCDSnapshotRestore API. properties: apiVersion: @@ -163,84 +166,24 @@ spec: metadata: type: object spec: - description: EtcdSnapshotRestoreSpec defines the desired state of EtcdSnapshotRestore. + description: ETCDSnapshotRestoreSpec defines the desired state of EtcdSnapshotRestore. properties: clusterName: type: string - configRef: - description: |- - ObjectReference contains enough information to let you inspect or modify the referred object. - --- - New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. - 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. - 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular - restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". - Those cannot be well described when embedded. - 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. - 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity - during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple - and the version of the actual struct is irrelevant. - 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type - will affect numerous schemas. Don't make new APIs embed an underspecified API type they do not control. - - - Instead of using this type, create a locally provided and used type that is well-focused on your reference. - For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 . - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic etcdMachineSnapshotName: type: string - ttlSecondsAfterFinished: - type: integer required: - clusterName - - configRef - etcdMachineSnapshotName - - ttlSecondsAfterFinished type: object + x-kubernetes-validations: + - message: Cluster Name can't be empty. + rule: size(self.clusterName)>0 + - message: ETCD machine snapshot name can't be empty. + rule: size(self.etcdMachineSnapshotName)>0 status: default: {} - description: EtcdSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore. + description: ETCDSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore. properties: conditions: description: Conditions provide observations of the operational state @@ -289,6 +232,7 @@ spec: type: object type: array phase: + default: Pending description: ETCDSnapshotPhase is a string representation of the phase of the etcd snapshot type: string diff --git a/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshot_types.go b/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshot_types.go index 1285db6c..61c089ef 100644 --- a/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshot_types.go +++ b/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshot_types.go @@ -39,8 +39,10 @@ const ( ETCDMachineSnapshotFinalizer = "etcdmachinesnapshot.turtles.cattle.io" ) -// EtcdMachineSnapshotSpec defines the desired state of EtcdMachineSnapshot -type EtcdMachineSnapshotSpec struct { +// +kubebuilder:validation:XValidation:message="ETCD snapshot location can't be empty.",rule="size(self.location)>0" +// +// ETCDMachineSnapshotSpec defines the desired state of EtcdMachineSnapshot +type ETCDMachineSnapshotSpec struct { ClusterName string `json:"clusterName"` MachineName string `json:"machineName"` ConfigRef string `json:"configRef"` @@ -49,32 +51,32 @@ type EtcdMachineSnapshotSpec struct { } // EtcdSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore -type EtcdMachineSnapshotStatus struct { +type ETCDMachineSnapshotStatus struct { Phase ETCDSnapshotPhase `json:"phase,omitempty"` Conditions clusterv1.Conditions `json:"conditions,omitempty"` } -// EtcdMachineSnapshot is the Schema for the EtcdMachineSnapshot API. +// ETCDMachineSnapshot is the Schema for the ETCDMachineSnapshot API. // // +kubebuilder:object:root=true // +kubebuilder:subresource:status -type EtcdMachineSnapshot struct { +type ETCDMachineSnapshot struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec EtcdMachineSnapshotSpec `json:"spec,omitempty"` - Status EtcdMachineSnapshotStatus `json:"status,omitempty"` + Spec ETCDMachineSnapshotSpec `json:"spec,omitempty"` + Status ETCDMachineSnapshotStatus `json:"status,omitempty"` } //+kubebuilder:object:root=true -// EtcdMachineSnapshotList contains a list of EtcdMachineSnapshots. -type EtcdMachineSnapshotList struct { +// ETCDMachineSnapshotList contains a list of EtcdMachineSnapshots. +type ETCDMachineSnapshotList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []EtcdMachineSnapshot `json:"items"` + Items []ETCDMachineSnapshot `json:"items"` } func init() { - objectTypes = append(objectTypes, &EtcdMachineSnapshot{}, &EtcdMachineSnapshotList{}) + objectTypes = append(objectTypes, &ETCDMachineSnapshot{}, &ETCDMachineSnapshotList{}) } diff --git a/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshotconfig_types.go b/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshotconfig_types.go index 841c1c36..68617926 100644 --- a/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshotconfig_types.go +++ b/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshotconfig_types.go @@ -61,7 +61,7 @@ type RKE2EtcdMachineSnapshotConfig struct { type RKE2EtcdMachineSnapshotConfigList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []EtcdSnapshotRestore `json:"items"` + Items []ETCDSnapshotRestore `json:"items"` } func init() { diff --git a/exp/etcdrestore/api/v1alpha1/etcdsnapshotrestore_types.go b/exp/etcdrestore/api/v1alpha1/etcdsnapshotrestore_types.go index 3f053305..a0231758 100644 --- a/exp/etcdrestore/api/v1alpha1/etcdsnapshotrestore_types.go +++ b/exp/etcdrestore/api/v1alpha1/etcdsnapshotrestore_types.go @@ -17,7 +17,6 @@ limitations under the License. package v1alpha1 import ( - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) @@ -44,41 +43,53 @@ const ( ETCDSnapshotRestorePhaseFinished ETCDSnapshotRestorePhase = "Done" ) -// EtcdSnapshotRestoreSpec defines the desired state of EtcdSnapshotRestore. -type EtcdSnapshotRestoreSpec struct { - ClusterName string `json:"clusterName"` - EtcdMachineSnapshotName string `json:"etcdMachineSnapshotName"` - TTLSecondsAfterFinished int `json:"ttlSecondsAfterFinished"` - ConfigRef corev1.ObjectReference `json:"configRef"` +// +kubebuilder:validation:XValidation:message="Cluster Name can't be empty.",rule="size(self.clusterName)>0" +// +kubebuilder:validation:XValidation:message="ETCD machine snapshot name can't be empty.",rule="size(self.etcdMachineSnapshotName)>0" +// +// ETCDSnapshotRestoreSpec defines the desired state of EtcdSnapshotRestore. +type ETCDSnapshotRestoreSpec struct { + // +required + ClusterName string `json:"clusterName"` + + // +required + ETCDMachineSnapshotName string `json:"etcdMachineSnapshotName"` + + // TTLSecondsAfterFinished int `json:"ttlSecondsAfterFinished"` + + // // +required + // ConfigRef corev1.LocalObjectReference `json:"configRef"` } -// EtcdSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore. -type EtcdSnapshotRestoreStatus struct { - Phase ETCDSnapshotPhase `json:"phase,omitempty"` - Conditions clusterv1.Conditions `json:"conditions,omitempty"` +// ETCDSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore. +type ETCDSnapshotRestoreStatus struct { + // +kubebuilder:default=Pending + Phase ETCDSnapshotRestorePhase `json:"phase,omitempty"` + Conditions clusterv1.Conditions `json:"conditions,omitempty"` } -// EtcdSnapshotRestore is the schema for the EtcdSnapshotRestore API. +// ETCDSnapshotRestore is the schema for the ETCDSnapshotRestore API. // // +kubebuilder:object:root=true // +kubebuilder:subresource:status -type EtcdSnapshotRestore struct { +type ETCDSnapshotRestore struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec EtcdSnapshotRestoreSpec `json:"spec,omitempty"` - Status EtcdSnapshotRestoreStatus `json:"status,omitempty"` + Spec ETCDSnapshotRestoreSpec `json:"spec,omitempty"` + + // +kubebuilder:default={} + Status ETCDSnapshotRestoreStatus `json:"status,omitempty"` } //+kubebuilder:object:root=true -// EtcdSnapshotRestoreList contains a list of EtcdSnapshotRestores. -type EtcdSnapshotRestoreList struct { +// ETCDSnapshotRestoreList contains a list of EtcdSnapshotRestores. +type ETCDSnapshotRestoreList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []EtcdSnapshotRestore `json:"items"` + Items []ETCDSnapshotRestore `json:"items"` } func init() { - objectTypes = append(objectTypes, &EtcdSnapshotRestore{}, &EtcdSnapshotRestoreList{}) + objectTypes = append(objectTypes, &ETCDSnapshotRestore{}, &ETCDSnapshotRestoreList{}) } diff --git a/exp/etcdrestore/api/v1alpha1/zz_generated.deepcopy.go b/exp/etcdrestore/api/v1alpha1/zz_generated.deepcopy.go index 060ca1f6..35f77c82 100644 --- a/exp/etcdrestore/api/v1alpha1/zz_generated.deepcopy.go +++ b/exp/etcdrestore/api/v1alpha1/zz_generated.deepcopy.go @@ -26,7 +26,7 @@ import ( ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EtcdMachineSnapshot) DeepCopyInto(out *EtcdMachineSnapshot) { +func (in *ETCDMachineSnapshot) DeepCopyInto(out *ETCDMachineSnapshot) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) @@ -34,18 +34,18 @@ func (in *EtcdMachineSnapshot) DeepCopyInto(out *EtcdMachineSnapshot) { in.Status.DeepCopyInto(&out.Status) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMachineSnapshot. -func (in *EtcdMachineSnapshot) DeepCopy() *EtcdMachineSnapshot { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDMachineSnapshot. +func (in *ETCDMachineSnapshot) DeepCopy() *ETCDMachineSnapshot { if in == nil { return nil } - out := new(EtcdMachineSnapshot) + out := new(ETCDMachineSnapshot) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *EtcdMachineSnapshot) DeepCopyObject() runtime.Object { +func (in *ETCDMachineSnapshot) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -53,31 +53,31 @@ func (in *EtcdMachineSnapshot) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EtcdMachineSnapshotList) DeepCopyInto(out *EtcdMachineSnapshotList) { +func (in *ETCDMachineSnapshotList) DeepCopyInto(out *ETCDMachineSnapshotList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]EtcdMachineSnapshot, len(*in)) + *out = make([]ETCDMachineSnapshot, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMachineSnapshotList. -func (in *EtcdMachineSnapshotList) DeepCopy() *EtcdMachineSnapshotList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDMachineSnapshotList. +func (in *ETCDMachineSnapshotList) DeepCopy() *ETCDMachineSnapshotList { if in == nil { return nil } - out := new(EtcdMachineSnapshotList) + out := new(ETCDMachineSnapshotList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *EtcdMachineSnapshotList) DeepCopyObject() runtime.Object { +func (in *ETCDMachineSnapshotList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -85,22 +85,22 @@ func (in *EtcdMachineSnapshotList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EtcdMachineSnapshotSpec) DeepCopyInto(out *EtcdMachineSnapshotSpec) { +func (in *ETCDMachineSnapshotSpec) DeepCopyInto(out *ETCDMachineSnapshotSpec) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMachineSnapshotSpec. -func (in *EtcdMachineSnapshotSpec) DeepCopy() *EtcdMachineSnapshotSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDMachineSnapshotSpec. +func (in *ETCDMachineSnapshotSpec) DeepCopy() *ETCDMachineSnapshotSpec { if in == nil { return nil } - out := new(EtcdMachineSnapshotSpec) + out := new(ETCDMachineSnapshotSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EtcdMachineSnapshotStatus) DeepCopyInto(out *EtcdMachineSnapshotStatus) { +func (in *ETCDMachineSnapshotStatus) DeepCopyInto(out *ETCDMachineSnapshotStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions @@ -111,18 +111,18 @@ func (in *EtcdMachineSnapshotStatus) DeepCopyInto(out *EtcdMachineSnapshotStatus } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMachineSnapshotStatus. -func (in *EtcdMachineSnapshotStatus) DeepCopy() *EtcdMachineSnapshotStatus { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDMachineSnapshotStatus. +func (in *ETCDMachineSnapshotStatus) DeepCopy() *ETCDMachineSnapshotStatus { if in == nil { return nil } - out := new(EtcdMachineSnapshotStatus) + out := new(ETCDMachineSnapshotStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EtcdSnapshotRestore) DeepCopyInto(out *EtcdSnapshotRestore) { +func (in *ETCDSnapshotRestore) DeepCopyInto(out *ETCDSnapshotRestore) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) @@ -130,18 +130,18 @@ func (in *EtcdSnapshotRestore) DeepCopyInto(out *EtcdSnapshotRestore) { in.Status.DeepCopyInto(&out.Status) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdSnapshotRestore. -func (in *EtcdSnapshotRestore) DeepCopy() *EtcdSnapshotRestore { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotRestore. +func (in *ETCDSnapshotRestore) DeepCopy() *ETCDSnapshotRestore { if in == nil { return nil } - out := new(EtcdSnapshotRestore) + out := new(ETCDSnapshotRestore) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *EtcdSnapshotRestore) DeepCopyObject() runtime.Object { +func (in *ETCDSnapshotRestore) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -149,31 +149,31 @@ func (in *EtcdSnapshotRestore) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EtcdSnapshotRestoreList) DeepCopyInto(out *EtcdSnapshotRestoreList) { +func (in *ETCDSnapshotRestoreList) DeepCopyInto(out *ETCDSnapshotRestoreList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]EtcdSnapshotRestore, len(*in)) + *out = make([]ETCDSnapshotRestore, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdSnapshotRestoreList. -func (in *EtcdSnapshotRestoreList) DeepCopy() *EtcdSnapshotRestoreList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotRestoreList. +func (in *ETCDSnapshotRestoreList) DeepCopy() *ETCDSnapshotRestoreList { if in == nil { return nil } - out := new(EtcdSnapshotRestoreList) + out := new(ETCDSnapshotRestoreList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *EtcdSnapshotRestoreList) DeepCopyObject() runtime.Object { +func (in *ETCDSnapshotRestoreList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -181,23 +181,22 @@ func (in *EtcdSnapshotRestoreList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EtcdSnapshotRestoreSpec) DeepCopyInto(out *EtcdSnapshotRestoreSpec) { +func (in *ETCDSnapshotRestoreSpec) DeepCopyInto(out *ETCDSnapshotRestoreSpec) { *out = *in - out.ConfigRef = in.ConfigRef } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdSnapshotRestoreSpec. -func (in *EtcdSnapshotRestoreSpec) DeepCopy() *EtcdSnapshotRestoreSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotRestoreSpec. +func (in *ETCDSnapshotRestoreSpec) DeepCopy() *ETCDSnapshotRestoreSpec { if in == nil { return nil } - out := new(EtcdSnapshotRestoreSpec) + out := new(ETCDSnapshotRestoreSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EtcdSnapshotRestoreStatus) DeepCopyInto(out *EtcdSnapshotRestoreStatus) { +func (in *ETCDSnapshotRestoreStatus) DeepCopyInto(out *ETCDSnapshotRestoreStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions @@ -208,12 +207,12 @@ func (in *EtcdSnapshotRestoreStatus) DeepCopyInto(out *EtcdSnapshotRestoreStatus } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdSnapshotRestoreStatus. -func (in *EtcdSnapshotRestoreStatus) DeepCopy() *EtcdSnapshotRestoreStatus { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotRestoreStatus. +func (in *ETCDSnapshotRestoreStatus) DeepCopy() *ETCDSnapshotRestoreStatus { if in == nil { return nil } - out := new(EtcdSnapshotRestoreStatus) + out := new(ETCDSnapshotRestoreStatus) in.DeepCopyInto(out) return out } @@ -266,7 +265,7 @@ func (in *RKE2EtcdMachineSnapshotConfigList) DeepCopyInto(out *RKE2EtcdMachineSn in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]EtcdSnapshotRestore, len(*in)) + *out = make([]ETCDSnapshotRestore, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdmachinesnapshots.yaml b/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdmachinesnapshots.yaml index 6fcd1ef6..42c5878e 100644 --- a/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdmachinesnapshots.yaml +++ b/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdmachinesnapshots.yaml @@ -8,8 +8,8 @@ metadata: spec: group: turtles-capi.cattle.io names: - kind: EtcdMachineSnapshot - listKind: EtcdMachineSnapshotList + kind: ETCDMachineSnapshot + listKind: ETCDMachineSnapshotList plural: etcdmachinesnapshots singular: etcdmachinesnapshot scope: Namespaced @@ -17,7 +17,7 @@ spec: - name: v1alpha1 schema: openAPIV3Schema: - description: EtcdMachineSnapshot is the Schema for the EtcdMachineSnapshot + description: ETCDMachineSnapshot is the Schema for the ETCDMachineSnapshot API. properties: apiVersion: @@ -38,7 +38,7 @@ spec: metadata: type: object spec: - description: EtcdMachineSnapshotSpec defines the desired state of EtcdMachineSnapshot + description: ETCDMachineSnapshotSpec defines the desired state of EtcdMachineSnapshot properties: clusterName: type: string @@ -57,6 +57,9 @@ spec: - machineName - manual type: object + x-kubernetes-validations: + - message: ETCD snapshot location can't be empty. + rule: size(self.location)>0 status: description: EtcdSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore properties: diff --git a/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdsnapshotrestores.yaml b/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdsnapshotrestores.yaml index 7e820575..ebd4b8fc 100644 --- a/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdsnapshotrestores.yaml +++ b/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdsnapshotrestores.yaml @@ -8,8 +8,8 @@ metadata: spec: group: turtles-capi.cattle.io names: - kind: EtcdSnapshotRestore - listKind: EtcdSnapshotRestoreList + kind: ETCDSnapshotRestore + listKind: ETCDSnapshotRestoreList plural: etcdsnapshotrestores singular: etcdsnapshotrestore scope: Namespaced @@ -17,7 +17,7 @@ spec: - name: v1alpha1 schema: openAPIV3Schema: - description: EtcdSnapshotRestore is the schema for the EtcdSnapshotRestore + description: ETCDSnapshotRestore is the schema for the ETCDSnapshotRestore API. properties: apiVersion: @@ -38,83 +38,24 @@ spec: metadata: type: object spec: - description: EtcdSnapshotRestoreSpec defines the desired state of EtcdSnapshotRestore. + description: ETCDSnapshotRestoreSpec defines the desired state of EtcdSnapshotRestore. properties: clusterName: type: string - configRef: - description: |- - ObjectReference contains enough information to let you inspect or modify the referred object. - --- - New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. - 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. - 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular - restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". - Those cannot be well described when embedded. - 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. - 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity - during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple - and the version of the actual struct is irrelevant. - 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type - will affect numerous schemas. Don't make new APIs embed an underspecified API type they do not control. - - - Instead of using this type, create a locally provided and used type that is well-focused on your reference. - For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 . - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic etcdMachineSnapshotName: type: string - ttlSecondsAfterFinished: - type: integer required: - clusterName - - configRef - etcdMachineSnapshotName - - ttlSecondsAfterFinished type: object + x-kubernetes-validations: + - message: Cluster Name can't be empty. + rule: size(self.clusterName)>0 + - message: ETCD machine snapshot name can't be empty. + rule: size(self.etcdMachineSnapshotName)>0 status: - description: EtcdSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore. + default: {} + description: ETCDSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore. properties: conditions: description: Conditions provide observations of the operational state @@ -163,6 +104,7 @@ spec: type: object type: array phase: + default: Pending description: ETCDSnapshotPhase is a string representation of the phase of the etcd snapshot type: string diff --git a/exp/etcdrestore/config/rbac/role.yaml b/exp/etcdrestore/config/rbac/role.yaml index 4a69a2b3..9de2966e 100644 --- a/exp/etcdrestore/config/rbac/role.yaml +++ b/exp/etcdrestore/config/rbac/role.yaml @@ -52,6 +52,18 @@ rules: - patch - update - watch +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - cluster.x-k8s.io resources: diff --git a/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go b/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go index 4ce613d0..5300b527 100644 --- a/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go +++ b/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go @@ -38,8 +38,8 @@ import ( const snapshotPhaseRequeueDuration = 1 * time.Minute -// EtcdMachineSnapshotReconciler reconciles an EtcdMachineSnapshot object. -type EtcdMachineSnapshotReconciler struct { +// ETCDMachineSnapshotReconciler reconciles an EtcdMachineSnapshot object. +type ETCDMachineSnapshotReconciler struct { client.Client WatchFilterValue string @@ -49,10 +49,10 @@ type EtcdMachineSnapshotReconciler struct { } // SetupWithManager sets up the controller with the Manager. -func (r *EtcdMachineSnapshotReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, _ controller.Options) error { +func (r *ETCDMachineSnapshotReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, _ controller.Options) error { // TODO: Setup predicates for the controller. c, err := ctrl.NewControllerManagedBy(mgr). - For(&snapshotrestorev1.EtcdMachineSnapshot{}). + For(&snapshotrestorev1.ETCDMachineSnapshot{}). Build(r) if err != nil { return fmt.Errorf("creating etcdMachineSnapshot controller: %w", err) @@ -68,10 +68,10 @@ func (r *EtcdMachineSnapshotReconciler) SetupWithManager(_ context.Context, mgr //+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=etcdmachinesnapshots/finalizers,verbs=update // Reconcile reconciles the EtcdMachineSnapshot object. -func (r *EtcdMachineSnapshotReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { +func (r *ETCDMachineSnapshotReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { log := log.FromContext(ctx) - etcdMachineSnapshot := &snapshotrestorev1.EtcdMachineSnapshot{} + etcdMachineSnapshot := &snapshotrestorev1.ETCDMachineSnapshot{} if err := r.Client.Get(ctx, req.NamespacedName, etcdMachineSnapshot); apierrors.IsNotFound(err) { // Object not found, return. Created objects are automatically garbage collected. return ctrl.Result{}, nil @@ -120,8 +120,8 @@ func (r *EtcdMachineSnapshotReconciler) Reconcile(ctx context.Context, req ctrl. return r.reconcileNormal(ctx, etcdMachineSnapshot) } -func (r *EtcdMachineSnapshotReconciler) reconcileNormal( - ctx context.Context, etcdMachineSnapshot *snapshotrestorev1.EtcdMachineSnapshot, +func (r *ETCDMachineSnapshotReconciler) reconcileNormal( + ctx context.Context, etcdMachineSnapshot *snapshotrestorev1.ETCDMachineSnapshot, ) (ctrl.Result, error) { // Handle different phases of the etcdmachinesnapshot creation process @@ -158,8 +158,8 @@ func (r *EtcdMachineSnapshotReconciler) reconcileNormal( return ctrl.Result{}, nil } -func (r *EtcdMachineSnapshotReconciler) reconcileDelete( - ctx context.Context, etcdMachineSnapshot *snapshotrestorev1.EtcdMachineSnapshot, +func (r *ETCDMachineSnapshotReconciler) reconcileDelete( + ctx context.Context, etcdMachineSnapshot *snapshotrestorev1.ETCDMachineSnapshot, ) error { log := log.FromContext(ctx) @@ -168,7 +168,7 @@ func (r *EtcdMachineSnapshotReconciler) reconcileDelete( // Perform any necessary cleanup of associated resources here // Example: Delete associated snapshot resources - snapshotList := &snapshotrestorev1.EtcdMachineSnapshotList{} + snapshotList := &snapshotrestorev1.ETCDMachineSnapshotList{} if err := r.Client.List(ctx, snapshotList, client.InNamespace(etcdMachineSnapshot.Namespace)); err != nil { log.Error(err, "Failed to list associated EtcdMachineSnapshots") return err @@ -194,7 +194,7 @@ func (r *EtcdMachineSnapshotReconciler) reconcileDelete( } // checkSnapshotStatus checks the status of the snapshot creation process. -func checkSnapshotStatus(ctx context.Context, r *EtcdMachineSnapshotReconciler, etcdMachineSnapshot *snapshotrestorev1.EtcdMachineSnapshot) error { +func checkSnapshotStatus(ctx context.Context, r *ETCDMachineSnapshotReconciler, etcdMachineSnapshot *snapshotrestorev1.ETCDMachineSnapshot) error { log := log.FromContext(ctx) etcdSnapshotFileList := &k3sv1.ETCDSnapshotFileList{} diff --git a/exp/etcdrestore/controllers/etcdsnapshotrestore_controller.go b/exp/etcdrestore/controllers/etcdsnapshotrestore_controller.go index d3fff63e..6bda25fc 100644 --- a/exp/etcdrestore/controllers/etcdsnapshotrestore_controller.go +++ b/exp/etcdrestore/controllers/etcdsnapshotrestore_controller.go @@ -19,65 +19,358 @@ package controllers import ( "context" "fmt" + "time" - "k8s.io/apimachinery/pkg/runtime" + kerrors "k8s.io/apimachinery/pkg/util/errors" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/collections" + "sigs.k8s.io/cluster-api/util/patch" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - - "sigs.k8s.io/cluster-api/controllers/remote" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" snapshotrestorev1 "github.com/rancher/turtles/exp/etcdrestore/api/v1alpha1" ) -// EtcdSnapshotRestoreReconciler reconciles a EtcdSnapshotRestore object. -type EtcdSnapshotRestoreReconciler struct { +// InitMachine is a filter matching on init machine of the ETCD snapshot +func InitMachine(etcdMachineSnapshot *snapshotrestorev1.ETCDMachineSnapshot) collections.Func { + return func(machine *clusterv1.Machine) bool { + return machine.Name == etcdMachineSnapshot.Spec.MachineName + } +} + +// ETCDSnapshotRestoreReconciler reconciles a ETCDSnapshotRestore object +type ETCDSnapshotRestoreReconciler struct { client.Client - WatchFilterValue string +} + +func clusterToRestore(c client.Client) handler.MapFunc { + return func(ctx context.Context, cluster client.Object) []ctrl.Request { + restoreList := &snapshotrestorev1.ETCDSnapshotRestoreList{} + if err := c.List(ctx, restoreList); err != nil { + return nil + } + + requests := []ctrl.Request{} + for _, restore := range restoreList.Items { + if restore.Spec.ClusterName == cluster.GetName() { + requests = append(requests, ctrl.Request{ + NamespacedName: client.ObjectKey{ + Namespace: cluster.GetNamespace(), + Name: restore.Name, + }, + }) + } + } - controller controller.Controller - Tracker *remote.ClusterCacheTracker - Scheme *runtime.Scheme + return requests + } } // SetupWithManager sets up the controller with the Manager. -func (r *EtcdSnapshotRestoreReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, _ controller.Options) error { - // TODO: Setup predicates for the controller. - c, err := ctrl.NewControllerManagedBy(mgr). - For(&snapshotrestorev1.EtcdSnapshotRestore{}). - Build(r) - if err != nil { - return fmt.Errorf("creating etcdSnapshotRestore controller: %w", err) - } +func (r *ETCDSnapshotRestoreReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, options controller.Options) error { + return ctrl.NewControllerManagedBy(mgr). + WithOptions(options). + For(&snapshotrestorev1.ETCDSnapshotRestore{}). + Watches(&clusterv1.Cluster{}, handler.EnqueueRequestsFromMapFunc(clusterToRestore(r.Client))). + Complete(reconcile.AsReconciler(r.Client, r)) +} + +// scope holds the different objects that are read and used during the reconcile. +type scope struct { + // cluster is the Cluster object the Machine belongs to. + // It is set at the beginning of the reconcile function. + cluster *clusterv1.Cluster - r.controller = c + // machine is the Machine object. It is set at the beginning + // of the reconcile function. + machines collections.Machines - return nil + // etcdMachineSnapshot is the EtcdMachineSnapshot object. + etcdMachineSnapshot *snapshotrestorev1.ETCDMachineSnapshot } //+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=etcdsnapshotrestores,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=etcdsnapshotrestores/status,verbs=get;update;patch //+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=etcdsnapshotrestores/finalizers,verbs=update //+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters/status,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=secrets;events;configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="management.cattle.io",resources=*,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=rke2configs;rke2configs/status;rke2configs/finalizers,verbs=get;list;watch;create;update;patch;delete -// Reconcile reconciles the EtcdSnapshotRestore object. -func (r *EtcdSnapshotRestoreReconciler) Reconcile(_ context.Context, _ ctrl.Request) (ctrl.Result, error) { +func (r *ETCDSnapshotRestoreReconciler) Reconcile(ctx context.Context, restore *snapshotrestorev1.ETCDSnapshotRestore) (_ ctrl.Result, reterr error) { + // Initialize the patch helper. + patchHelper, err := patch.NewHelper(restore, r.Client) + if err != nil { + return ctrl.Result{}, err + } + + defer func() { + // Always attempt to patch the object and status after each reconciliation. + // Patch ObservedGeneration only if the reconciliation completed successfully. + patchOpts := []patch.Option{} + if reterr == nil { + patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{}) + } + if err := patchHelper.Patch(ctx, restore, patchOpts...); err != nil { + reterr = kerrors.NewAggregate([]error{reterr, err}) + } + }() + + // Handle deleted etcdSnapshot + if !restore.ObjectMeta.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, restore) + } + + var errs []error + + result, err := r.reconcileNormal(ctx, restore) + if err != nil { + errs = append(errs, fmt.Errorf("error reconciling etcd snapshot restore: %w", err)) + } + + return result, kerrors.NewAggregate(errs) +} + +func (r *ETCDSnapshotRestoreReconciler) reconcileNormal(ctx context.Context, etcdSnapshotRestore *snapshotrestorev1.ETCDSnapshotRestore) (_ ctrl.Result, reterr error) { + log := log.FromContext(ctx) + + scope, err := initScope(ctx, r.Client, etcdSnapshotRestore) + if err != nil { + return ctrl.Result{}, err + } + + running := func(machine *clusterv1.Machine) bool { + return machine.Status.Phase == string(clusterv1.MachinePhaseRunning) + } + + if scope.machines.Filter(running).Len() != scope.machines.Len() { + log.Info("Not all machines are running yet, requeuing") + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + if scope.machines.Filter(InitMachine(scope.etcdMachineSnapshot)).Len() != 1 { + return ctrl.Result{}, fmt.Errorf( + "init machine %s for snapshot %s is not found", + scope.etcdMachineSnapshot.Spec.MachineName, + etcdSnapshotRestore.Spec.ETCDMachineSnapshotName) + } + + patchHelper, err := patch.NewHelper(scope.cluster, r.Client) + if err != nil { + return ctrl.Result{}, err + } + + defer func() { + // Always attempt to patch the object and status after each reconciliation. + // Patch ObservedGeneration only if the reconciliation completed successfully. + patchOpts := []patch.Option{} + if reterr == nil { + patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{}) + } + if err := patchHelper.Patch(ctx, scope.cluster, patchOpts...); err != nil { + reterr = kerrors.NewAggregate([]error{reterr, err}) + } + }() + + switch etcdSnapshotRestore.Status.Phase { + case snapshotrestorev1.ETCDSnapshotRestorePhasePending: + // Pause CAPI cluster. + scope.cluster.Spec.Paused = true + etcdSnapshotRestore.Status.Phase = snapshotrestorev1.ETCDSnapshotRestorePhaseStarted + + return ctrl.Result{}, nil + case snapshotrestorev1.ETCDSnapshotRestorePhaseStarted: + etcdSnapshotRestore.Status.Phase = snapshotrestorev1.ETCDSnapshotRestorePhaseShutdown + + return ctrl.Result{}, nil + case snapshotrestorev1.ETCDSnapshotRestorePhaseShutdown: + // Stop RKE2 on all the machines. + return r.stopRKE2OnAllMachines(ctx, scope, etcdSnapshotRestore) + case snapshotrestorev1.ETCDSnapshotRestorePhaseRunning: + // Restore the etcd snapshot on the init machine. + return r.restoreSnaphotOnInitMachine(ctx, scope, etcdSnapshotRestore) + case snapshotrestorev1.ETCDSnapshotRestorePhaseAgentRestart: + // Start RKE2 on all the machines. + return r.startRKE2OnAllMachines(ctx, scope, etcdSnapshotRestore) + case snapshotrestorev1.ETCDSnapshotRestorePhaseJoinAgents: + etcdSnapshotRestore.Status.Phase = snapshotrestorev1.ETCDSnapshotRestorePhaseFinished + + // TODO: wait for machines to join + return ctrl.Result{}, nil + case snapshotrestorev1.ETCDSnapshotRestorePhaseFinished: + scope.cluster.Spec.Paused = false + + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil } -func (r *EtcdSnapshotRestoreReconciler) reconcileNormal( - _ context.Context, _ *snapshotrestorev1.EtcdSnapshotRestore, -) (_ ctrl.Result, err error) { +func (r *ETCDSnapshotRestoreReconciler) reconcileDelete(_ context.Context, _ *snapshotrestorev1.ETCDSnapshotRestore) (ctrl.Result, error) { return ctrl.Result{}, nil } -func (r *EtcdSnapshotRestoreReconciler) reconcileDelete( - _ context.Context, _ *snapshotrestorev1.EtcdSnapshotRestore, -) (ctrl.Result, error) { +func initScope(ctx context.Context, c client.Client, etcdSnapshotRestore *snapshotrestorev1.ETCDSnapshotRestore) (*scope, error) { + // Get the cluster object. + cluster := &clusterv1.Cluster{} + + if err := c.Get(ctx, client.ObjectKey{Namespace: etcdSnapshotRestore.Namespace, Name: etcdSnapshotRestore.Spec.ClusterName}, cluster); err != nil { + return nil, fmt.Errorf("failed to get cluster: %w", err) + } + + machines, err := collections.GetFilteredMachinesForCluster(ctx, c, cluster) + if err != nil { + return nil, fmt.Errorf("failed to collect machines for cluster: %w", err) + } + + // Get etcd machine backup object. + etcdMachineSnapshot := &snapshotrestorev1.ETCDMachineSnapshot{} + if err := c.Get(ctx, client.ObjectKey{ + Namespace: etcdSnapshotRestore.Namespace, + Name: etcdSnapshotRestore.Spec.ETCDMachineSnapshotName, + }, etcdMachineSnapshot); err != nil { + return nil, fmt.Errorf("failed to get etcd machine backup: %w", err) + } + + return &scope{ + cluster: cluster, + machines: machines.Filter(collections.ControlPlaneMachines(cluster.Name)), + etcdMachineSnapshot: etcdMachineSnapshot, + }, nil +} + +func (r *ETCDSnapshotRestoreReconciler) stopRKE2OnAllMachines(ctx context.Context, scope *scope, etcdSnapshotRestore *snapshotrestorev1.ETCDSnapshotRestore) (ctrl.Result, error) { + log := log.FromContext(ctx) + + results := []Output{} + for _, machine := range scope.machines { + log.Info("Stopping RKE2 on machine", "machine", machine.Name) + + // Get the plan secret for the machine. + applied, err := Plan(ctx, r.Client, machine).Apply(ctx, RKE2KillAll()) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to get plan secret for machine: %w", err) + } + + results = append(results, applied) + } + + for _, result := range results { + if !result.Finished { + log.Info("Plan is not yet applied, requeuing", "machine", result.Machine.Name) + + // Requeue after 30 seconds if not ready. + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + log.Info(fmt.Sprintf("Decompressed plan output: %s", result.Result), "machine", result.Machine.Name) + + } + + log.Info("All machines are ready to proceed to the next phase, setting phase to shutdown") + + etcdSnapshotRestore.Status.Phase = snapshotrestorev1.ETCDSnapshotRestorePhaseRunning + return ctrl.Result{}, nil } + +func (r *ETCDSnapshotRestoreReconciler) restoreSnaphotOnInitMachine(ctx context.Context, scope *scope, etcdSnapshotRestore *snapshotrestorev1.ETCDSnapshotRestore) (ctrl.Result, error) { + log := log.FromContext(ctx) + + initMachine := scope.machines.Filter(InitMachine(scope.etcdMachineSnapshot)).UnsortedList()[0] + + log.Info("Filling plan secret with etcd restore instructions", "machine", initMachine.Name) + + // Get the plan secret for the machine. + applied, err := Plan(ctx, r.Client, initMachine).Apply( + ctx, + RemoveServerURL(), + ManifestRemoval(), + ETCDRestore(scope.etcdMachineSnapshot), + ) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to get plan secret for machine: %w", err) + } else if !applied.Finished { + log.Info("Plan not applied yet", "machine", initMachine.Name) + + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + log.Info(fmt.Sprintf("Decompressed plan output: %s", applied.Result), "machine", initMachine.Name) + + etcdSnapshotRestore.Status.Phase = snapshotrestorev1.ETCDSnapshotRestorePhaseAgentRestart + + return ctrl.Result{}, nil +} + +func (r *ETCDSnapshotRestoreReconciler) startRKE2OnAllMachines(ctx context.Context, scope *scope, etcdSnapshotRestore *snapshotrestorev1.ETCDSnapshotRestore) (ctrl.Result, error) { + log := log.FromContext(ctx) + + initMachine := scope.machines.Filter(InitMachine(scope.etcdMachineSnapshot)).UnsortedList()[0] + + // TODO: other registration methods + initMachineIP := getInternalMachineIP(initMachine) + if initMachineIP == "" { + log.Info("failed to get internal machine IP, field is empty") + + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + // Start from the init machine. + sortedMachines := append( + []*clusterv1.Machine{initMachine}, + scope.machines.UnsortedList()..., + ) + + for _, machine := range sortedMachines { + instructions := Instructions{} + if machine.Name == initMachine.Name { + log.Info("Starting RKE2 on init machine", "machine", initMachine.Name) + + instructions = append(instructions, StartRKE2()) + } else { + log.Info("Starting RKE2 on machine", "machine", machine.Name) + + instructions = append(instructions, RemoveServerURL(), + SetServerURL(initMachineIP), + RemoveETCDData(), + ManifestRemoval(), + StartRKE2()) + } + + applied, err := Plan(ctx, r.Client, machine).Apply(ctx, instructions...) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to patch plan secret: %w", err) + } else if !applied.Finished { + log.Info("Plan is not yet applied, requeuing", "machine", machine.Name) + + // Requeue after 30 seconds if not ready. + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + log.Info(fmt.Sprintf("Decompressed plan output: %s", applied.Result), "machine", machine.Name) + } + + log.Info("All machines are ready and started RKE2") + + etcdSnapshotRestore.Status.Phase = snapshotrestorev1.ETCDSnapshotRestorePhaseJoinAgents + + return ctrl.Result{}, nil +} + +// getInternalMachineIP collects internal machine IP for the init machine +func getInternalMachineIP(machine *clusterv1.Machine) string { + for _, address := range machine.Status.Addresses { + if address.Type == clusterv1.MachineInternalIP { + return address.Address + } + } + return "" +} diff --git a/exp/etcdrestore/controllers/etcdsnapshotsync_controller.go b/exp/etcdrestore/controllers/etcdsnapshotsync_controller.go index 563a57c1..e604518d 100644 --- a/exp/etcdrestore/controllers/etcdsnapshotsync_controller.go +++ b/exp/etcdrestore/controllers/etcdsnapshotsync_controller.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "errors" "fmt" "time" @@ -148,13 +147,13 @@ func (r *EtcdSnapshotSyncReconciler) etcdSnapshotFile(ctx context.Context, clust log.Info("Cluster name", "name", cluster.GetName()) gvk := schema.GroupVersionKind{ - Group: "k3s.cattle.io", + Group: k3sv1.GroupVersion.Group, Kind: "ETCDSnapshotFile", - Version: "v1", + Version: k3sv1.GroupVersion.Version, } if o.GetObjectKind().GroupVersionKind() != gvk { - log.Error(errors.New("got a different object"), "objectGVK", o.GetObjectKind().GroupVersionKind()) + log.Error(fmt.Errorf("got a different object"), "objectGVK", o.GetObjectKind().GroupVersionKind().String()) return nil } diff --git a/exp/etcdrestore/controllers/planner.go b/exp/etcdrestore/controllers/planner.go index 7c4271de..2c9c05f9 100644 --- a/exp/etcdrestore/controllers/planner.go +++ b/exp/etcdrestore/controllers/planner.go @@ -27,6 +27,7 @@ import ( "io" "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kerrors "k8s.io/apimachinery/pkg/util/errors" bootstrapv1 "github.com/rancher/cluster-api-provider-rke2/bootstrap/api/v1beta1" @@ -65,7 +66,25 @@ type plan struct { // Plan is initializing Planner, used to perform instructions in a specific order and collect results func Plan(ctx context.Context, c client.Client, machine *clusterv1.Machine) *Planner { return &Planner{ - Client: c, + Client: c, + machine: machine, + secret: initSecret(machine, map[string][]byte{}), + } +} + +func initSecret(machine *clusterv1.Machine, data map[string][]byte) *corev1.Secret { + planSecretName := strings.Join([]string{machine.Spec.Bootstrap.ConfigRef.Name, "rke2config", "plan"}, "-") + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: machine.Namespace, + Name: planSecretName, + }, + Data: data, } } @@ -75,10 +94,15 @@ func (p *Planner) Apply(ctx context.Context, instructions ...Instruction) (Outpu var err error errs := []error{} + if err := p.refresh(ctx); err != nil { + return Output{}, err + } + data, err := json.Marshal(plan{Instructions: instructions}) - errs = append(errs, err) + if err != nil { + return Output{}, err + } - errs = append(errs, p.refresh(ctx)) errs = append(errs, p.updatePlanSecret(ctx, data)) errs = append(errs, p.refresh(ctx)) @@ -137,7 +161,7 @@ func RKE2KillAll() Instruction { } // ETCDRestore performs restore form a snapshot path on the init node -func ETCDRestore(snapshot *snapshotrestorev1.EtcdMachineSnapshot) Instruction { +func ETCDRestore(snapshot *snapshotrestorev1.ETCDMachineSnapshot) Instruction { return Instruction{ Name: "etcd-restore", Command: "/bin/sh", @@ -226,15 +250,21 @@ func (p *Planner) applied(plan, appliedChecksum []byte) bool { func (p *Planner) updatePlanSecret(ctx context.Context, data []byte) error { log := log.FromContext(ctx) + if p.secret.Data == nil { + p.secret.Data = map[string][]byte{} + } + if !bytes.Equal(p.secret.Data["plan"], data) { log.Info("Plan secret not filled with proper plan", "machine", p.machine.Name) } - patchBase := client.MergeFromWithOptions(p.secret.DeepCopy(), client.MergeFromWithOptimisticLock{}) - p.secret.Data["plan"] = []byte(data) + p.secret = initSecret(p.machine, p.secret.Data) - if err := p.Client.Patch(ctx, p.secret, patchBase); err != nil { + if err := p.Client.Patch(ctx, p.secret, client.Apply, []client.PatchOption{ + client.ForceOwnership, + client.FieldOwner("etcdrestore-controller"), + }...); err != nil { return fmt.Errorf("failed to patch plan secret: %w", err) } @@ -247,19 +277,18 @@ func (p *Planner) updatePlanSecret(ctx context.Context, data []byte) error { func (p *Planner) refresh(ctx context.Context) error { rke2Config := &bootstrapv1.RKE2Config{} - if err := p.Client.Get(ctx, client.ObjectKey{Namespace: p.machine.Namespace, Name: p.machine.Spec.Bootstrap.ConfigRef.Name}, rke2Config); err != nil { + if err := p.Client.Get(ctx, client.ObjectKey{ + Namespace: p.machine.Namespace, + Name: p.machine.Spec.Bootstrap.ConfigRef.Name, + }, rke2Config); err != nil { return fmt.Errorf("failed to get RKE2Config: %w", err) } - planSecretName := strings.Join([]string{rke2Config.Name, "rke2config", "plan"}, "-") - secret := &corev1.Secret{} - if err := p.Client.Get(ctx, client.ObjectKey{Namespace: p.machine.Namespace, Name: planSecretName}, secret); err != nil { + if err := p.Client.Get(ctx, client.ObjectKeyFromObject(p.secret), secret); client.IgnoreNotFound(err) != nil { return fmt.Errorf("failed to get plan secret: %w", err) - } - - if secret.Data == nil { - secret.Data = map[string][]byte{} + } else if err == nil { + p.secret = secret } return nil diff --git a/exp/etcdrestore/controllers/snapshotters/rke2snapshotter.go b/exp/etcdrestore/controllers/snapshotters/rke2snapshotter.go index 89e86f8b..f92fc53d 100644 --- a/exp/etcdrestore/controllers/snapshotters/rke2snapshotter.go +++ b/exp/etcdrestore/controllers/snapshotters/rke2snapshotter.go @@ -121,15 +121,16 @@ func (s *RKE2Snapshotter) Sync(ctx context.Context) error { } } - etcdMachineSnapshot := &snapshotrestorev1.EtcdMachineSnapshot{ + etcdMachineSnapshot := &snapshotrestorev1.ETCDMachineSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: snapshotFile.Name, Namespace: s.cluster.Namespace, }, - Spec: snapshotrestorev1.EtcdMachineSnapshotSpec{ + Spec: snapshotrestorev1.ETCDMachineSnapshotSpec{ ClusterName: s.cluster.Name, MachineName: machineName, ConfigRef: snapshotFile.Name, + Location: snapshotFile.Spec.Location, }, } diff --git a/exp/etcdrestore/main.go b/exp/etcdrestore/main.go index f073370a..e9f390b1 100644 --- a/exp/etcdrestore/main.go +++ b/exp/etcdrestore/main.go @@ -224,9 +224,9 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { os.Exit(1) } - setupLog.Info("enabling EtcdMachineSnapshot controller") + setupLog.Info("enabling ETCDMachineSnapshot controller") - if err := (&expcontrollers.EtcdMachineSnapshotReconciler{ + if err := (&expcontrollers.ETCDMachineSnapshotReconciler{ Client: mgr.GetClient(), Tracker: tracker, WatchFilterValue: watchFilterValue, @@ -237,10 +237,8 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { setupLog.Info("enabling EtcdSnapshotRestore controller") - if err := (&expcontrollers.EtcdSnapshotRestoreReconciler{ - Client: mgr.GetClient(), - Tracker: tracker, - WatchFilterValue: watchFilterValue, + if err := (&expcontrollers.ETCDSnapshotRestoreReconciler{ + Client: mgr.GetClient(), }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: concurrencyNumber}); err != nil { setupLog.Error(err, "unable to create EtcdSnapshotRestore controller") os.Exit(1) diff --git a/exp/etcdrestore/webhooks/etcdmachinesnapshot.go b/exp/etcdrestore/webhooks/etcdmachinesnapshot.go index 73bc0fe4..aaeaa1da 100644 --- a/exp/etcdrestore/webhooks/etcdmachinesnapshot.go +++ b/exp/etcdrestore/webhooks/etcdmachinesnapshot.go @@ -44,7 +44,7 @@ var _ webhook.CustomValidator = &EtcdMachineSnapshotWebhook{} // SetupWebhookWithManager sets up and registers the webhook with the manager. func (r *EtcdMachineSnapshotWebhook) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). - For(&snapshotrestorev1.EtcdMachineSnapshot{}). + For(&snapshotrestorev1.ETCDMachineSnapshot{}). WithValidator(r). Complete() } @@ -55,7 +55,7 @@ func (r *EtcdMachineSnapshotWebhook) ValidateCreate(ctx context.Context, obj run logger.Info("Validating EtcdMachineSnapshot") - etcdMachineSnapshot, ok := obj.(*snapshotrestorev1.EtcdMachineSnapshot) + etcdMachineSnapshot, ok := obj.(*snapshotrestorev1.ETCDMachineSnapshot) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a EtcdMachineSnapshot but got a %T", obj)) } @@ -69,7 +69,7 @@ func (r *EtcdMachineSnapshotWebhook) ValidateUpdate(ctx context.Context, oldObj, logger.Info("Validating EtcdMachineSnapshot") - etcdMachineSnapshot, ok := newObj.(*snapshotrestorev1.EtcdMachineSnapshot) + etcdMachineSnapshot, ok := newObj.(*snapshotrestorev1.ETCDMachineSnapshot) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a EtcdMachineSnapshot but got a %T", newObj)) } @@ -82,7 +82,7 @@ func (r *EtcdMachineSnapshotWebhook) ValidateDelete(_ context.Context, obj runti return nil, nil } -func (r *EtcdMachineSnapshotWebhook) validateSpec(ctx context.Context, etcdMachineSnapshot *snapshotrestorev1.EtcdMachineSnapshot) (admission.Warnings, error) { +func (r *EtcdMachineSnapshotWebhook) validateSpec(ctx context.Context, etcdMachineSnapshot *snapshotrestorev1.ETCDMachineSnapshot) (admission.Warnings, error) { var allErrs field.ErrorList if etcdMachineSnapshot.Spec.ClusterName == "" { diff --git a/exp/etcdrestore/webhooks/etcdsnapshotrestore.go b/exp/etcdrestore/webhooks/etcdsnapshotrestore.go index 6b257231..ecf19daf 100644 --- a/exp/etcdrestore/webhooks/etcdsnapshotrestore.go +++ b/exp/etcdrestore/webhooks/etcdsnapshotrestore.go @@ -44,7 +44,7 @@ var _ webhook.CustomValidator = &EtcdSnapshotRestoreWebhook{} // SetupWebhookWithManager sets up and registers the webhook with the manager. func (r *EtcdSnapshotRestoreWebhook) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). - For(&snapshotrestorev1.EtcdSnapshotRestore{}). + For(&snapshotrestorev1.ETCDSnapshotRestore{}). WithValidator(r). Complete() } @@ -55,7 +55,7 @@ func (r *EtcdSnapshotRestoreWebhook) ValidateCreate(ctx context.Context, obj run logger.Info("Validating EtcdSnapshotRestore") - etcdSnapshotRestore, ok := obj.(*snapshotrestorev1.EtcdSnapshotRestore) + etcdSnapshotRestore, ok := obj.(*snapshotrestorev1.ETCDSnapshotRestore) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a EtcdSnapshotRestore but got a %T", obj)) } @@ -73,7 +73,7 @@ func (r *EtcdSnapshotRestoreWebhook) ValidateDelete(_ context.Context, obj runti return nil, nil } -func (r *EtcdSnapshotRestoreWebhook) validateSpec(ctx context.Context, etcdSnapshotRestore *snapshotrestorev1.EtcdSnapshotRestore) (admission.Warnings, error) { +func (r *EtcdSnapshotRestoreWebhook) validateSpec(ctx context.Context, etcdSnapshotRestore *snapshotrestorev1.ETCDSnapshotRestore) (admission.Warnings, error) { var allErrs field.ErrorList if etcdSnapshotRestore.Spec.ClusterName == "" {