diff --git a/Makefile b/Makefile index 5c0ecfc3..62ac1d2b 100644 --- a/Makefile +++ b/Makefile @@ -239,7 +239,7 @@ generate-manifests-api: controller-gen ## Generate ClusterRole and CustomResourc .PHONY: generate-exp-etcdrestore-manifests-api generate-exp-etcdrestore-manifests-api: controller-gen ## Generate ClusterRole and CustomResourceDefinition objects for experimental API. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./exp/etcdrestore/api/..." \ + $(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./exp/etcdrestore/api/v1alpha1/..." \ paths=./exp/etcdrestore/controllers/... \ paths=./exp/etcdrestore/webhooks/... \ output:crd:artifacts:config=./exp/etcdrestore/config/crd/bases \ diff --git a/PROJECT b/PROJECT index 90a2cd33..eb17bc61 100644 --- a/PROJECT +++ b/PROJECT @@ -35,4 +35,13 @@ resources: kind: EtcdSnapshotRestore path: github.com/rancher/turtles/exp/etcdrestore/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: cattle.io + group: turtles-capi + kind: RKE2EtcdMachineSnapshotConfig + path: github.com/rancher/turtles/exp/etcdrestore/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/rancher/k3s/v1/etcdsnapshotfile.go b/api/rancher/k3s/v1/etcdsnapshotfile.go index 830b3c15..7f2d810a 100644 --- a/api/rancher/k3s/v1/etcdsnapshotfile.go +++ b/api/rancher/k3s/v1/etcdsnapshotfile.go @@ -36,6 +36,17 @@ type ETCDSnapshotSpec struct { NodeName string `json:"nodeName"` Location string `json:"location"` Metadata map[string]string `json:"metadata,omitempty"` + S3 *ETCDSnapshotS3 `json:"s3,omitempty"` +} + +// ETCDSnapshotS3 is the struct representing a k3s ETCDSnapshotFile S3. +type ETCDSnapshotS3 struct { + Endpoint string `json:"endpoint,omitempty"` + EndpointCA string `json:"endpointCA,omitempty"` + SkipSSLVerify bool `json:"skipSSLVerify,omitempty"` + Bucket string `json:"bucket,omitempty"` + Region string `json:"region,omitempty"` + Insecure bool `json:"insecure,omitempty"` } // ETCDSnapshotStatus is the status of the k3s ETCDSnapshotFile. diff --git a/api/rancher/k3s/v1/zz_generated.deepcopy.go b/api/rancher/k3s/v1/zz_generated.deepcopy.go index 685a9f9f..8fdeb298 100644 --- a/api/rancher/k3s/v1/zz_generated.deepcopy.go +++ b/api/rancher/k3s/v1/zz_generated.deepcopy.go @@ -83,6 +83,21 @@ func (in *ETCDSnapshotFileList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDSnapshotS3) DeepCopyInto(out *ETCDSnapshotS3) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotS3. +func (in *ETCDSnapshotS3) DeepCopy() *ETCDSnapshotS3 { + if in == nil { + return nil + } + out := new(ETCDSnapshotS3) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ETCDSnapshotSpec) DeepCopyInto(out *ETCDSnapshotSpec) { *out = *in @@ -93,6 +108,11 @@ func (in *ETCDSnapshotSpec) DeepCopyInto(out *ETCDSnapshotSpec) { (*out)[key] = val } } + if in.S3 != nil { + in, out := &in.S3, &out.S3 + *out = new(ETCDSnapshotS3) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotSpec. diff --git a/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml b/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml index 90757d04..0e828c1b 100644 --- a/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml +++ b/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml @@ -47,71 +47,26 @@ spec: type: string configRef: description: |- - ObjectReference contains enough information to let you inspect or modify the referred object. - --- - New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. - 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. - 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular - restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". - Those cannot be well described when embedded. - 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. - 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity - during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple - and the version of the actual struct is irrelevant. - 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type - will affect numerous schemas. Don't make new APIs embed an underspecified API type they do not control. - - - Instead of using this type, create a locally provided and used type that is well-focused on your reference. - For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 . + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic machineName: type: string + manual: + type: boolean required: - clusterName - configRef - machineName + - manual type: object status: default: {} @@ -352,6 +307,111 @@ spec: subresources: status: {} --- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: rancher-turtles-system/rancher-turtles-etcdsnapshotrestore-serving-cert + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + turtles-capi.cattle.io: etcd-restore + name: rke2etcdmachinesnapshotconfigs.turtles-capi.cattle.io +spec: + group: turtles-capi.cattle.io + names: + kind: RKE2EtcdMachineSnapshotConfig + listKind: RKE2EtcdMachineSnapshotConfigList + plural: rke2etcdmachinesnapshotconfigs + singular: rke2etcdmachinesnapshotconfig + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RKE2EtcdMachineSnapshotConfig is the config for the RKE2EtcdMachineSnapshotConfig + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: RKE2EtcdMachineSnapshotConfigSpec defines the desired state + of RKE2EtcdMachineSnapshotConfig + properties: + local: + properties: + dataDir: + type: string + required: + - dataDir + type: object + s3: + properties: + bucket: + type: string + endpoint: + type: string + endpointCAsecret: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + folder: + type: string + insecure: + type: boolean + location: + type: string + region: + type: string + s3CredentialSecret: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + skipSSLVerify: + type: boolean + type: object + required: + - local + - s3 + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -563,6 +623,32 @@ rules: - get - patch - update +- apiGroups: + - turtles-capi.cattle.io + resources: + - rke2etcdmachinesnapshotconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - turtles-capi.cattle.io + resources: + - rke2etcdmachinesnapshotconfigs/finalizers + verbs: + - update +- apiGroups: + - turtles-capi.cattle.io + resources: + - rke2etcdmachinesnapshotconfigs/status + verbs: + - get + - patch + - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshot_types.go b/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshot_types.go index cdca134c..e03c8eae 100644 --- a/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshot_types.go +++ b/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshot_types.go @@ -42,9 +42,10 @@ const ( // EtcdMachineSnapshotSpec defines the desired state of EtcdMachineSnapshot type EtcdMachineSnapshotSpec struct { - ClusterName string `json:"clusterName"` - MachineName string `json:"machineName"` - ConfigRef corev1.ObjectReference `json:"configRef"` + ClusterName string `json:"clusterName"` + MachineName string `json:"machineName"` + ConfigRef corev1.LocalObjectReference `json:"configRef"` + Manual bool `json:"manual"` } // EtcdSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore diff --git a/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshotconfig_types.go b/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshotconfig_types.go index 415e46f1..b60a9429 100644 --- a/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshotconfig_types.go +++ b/exp/etcdrestore/api/v1alpha1/etcdmachinesnapshotconfig_types.go @@ -23,9 +23,8 @@ import ( // RKE2EtcdMachineSnapshotConfigSpec defines the desired state of RKE2EtcdMachineSnapshotConfig type RKE2EtcdMachineSnapshotConfigSpec struct { - Manual bool `json:"manual"` - S3 S3Config `json:"s3"` - Local LocalConfig `json:"local"` + S3 S3Config `json:"s3"` + Local LocalConfig `json:"local"` } type LocalConfig struct { @@ -33,13 +32,15 @@ type LocalConfig struct { } type S3Config struct { - Endpoint string `json:"endpoint"` - EndpointCASecret *corev1.ObjectReference `json:"endpointCAsecret,omitempty"` - EnforceSSLVerify bool `json:"enforceSslVerify,omitempty"` - S3CredentialSecret corev1.ObjectReference `json:"s3CredentialSecret"` - Bucket string `json:"bucket,omitempty"` - Region string `json:"region,omitempty"` - Folder string `json:"folder,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + EndpointCASecret *corev1.LocalObjectReference `json:"endpointCAsecret,omitempty"` + SkipSSLVerify bool `json:"skipSSLVerify,omitempty"` + S3CredentialSecret *corev1.LocalObjectReference `json:"s3CredentialSecret,omitempty"` + Bucket string `json:"bucket,omitempty"` + Region string `json:"region,omitempty"` + Folder string `json:"folder,omitempty"` + Insecure bool `json:"insecure,omitempty"` + Location string `json:"location,omitempty"` } // RKE2EtcdMachineSnapshotConfig is the schema for the snapshot config. diff --git a/exp/etcdrestore/api/v1alpha1/zz_generated.deepcopy.go b/exp/etcdrestore/api/v1alpha1/zz_generated.deepcopy.go index 012e2608..11060da5 100644 --- a/exp/etcdrestore/api/v1alpha1/zz_generated.deepcopy.go +++ b/exp/etcdrestore/api/v1alpha1/zz_generated.deepcopy.go @@ -315,10 +315,14 @@ func (in *S3Config) DeepCopyInto(out *S3Config) { *out = *in if in.EndpointCASecret != nil { in, out := &in.EndpointCASecret, &out.EndpointCASecret - *out = new(v1.ObjectReference) + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.S3CredentialSecret != nil { + in, out := &in.S3CredentialSecret, &out.S3CredentialSecret + *out = new(v1.LocalObjectReference) **out = **in } - out.S3CredentialSecret = in.S3CredentialSecret } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3Config. diff --git a/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdmachinesnapshots.yaml b/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdmachinesnapshots.yaml index a0a99b86..e7de0d1b 100644 --- a/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdmachinesnapshots.yaml +++ b/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_etcdmachinesnapshots.yaml @@ -44,71 +44,26 @@ spec: type: string configRef: description: |- - ObjectReference contains enough information to let you inspect or modify the referred object. - --- - New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. - 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. - 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular - restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". - Those cannot be well described when embedded. - 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. - 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity - during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple - and the version of the actual struct is irrelevant. - 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type - will affect numerous schemas. Don't make new APIs embed an underspecified API type they do not control. - - - Instead of using this type, create a locally provided and used type that is well-focused on your reference. - For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 . + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic machineName: type: string + manual: + type: boolean required: - clusterName - configRef - machineName + - manual type: object status: description: EtcdSnapshotRestoreStatus defines observed state of EtcdSnapshotRestore diff --git a/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_rke2etcdmachinesnapshotconfigs.yaml b/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_rke2etcdmachinesnapshotconfigs.yaml index d4d868e4..aafa28f7 100644 --- a/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_rke2etcdmachinesnapshotconfigs.yaml +++ b/exp/etcdrestore/config/crd/bases/turtles-capi.cattle.io_rke2etcdmachinesnapshotconfigs.yaml @@ -48,8 +48,6 @@ spec: required: - dataDir type: object - manual: - type: boolean s3: properties: bucket: @@ -58,139 +56,43 @@ spec: type: string endpointCAsecret: description: |- - ObjectReference contains enough information to let you inspect or modify the referred object. - --- - New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. - 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. - 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular - restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". - Those cannot be well described when embedded. - 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. - 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity - during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple - and the version of the actual struct is irrelevant. - 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type - will affect numerous schemas. Don't make new APIs embed an underspecified API type they do not control. - - - Instead of using this type, create a locally provided and used type that is well-focused on your reference. - For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 . + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic - enforceSslVerify: - type: boolean folder: type: string + insecure: + type: boolean + location: + type: string region: type: string s3CredentialSecret: description: |- - ObjectReference contains enough information to let you inspect or modify the referred object. - --- - New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. - 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. - 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular - restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". - Those cannot be well described when embedded. - 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. - 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity - during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple - and the version of the actual struct is irrelevant. - 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type - will affect numerous schemas. Don't make new APIs embed an underspecified API type they do not control. - - - Instead of using this type, create a locally provided and used type that is well-focused on your reference. - For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 . + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic - required: - - endpoint - - s3CredentialSecret + skipSSLVerify: + type: boolean type: object required: - local - - manual - s3 type: object type: object diff --git a/exp/etcdrestore/config/crd/kustomization.yaml b/exp/etcdrestore/config/crd/kustomization.yaml index a3c7ecb9..df4a968e 100644 --- a/exp/etcdrestore/config/crd/kustomization.yaml +++ b/exp/etcdrestore/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/turtles-capi.cattle.io_etcdmachinesnapshots.yaml - bases/turtles-capi.cattle.io_etcdsnapshotrestores.yaml +- bases/turtles-capi.cattle.io_rke2etcdmachinesnapshotconfigs.yaml #+kubebuilder:scaffold:crdkustomizeresource patches: @@ -21,6 +22,12 @@ patches: kind: CustomResourceDefinition name: etcdsnapshotrestores.turtles-capi.cattle.io path: patches/turtles-capi.cattle.io_etcdsnapshotrestores.yaml +- target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: etcdsnapshotrestores.turtles-capi.cattle.io + path: patches/turtles-capi.cattle.io_rke2etcdmachinesnapshotconfigs.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. diff --git a/exp/etcdrestore/config/crd/patches/turtles-capi.cattle.io_rke2etcdmachinesnapshotconfigs.yaml b/exp/etcdrestore/config/crd/patches/turtles-capi.cattle.io_rke2etcdmachinesnapshotconfigs.yaml new file mode 100644 index 00000000..875b7a46 --- /dev/null +++ b/exp/etcdrestore/config/crd/patches/turtles-capi.cattle.io_rke2etcdmachinesnapshotconfigs.yaml @@ -0,0 +1,3 @@ +- op: add + path: /spec/versions/0/schema/openAPIV3Schema/properties/status/default + value: {} diff --git a/exp/etcdrestore/config/rbac/role.yaml b/exp/etcdrestore/config/rbac/role.yaml index 1e737a41..f50ad44f 100644 --- a/exp/etcdrestore/config/rbac/role.yaml +++ b/exp/etcdrestore/config/rbac/role.yaml @@ -134,3 +134,29 @@ rules: - get - patch - update +- apiGroups: + - turtles-capi.cattle.io + resources: + - rke2etcdmachinesnapshotconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - turtles-capi.cattle.io + resources: + - rke2etcdmachinesnapshotconfigs/finalizers + verbs: + - update +- apiGroups: + - turtles-capi.cattle.io + resources: + - rke2etcdmachinesnapshotconfigs/status + verbs: + - get + - patch + - update diff --git a/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go b/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go index 3061ce0c..4ce613d0 100644 --- a/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go +++ b/exp/etcdrestore/controllers/etcdmachinesnapshot_controller.go @@ -80,6 +80,11 @@ func (r *EtcdMachineSnapshotReconciler) Reconcile(ctx context.Context, req ctrl. return ctrl.Result{}, err } + if !etcdMachineSnapshot.Spec.Manual { + log.V(5).Info("Skipping snapshot creation for non-manual EtcdMachineSnapshot") + return ctrl.Result{}, nil + } + // Initialize the patch helper. patchHelper, err := patch.NewHelper(etcdMachineSnapshot, r.Client) if err != nil { @@ -227,19 +232,19 @@ func checkSnapshotStatus(ctx context.Context, r *EtcdMachineSnapshotReconciler, // validateETCDSnapshotFile validates the fields of an ETCDSnapshotFile resource. func validateETCDSnapshotFile(snapshotFile k3sv1.ETCDSnapshotFile) error { if snapshotFile.Spec.SnapshotName == "" { - return fmt.Errorf("SnapshotName is empty for etcdsnapshotfile %s", snapshotFile.Name) + return fmt.Errorf("snapshotName is empty for etcdsnapshotfile %s", snapshotFile.Name) } if snapshotFile.Spec.Location == "" { - return fmt.Errorf("Location is empty for etcdsnapshotfile %s", snapshotFile.Name) + return fmt.Errorf("location is empty for etcdsnapshotfile %s", snapshotFile.Name) } if snapshotFile.Spec.NodeName == "" { - return fmt.Errorf("Node name is empty for etcdsnapshotfile %s", snapshotFile.Name) + return fmt.Errorf("node name is empty for etcdsnapshotfile %s", snapshotFile.Name) } if snapshotFile.Status.ReadyToUse == nil { - return fmt.Errorf("ReadyToUse field is nil for etcdsnapshotfile %s", snapshotFile.Name) + return fmt.Errorf("readyToUse field is nil for etcdsnapshotfile %s", snapshotFile.Name) } return nil diff --git a/exp/etcdrestore/controllers/etcdsnaphotsync_controller.go b/exp/etcdrestore/controllers/etcdsnaphotsync_controller.go deleted file mode 100644 index 6952b98e..00000000 --- a/exp/etcdrestore/controllers/etcdsnaphotsync_controller.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright © 2023 - 2024 SUSE LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "fmt" - - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" -) - -// EtcdSnapshotSyncReconciler reconciles a EtcdSnapshotSync object. -type EtcdSnapshotSyncReconciler struct { - Client client.Client - WatchFilterValue string - - controller controller.Controller - Tracker *remote.ClusterCacheTracker -} - -func (r *EtcdSnapshotSyncReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, _ controller.Options) error { - // TODO: Setup predicates for the controller. - c, err := ctrl.NewControllerManagedBy(mgr). - For(&clusterv1.Cluster{}). - Build(r) - if err != nil { - return fmt.Errorf("creating etcdSnapshotSync controller: %w", err) - } - - r.controller = c - - return nil -} - -func (r *EtcdSnapshotSyncReconciler) Reconcile(_ context.Context, _ ctrl.Request) (res ctrl.Result, reterr error) { - return ctrl.Result{}, nil -} diff --git a/exp/etcdrestore/controllers/etcdsnapshotsync_controller.go b/exp/etcdrestore/controllers/etcdsnapshotsync_controller.go new file mode 100644 index 00000000..563a57c1 --- /dev/null +++ b/exp/etcdrestore/controllers/etcdsnapshotsync_controller.go @@ -0,0 +1,163 @@ +/* +Copyright © 2023 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "errors" + "fmt" + "time" + + k3sv1 "github.com/rancher/turtles/api/rancher/k3s/v1" + "github.com/rancher/turtles/exp/etcdrestore/controllers/snapshotters" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/remote" + capiutil "sigs.k8s.io/cluster-api/util" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + RKE2ControlPlaneKind = "RKE2ControlPlane" +) + +// EtcdSnapshotSyncReconciler reconciles a EtcdSnapshotSync object. +type EtcdSnapshotSyncReconciler struct { + client.Client + WatchFilterValue string + + controller controller.Controller + Tracker *remote.ClusterCacheTracker +} + +func (r *EtcdSnapshotSyncReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, _ controller.Options) error { + // TODO: Setup predicates for the controller. + c, err := ctrl.NewControllerManagedBy(mgr). + For(&clusterv1.Cluster{}). + Build(r) + if err != nil { + return fmt.Errorf("creating etcdSnapshotSync controller: %w", err) + } + + r.controller = c + + return nil +} + +//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=rke2etcdmachinesnapshotconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=rke2etcdmachinesnapshotconfigs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=rke2etcdmachinesnapshotconfigs/finalizers,verbs=update + +func (r *EtcdSnapshotSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, reterr error) { + log := log.FromContext(ctx) + + log.Info("Reconciling CAPI cluster and syncing etcd snapshots") + + cluster := &clusterv1.Cluster{} + if err := r.Get(ctx, req.NamespacedName, cluster); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{Requeue: true}, nil + } + + return ctrl.Result{Requeue: true}, err + } + + if cluster.Spec.Paused { + log.Info("Cluster is paused, skipping reconciliation") + return ctrl.Result{}, nil + } + + // Only reconcile RKE2 clusters + if cluster.Spec.ControlPlaneRef.Kind != RKE2ControlPlaneKind { // TODO: Move to predicate + log.Info("Cluster is not an RKE2 cluster, skipping reconciliation") + return ctrl.Result{RequeueAfter: 3 * time.Minute}, nil + } + + if !cluster.Status.ControlPlaneReady { + log.Info("Control plane is not ready, skipping reconciliation") + return ctrl.Result{RequeueAfter: 3 * time.Minute}, nil + } + + if err := r.watchEtcdSnapshotFiles(ctx, cluster); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to start watch on ETCDSnapshotFile: %w", err) + } + + return r.reconcileNormal(ctx, cluster) +} + +func (r *EtcdSnapshotSyncReconciler) reconcileNormal(ctx context.Context, cluster *clusterv1.Cluster) (ctrl.Result, error) { + remoteClient, err := r.Tracker.GetClient(ctx, capiutil.ObjectKey(cluster)) + if err != nil { + return ctrl.Result{}, err + } + + var snapshotter snapshotters.Snapshotter + + switch cluster.Spec.ControlPlaneRef.Kind { + case RKE2ControlPlaneKind: + snapshotter = snapshotters.NewRKE2Snapshotter(r.Client, remoteClient, cluster) + default: + return ctrl.Result{}, fmt.Errorf("unsupported control plane kind: %s", cluster.Spec.ControlPlaneRef.Kind) + } + + if err := snapshotter.Sync(ctx); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to sync etcd snapshots: %w", err) + } + + return ctrl.Result{}, nil +} + +func (r *EtcdSnapshotSyncReconciler) watchEtcdSnapshotFiles(ctx context.Context, cluster *clusterv1.Cluster) error { + log := ctrl.LoggerFrom(ctx) + + log.V(5).Info("Setting up watch on ETCDSnapshotFile") + + return r.Tracker.Watch(ctx, remote.WatchInput{ + Name: "ETCDSnapshotFiles-watcher", + Cluster: capiutil.ObjectKey(cluster), + Watcher: r.controller, + Kind: &k3sv1.ETCDSnapshotFile{}, + EventHandler: handler.EnqueueRequestsFromMapFunc(r.etcdSnapshotFile(ctx, cluster)), + }) +} + +func (r *EtcdSnapshotSyncReconciler) etcdSnapshotFile(ctx context.Context, cluster *clusterv1.Cluster) handler.MapFunc { + log := log.FromContext(ctx) + + return func(_ context.Context, o client.Object) []ctrl.Request { + log.Info("Cluster name", "name", cluster.GetName()) + + gvk := schema.GroupVersionKind{ + Group: "k3s.cattle.io", + Kind: "ETCDSnapshotFile", + Version: "v1", + } + + if o.GetObjectKind().GroupVersionKind() != gvk { + log.Error(errors.New("got a different object"), "objectGVK", o.GetObjectKind().GroupVersionKind()) + return nil + } + + return []reconcile.Request{{NamespacedName: capiutil.ObjectKey(cluster)}} + } +} diff --git a/exp/etcdrestore/controllers/snapshotters/rke2snapshotter.go b/exp/etcdrestore/controllers/snapshotters/rke2snapshotter.go new file mode 100644 index 00000000..6759188f --- /dev/null +++ b/exp/etcdrestore/controllers/snapshotters/rke2snapshotter.go @@ -0,0 +1,167 @@ +/* +Copyright © 2023 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package snapshotters + +import ( + "context" + "fmt" + + k3sv1 "github.com/rancher/turtles/api/rancher/k3s/v1" + snapshotrestorev1 "github.com/rancher/turtles/exp/etcdrestore/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type RKE2Snapshotter struct { + client.Client + remoteClient client.Client + cluster *clusterv1.Cluster +} + +func NewRKE2Snapshotter(client client.Client, remoteClient client.Client, cluster *clusterv1.Cluster) *RKE2Snapshotter { + return &RKE2Snapshotter{ + Client: client, + remoteClient: remoteClient, + cluster: cluster, + } +} + +func (s *RKE2Snapshotter) Sync(ctx context.Context) error { + log := log.FromContext(ctx) + + etcdnapshotFileList := &k3sv1.ETCDSnapshotFileList{} + + if err := s.remoteClient.List(ctx, etcdnapshotFileList); err != nil { + return fmt.Errorf("failed to list etcd snapshot files: %w", err) + } + + for _, snapshotFile := range etcdnapshotFileList.Items { + log.V(5).Info("Found etcd snapshot file", "name", snapshotFile.GetName()) + + readyToUse := *snapshotFile.Status.ReadyToUse + if !readyToUse { + log.V(5).Info("Snapshot is not ready to use, skipping") + continue + } + + machineName, err := s.findMachineForSnapshot(ctx, snapshotFile.Spec.NodeName) + if err != nil { + return fmt.Errorf("failed to find machine to take a snapshot: %w", err) + } + + if machineName == "" { + log.V(5).Info("Machine not found to take a snapshot, skipping. Will try again later.") + continue + } + + rke2EtcdMachineSnapshotConfig := &snapshotrestorev1.RKE2EtcdMachineSnapshotConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: snapshotFile.Name, + Namespace: s.cluster.Namespace, + }, + } + + if snapshotFile.Spec.S3 != nil { + s3EndpointCASecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: snapshotFile.Name + "-s3-endpoint-ca", + Namespace: s.cluster.Namespace, + }, + StringData: map[string]string{ + "ca.crt": snapshotFile.Spec.S3.EndpointCA, + }, + } + + if err := s.Create(ctx, s3EndpointCASecret); err != nil { + if apierrors.IsAlreadyExists(err) { + log.V(5).Info("S3 endpoint CA secret already exists") + } else { + return fmt.Errorf("failed to create S3 endpoint CA secret: %w", err) + } + } + + rke2EtcdMachineSnapshotConfig.Spec.S3 = snapshotrestorev1.S3Config{ + Endpoint: snapshotFile.Spec.S3.Endpoint, + EndpointCASecret: &corev1.LocalObjectReference{ + Name: s3EndpointCASecret.Name, + }, + SkipSSLVerify: snapshotFile.Spec.S3.SkipSSLVerify, + Bucket: snapshotFile.Spec.S3.Bucket, + Region: snapshotFile.Spec.S3.Region, + Insecure: snapshotFile.Spec.S3.Insecure, + Location: snapshotFile.Spec.Location, + } + } else { + rke2EtcdMachineSnapshotConfig.Spec.Local = snapshotrestorev1.LocalConfig{ + DataDir: snapshotFile.Spec.Location, + } + } + + if err := s.Create(ctx, rke2EtcdMachineSnapshotConfig); err != nil { + if apierrors.IsAlreadyExists(err) { + log.V(5).Info("RKE2EtcdMachineSnapshotConfig already exists") + } else { + return fmt.Errorf("failed to create RKE2EtcdMachineSnapshotConfig: %w", err) + } + } + + etcdMachineSnapshot := &snapshotrestorev1.EtcdMachineSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: snapshotFile.Name, + Namespace: s.cluster.Namespace, + }, + Spec: snapshotrestorev1.EtcdMachineSnapshotSpec{ + ClusterName: s.cluster.Name, + MachineName: machineName, + ConfigRef: corev1.LocalObjectReference{ + Name: snapshotFile.Name, + }, + }, + } + + if err := s.Create(ctx, etcdMachineSnapshot); err != nil { + if apierrors.IsAlreadyExists(err) { + log.V(5).Info("EtcdMachineSnapshot already exists") + } else { + return fmt.Errorf("failed to create EtcdMachineSnapshot: %w", err) + } + } + } + + return nil +} + +func (s *RKE2Snapshotter) findMachineForSnapshot(ctx context.Context, nodeName string) (string, error) { + machineList := &clusterv1.MachineList{} + if err := s.List(ctx, machineList, client.InNamespace(s.cluster.Namespace)); err != nil { + return "", fmt.Errorf("failed to list machines: %w", err) + } + + for _, machine := range machineList.Items { + if machine.Spec.ClusterName == s.cluster.Name { + if machine.Status.NodeRef != nil && machine.Status.NodeRef.Name == nodeName { + return machine.Name, nil + } + } + } + + return "", nil +} diff --git a/exp/etcdrestore/controllers/snapshotters/snapshotter.go b/exp/etcdrestore/controllers/snapshotters/snapshotter.go new file mode 100644 index 00000000..29903ad8 --- /dev/null +++ b/exp/etcdrestore/controllers/snapshotters/snapshotter.go @@ -0,0 +1,25 @@ +/* +Copyright © 2023 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package snapshotters + +import ( + "context" +) + +type Snapshotter interface { + Sync(ctx context.Context) error +} diff --git a/exp/etcdrestore/main.go b/exp/etcdrestore/main.go index b464aad5..4aa83a46 100644 --- a/exp/etcdrestore/main.go +++ b/exp/etcdrestore/main.go @@ -25,6 +25,8 @@ import ( bootstrapv1 "github.com/rancher/cluster-api-provider-rke2/bootstrap/api/v1beta1" k3sv1 "github.com/rancher/turtles/api/rancher/k3s/v1" + managementv3 "github.com/rancher/turtles/api/rancher/management/v3" + provisioningv1 "github.com/rancher/turtles/api/rancher/provisioning/v1" snapshotrestorev1 "github.com/rancher/turtles/exp/etcdrestore/api/v1alpha1" expcontrollers "github.com/rancher/turtles/exp/etcdrestore/controllers" expwebhooks "github.com/rancher/turtles/exp/etcdrestore/webhooks" @@ -77,6 +79,8 @@ func init() { utilruntime.Must(snapshotrestorev1.AddToScheme(scheme)) utilruntime.Must(bootstrapv1.AddToScheme(scheme)) utilruntime.Must(k3sv1.AddToScheme(scheme)) + utilruntime.Must(provisioningv1.AddToScheme(scheme)) + utilruntime.Must(managementv3.AddToScheme(scheme)) } // initFlags initializes the flags. diff --git a/exp/etcdrestore/webhooks/rke2config.go b/exp/etcdrestore/webhooks/rke2config.go index c5b82aa5..24b8ebc6 100644 --- a/exp/etcdrestore/webhooks/rke2config.go +++ b/exp/etcdrestore/webhooks/rke2config.go @@ -134,8 +134,8 @@ func (r *RKE2ConfigWebhook) Default(ctx context.Context, obj runtime.Object) err systemAgentVersionSetting := &managementv3.Setting{} if err := r.Get(context.Background(), client.ObjectKey{ Name: "system-agent-version", - }, caSetting); err != nil { - return apierrors.NewBadRequest(fmt.Sprintf("failed to get ca setting: %s", err)) + }, systemAgentVersionSetting); err != nil { + return apierrors.NewBadRequest(fmt.Sprintf("failed to get system agent version setting: %s", err)) } systemAgentVersion := systemAgentVersionSetting.Value diff --git a/tilt/project/Tiltfile b/tilt/project/Tiltfile index e40aba7f..72d9d301 100644 --- a/tilt/project/Tiltfile +++ b/tilt/project/Tiltfile @@ -139,6 +139,7 @@ def update_manager(yaml, containerName, debug, env): debugArgs.append(arg) if debug_insecure_skip_verify: debugArgs.append("--insecure-skip-verify=true") + debugArgs.append("--v=5") c["command"] = cmd print(cmd) if len(debugArgs) == 0: