Skip to content

Commit

Permalink
refactor: tls policy status to state of the world tasks
Browse files Browse the repository at this point in the history 
Signed-off-by: KevFan <chfan@redhat.com>
  • Loading branch information
@KevFan
KevFan committed Oct 2, 2024
1 parent 94e7f83 commit ff91d70
Show file tree
Hide file tree
Showing 14 changed files with 641 additions and 955 deletions.
167 changes: 146 additions & 21 deletions controllers/state_of_the_world.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package controllers
import (
"context"
"fmt"
"reflect"
"strings"
"sync"

Expand All @@ -16,11 +17,14 @@ import (
istioclientnetworkingv1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
istioclientgosecurityv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"k8s.io/utils/env"
ctrlruntime "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/event"
ctrlruntimepredicate "sigs.k8s.io/controller-runtime/pkg/predicate"
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"

kuadrantv1alpha1 "github.com/kuadrant/kuadrant-operator/api/v1alpha1"
Expand All @@ -44,12 +48,41 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
controller.ManagedBy(manager),
controller.WithLogger(logger),
controller.WithClient(client),
controller.WithRunnable("kuadrant watcher", controller.Watch(&kuadrantv1beta1.Kuadrant{}, kuadrantv1beta1.KuadrantResource, metav1.NamespaceAll)),
controller.WithRunnable("dnspolicy watcher", controller.Watch(&kuadrantv1alpha1.DNSPolicy{}, kuadrantv1alpha1.DNSPoliciesResource, metav1.NamespaceAll)),
controller.WithRunnable("tlspolicy watcher", controller.Watch(&kuadrantv1alpha1.TLSPolicy{}, kuadrantv1alpha1.TLSPoliciesResource, metav1.NamespaceAll)),
controller.WithRunnable("authpolicy watcher", controller.Watch(&kuadrantv1beta2.AuthPolicy{}, kuadrantv1beta2.AuthPoliciesResource, metav1.NamespaceAll)),
controller.WithRunnable("ratelimitpolicy watcher", controller.Watch(&kuadrantv1beta2.RateLimitPolicy{}, kuadrantv1beta2.RateLimitPoliciesResource, metav1.NamespaceAll)),
controller.WithRunnable("topology configmap watcher", controller.Watch(&corev1.ConfigMap{}, controller.ConfigMapsResource, operatorNamespace, controller.FilterResourcesByLabel[*corev1.ConfigMap](fmt.Sprintf("%s=true", kuadrant.TopologyLabel)))),
controller.WithRunnable("kuadrant watcher", controller.Watch(
&kuadrantv1beta1.Kuadrant{},
kuadrantv1beta1.KuadrantResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*kuadrantv1beta1.Kuadrant]{})),
),
controller.WithRunnable("dnspolicy watcher", controller.Watch(
&kuadrantv1alpha1.DNSPolicy{},
kuadrantv1alpha1.DNSPoliciesResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*kuadrantv1alpha1.DNSPolicy]{})),
),
controller.WithRunnable("tlspolicy watcher", controller.Watch(
&kuadrantv1alpha1.TLSPolicy{},
kuadrantv1alpha1.TLSPoliciesResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*kuadrantv1alpha1.TLSPolicy]{})),
),
controller.WithRunnable("authpolicy watcher", controller.Watch(
&kuadrantv1beta2.AuthPolicy{},
kuadrantv1beta2.AuthPoliciesResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*kuadrantv1beta2.AuthPolicy]{})),
),
controller.WithRunnable("ratelimitpolicy watcher", controller.Watch(
&kuadrantv1beta2.RateLimitPolicy{},
kuadrantv1beta2.RateLimitPoliciesResource,
metav1.NamespaceAll),
),
controller.WithRunnable("topology configmap watcher", controller.Watch(
&corev1.ConfigMap{},
controller.ConfigMapsResource,
operatorNamespace,
controller.FilterResourcesByLabel[*corev1.ConfigMap](fmt.Sprintf("%s=true", kuadrant.TopologyLabel))),
),
controller.WithPolicyKinds(
kuadrantv1alpha1.DNSPolicyKind,
kuadrantv1alpha1.TLSPolicyKind,
Expand All @@ -70,9 +103,24 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
logger.Info("gateway api is not installed, skipping watches and reconcilers", "err", err)
} else {
controllerOpts = append(controllerOpts,
controller.WithRunnable("gatewayclass watcher", controller.Watch(&gwapiv1.GatewayClass{}, controller.GatewayClassesResource, metav1.NamespaceAll)),
controller.WithRunnable("gateway watcher", controller.Watch(&gwapiv1.Gateway{}, controller.GatewaysResource, metav1.NamespaceAll)),
controller.WithRunnable("httproute watcher", controller.Watch(&gwapiv1.HTTPRoute{}, controller.HTTPRoutesResource, metav1.NamespaceAll)),
controller.WithRunnable("gatewayclass watcher", controller.Watch(
&gwapiv1.GatewayClass{},
controller.GatewayClassesResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*gwapiv1.GatewayClass]{})),
),
controller.WithRunnable("gateway watcher", controller.Watch(
&gwapiv1.Gateway{},
controller.GatewaysResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*gwapiv1.Gateway]{})),
),
controller.WithRunnable("httproute watcher", controller.Watch(
&gwapiv1.HTTPRoute{},
controller.HTTPRoutesResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*gwapiv1.HTTPRoute]{})),
),
)
}

Expand All @@ -81,9 +129,24 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
logger.Info("envoygateway is not installed, skipping related watches and reconcilers", "err", err)
} else {
controllerOpts = append(controllerOpts,
controller.WithRunnable("envoypatchpolicy watcher", controller.Watch(&egv1alpha1.EnvoyPatchPolicy{}, envoygateway.EnvoyPatchPoliciesResource, metav1.NamespaceAll)),
controller.WithRunnable("envoyextensionpolicy watcher", controller.Watch(&egv1alpha1.EnvoyExtensionPolicy{}, envoygateway.EnvoyExtensionPoliciesResource, metav1.NamespaceAll)),
controller.WithRunnable("envoysecuritypolicy watcher", controller.Watch(&egv1alpha1.SecurityPolicy{}, envoygateway.SecurityPoliciesResource, metav1.NamespaceAll)),
controller.WithRunnable("envoypatchpolicy watcher", controller.Watch(
&egv1alpha1.EnvoyPatchPolicy{},
envoygateway.EnvoyPatchPoliciesResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*egv1alpha1.EnvoyPatchPolicy]{})),
),
controller.WithRunnable("envoyextensionpolicy watcher", controller.Watch(
&egv1alpha1.EnvoyExtensionPolicy{},
envoygateway.EnvoyExtensionPoliciesResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*egv1alpha1.EnvoyExtensionPolicy]{})),
),
controller.WithRunnable("envoysecuritypolicy watcher", controller.Watch(
&egv1alpha1.SecurityPolicy{},
envoygateway.SecurityPoliciesResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*egv1alpha1.SecurityPolicy]{})),
),
controller.WithObjectKinds(
envoygateway.EnvoyPatchPolicyGroupKind,
envoygateway.EnvoyExtensionPolicyGroupKind,
Expand All @@ -99,9 +162,24 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
logger.Info("istio is not installed, skipping related watches and reconcilers", "err", err)
} else {
controllerOpts = append(controllerOpts,
controller.WithRunnable("envoyfilter watcher", controller.Watch(&istioclientnetworkingv1alpha3.EnvoyFilter{}, istio.EnvoyFiltersResource, metav1.NamespaceAll)),
controller.WithRunnable("wasmplugin watcher", controller.Watch(&istioclientgoextensionv1alpha1.WasmPlugin{}, istio.WasmPluginsResource, metav1.NamespaceAll)),
controller.WithRunnable("authorizationpolicy watcher", controller.Watch(&istioclientgosecurityv1beta1.AuthorizationPolicy{}, istio.AuthorizationPoliciesResource, metav1.NamespaceAll)),
controller.WithRunnable("envoyfilter watcher", controller.Watch(
&istioclientnetworkingv1alpha3.EnvoyFilter{},
istio.EnvoyFiltersResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*istioclientnetworkingv1alpha3.EnvoyFilter]{})),
),
controller.WithRunnable("wasmplugin watcher", controller.Watch(
&istioclientgoextensionv1alpha1.WasmPlugin{},
istio.WasmPluginsResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*istioclientgoextensionv1alpha1.WasmPlugin]{})),
),
controller.WithRunnable("authorizationpolicy watcher", controller.Watch(
&istioclientgosecurityv1beta1.AuthorizationPolicy{},
istio.AuthorizationPoliciesResource,
metav1.NamespaceAll,
controller.WithPredicates(&ctrlruntimepredicate.TypedGenerationChangedPredicate[*istioclientgosecurityv1beta1.AuthorizationPolicy]{})),
),
controller.WithObjectKinds(
istio.EnvoyFilterGroupKind,
istio.WasmPluginGroupKind,
Expand All @@ -117,15 +195,45 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D
logger.Info("cert manager is not installed, skipping related watches and reconcilers", "err", err)
} else {
controllerOpts = append(controllerOpts,
controller.WithRunnable("certificate watcher", controller.Watch(&certmanagerv1.Certificate{}, CertManagerCertificatesResource, metav1.NamespaceAll)),
controller.WithRunnable("issuers watcher", controller.Watch(&certmanagerv1.Issuer{}, CertManagerIssuersResource, metav1.NamespaceAll)),
controller.WithRunnable("clusterissuers watcher", controller.Watch(&certmanagerv1.Certificate{}, CertMangerClusterIssuersResource, metav1.NamespaceAll)),
controller.WithRunnable("certificate watcher", controller.Watch(
&certmanagerv1.Certificate{},
CertManagerCertificatesResource,
metav1.NamespaceAll),
),
controller.WithRunnable("issuers watcher", controller.Watch(
&certmanagerv1.Issuer{},
CertManagerIssuersResource,
metav1.NamespaceAll,
controller.WithPredicates(ctrlruntimepredicate.TypedFuncs[*certmanagerv1.Issuer]{
UpdateFunc: func(e event.TypedUpdateEvent[*certmanagerv1.Issuer]) bool {
oldStatus := e.ObjectOld.GetStatus()
newStatus := e.ObjectOld.GetStatus()
return !reflect.DeepEqual(oldStatus, newStatus)
},
})),
),
controller.WithRunnable("clusterissuers watcher", controller.Watch(
&certmanagerv1.ClusterIssuer{},
CertMangerClusterIssuersResource,
metav1.NamespaceAll,
controller.WithPredicates(ctrlruntimepredicate.TypedFuncs[*certmanagerv1.ClusterIssuer]{
UpdateFunc: func(e event.TypedUpdateEvent[*certmanagerv1.ClusterIssuer]) bool {
oldStatus := e.ObjectOld.GetStatus()
newStatus := e.ObjectOld.GetStatus()
return !reflect.DeepEqual(oldStatus, newStatus)
},
})),
),
controller.WithObjectKinds(
CertManagerCertificateKind,
CertManagerIssuerKind,
CertManagerClusterIssuerKind,
),
// TODO: add object links
controller.WithObjectLinks(
LinkGatewayToCertificateFunc,
LinkGatewayToIssuerFunc,
LinkGatewayToClusterIssuerFunc,
),
)
// TODO: add tls policy specific tasks to workflow
}
Expand All @@ -135,9 +243,14 @@ func NewPolicyMachineryController(manager ctrlruntime.Manager, client *dynamic.D

func buildReconciler(client *dynamic.DynamicClient) controller.ReconcileFunc {
reconciler := &controller.Workflow{
Precondition: NewEventLogger().Log,
Precondition: (&controller.Workflow{
Precondition: NewEventLogger().Log,
Tasks: []controller.ReconcileFunc{
NewTopologyFileReconciler(client, operatorNamespace).Reconcile,
},
}).Run,
Tasks: []controller.ReconcileFunc{
NewTopologyFileReconciler(client, operatorNamespace).Reconcile,
NewTLSPolicyWorkflow(client).Run,
},
}
return reconciler.Run
Expand Down Expand Up @@ -181,6 +294,11 @@ func (r *TopologyFileReconciler) Reconcile(ctx context.Context, _ []controller.R
if len(existingTopologyConfigMaps) == 0 {
_, err := r.Client.Resource(controller.ConfigMapsResource).Namespace(cm.Namespace).Create(ctx, unstructuredCM, metav1.CreateOptions{})
if err != nil {
if errors.IsAlreadyExists(err) {
// This error can happen when the operator is starting, and the create event for the topology has not being processed.
logger.Info("already created topology configmap, must not be in topology yet")
return nil
}
logger.Error(err, "failed to write topology configmap")
}
return err
Expand Down Expand Up @@ -234,3 +352,10 @@ func (e *EventLogger) Log(ctx context.Context, resourceEvents []controller.Resou

return nil
}

func NewTLSPolicyWorkflow(client *dynamic.DynamicClient) *controller.Workflow {
return &controller.Workflow{
Precondition: NewValidateTLSPolicyTask().Reconcile,
Postcondition: NewTLSPolicyStatusTask(client).Reconcile,
}
}
8 changes: 4 additions & 4 deletions controllers/tlspolicy_certmanager_certificates.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func (r *TLSPolicyReconciler) reconcileCertificates(ctx context.Context, tlsPoli
// Reconcile Certificates for each gateway directly referred by the policy (existing and new)
for _, gw := range append(gwDiffObj.GatewaysWithValidPolicyRef, gwDiffObj.GatewaysMissingPolicyRef...) {
log.V(1).Info("reconcileCertificates: gateway with valid or missing policy ref", "key", gw.Key())
expectedCertificates := r.expectedCertificatesForGateway(ctx, gw.Gateway, tlsPolicy)
expectedCertificates := expectedCertificatesForGateway(ctx, gw.Gateway, tlsPolicy)
if err := r.createOrUpdateGatewayCertificates(ctx, tlsPolicy, expectedCertificates); err != nil {
return fmt.Errorf("error creating and updating expected certificates for gateway %v: %w", gw.Gateway.Name, err)
}
Expand Down Expand Up @@ -102,7 +102,7 @@ func (r *TLSPolicyReconciler) deleteUnexpectedCertificates(ctx context.Context,
return nil
}

func (r *TLSPolicyReconciler) expectedCertificatesForGateway(ctx context.Context, gateway *gatewayapiv1.Gateway, tlsPolicy *v1alpha1.TLSPolicy) []*certmanv1.Certificate {
func expectedCertificatesForGateway(ctx context.Context, gateway *gatewayapiv1.Gateway, tlsPolicy *v1alpha1.TLSPolicy) []*certmanv1.Certificate {
log := crlog.FromContext(ctx)

tlsHosts := make(map[corev1.ObjectReference][]string)
Expand Down Expand Up @@ -130,12 +130,12 @@ func (r *TLSPolicyReconciler) expectedCertificatesForGateway(ctx context.Context

certs := make([]*certmanv1.Certificate, 0, len(tlsHosts))
for secretRef, hosts := range tlsHosts {
certs = append(certs, r.buildCertManagerCertificate(gateway, tlsPolicy, secretRef, hosts))
certs = append(certs, buildCertManagerCertificate(gateway, tlsPolicy, secretRef, hosts))
}
return certs
}

func (r *TLSPolicyReconciler) buildCertManagerCertificate(gateway *gatewayapiv1.Gateway, tlsPolicy *v1alpha1.TLSPolicy, secretRef corev1.ObjectReference, hosts []string) *certmanv1.Certificate {
func buildCertManagerCertificate(gateway *gatewayapiv1.Gateway, tlsPolicy *v1alpha1.TLSPolicy, secretRef corev1.ObjectReference, hosts []string) *certmanv1.Certificate {
tlsCertLabels := commonTLSCertificateLabels(client.ObjectKeyFromObject(gateway), tlsPolicy)

crt := &certmanv1.Certificate{
Expand Down
68 changes: 2 additions & 66 deletions controllers/tlspolicy_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,43 +19,25 @@ package controllers
import (
"context"
"fmt"
"reflect"

"github.com/cert-manager/cert-manager/pkg/apis/certmanager"
certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
crlog "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1"

"github.com/kuadrant/kuadrant-operator/api/v1alpha1"
kuadrantgatewayapi "github.com/kuadrant/kuadrant-operator/pkg/library/gatewayapi"
"github.com/kuadrant/kuadrant-operator/pkg/library/kuadrant"
"github.com/kuadrant/kuadrant-operator/pkg/library/mappers"
"github.com/kuadrant/kuadrant-operator/pkg/library/reconcilers"
)

const TLSPolicyFinalizer = "kuadrant.io/tls-policy"

var (
CertManagerCertificatesResource = certmanagerv1.SchemeGroupVersion.WithResource("certificates")
CertManagerIssuersResource = certmanagerv1.SchemeGroupVersion.WithResource("issuers")
CertMangerClusterIssuersResource = certmanagerv1.SchemeGroupVersion.WithResource("clusterissuers")

CertManagerCertificateKind = schema.GroupKind{Group: certmanager.GroupName, Kind: certmanagerv1.CertificateKind}
CertManagerIssuerKind = schema.GroupKind{Group: certmanager.GroupName, Kind: certmanagerv1.IssuerKind}
CertManagerClusterIssuerKind = schema.GroupKind{Group: certmanager.GroupName, Kind: certmanagerv1.ClusterIssuerKind}
)

// TLSPolicyReconciler reconciles a TLSPolicy object
type TLSPolicyReconciler struct {
*reconcilers.BaseReconciler
Expand Down Expand Up @@ -99,7 +81,7 @@ func (r *TLSPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
if delResErr == nil {
delResErr = err
}
return r.reconcileStatus(ctx, tlsPolicy, targetReferenceObject, kuadrant.NewErrTargetNotFound(tlsPolicy.Kind(), tlsPolicy.GetTargetRef(), delResErr))
return ctrl.Result{}, delResErr
}
return ctrl.Result{}, err
}
Expand All @@ -125,25 +107,9 @@ func (r *TLSPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
return ctrl.Result{}, err
}
}

specErr := r.reconcileResources(ctx, tlsPolicy, targetReferenceObject)

statusResult, statusErr := r.reconcileStatus(ctx, tlsPolicy, targetReferenceObject, specErr)

if specErr != nil {
return ctrl.Result{}, specErr
}

if statusErr != nil {
return ctrl.Result{}, statusErr
}

if statusResult.Requeue {
log.V(1).Info("Reconciling status not finished. Requeing.")
return statusResult, nil
}

return statusResult, statusErr
return ctrl.Result{}, specErr
}

func (r *TLSPolicyReconciler) reconcileResources(ctx context.Context, tlsPolicy *v1alpha1.TLSPolicy, targetNetworkObject client.Object) error {
Expand Down Expand Up @@ -223,39 +189,9 @@ func (r *TLSPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
mappers.WithClient(mgr.GetClient()),
)

issuerStatusChangedPredicate := predicate.Funcs{
UpdateFunc: func(ev event.UpdateEvent) bool {
oldPolicy, ok := ev.ObjectOld.(certmanagerv1.GenericIssuer)
if !ok {
return false
}
newPolicy, ok := ev.ObjectNew.(certmanagerv1.GenericIssuer)
if !ok {
return false
}
oldStatus := oldPolicy.GetStatus()
newStatus := newPolicy.GetStatus()
return !reflect.DeepEqual(oldStatus, newStatus)
},
}

return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.TLSPolicy{}).
Owns(&certmanagerv1.Certificate{}).
Watches(&gatewayapiv1.Gateway{}, handler.EnqueueRequestsFromMapFunc(gatewayEventMapper.Map)).
Watches(
&certmanagerv1.Issuer{},
handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, object client.Object) []reconcile.Request {
return mapIssuerToPolicy(ctx, mgr.GetClient(), r.Logger(), object)
}),
builder.WithPredicates(issuerStatusChangedPredicate),
).
Watches(
&certmanagerv1.ClusterIssuer{},
handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, object client.Object) []reconcile.Request {
return mapClusterIssuerToPolicy(ctx, mgr.GetClient(), r.Logger(), object)
}),
builder.WithPredicates(issuerStatusChangedPredicate),
).
Complete(r)
}
Loading

0 comments on commit ff91d70

Please sign in to comment.