Skip to content

Commit

Permalink
fix: fix the perf test (#796)
Browse files Browse the repository at this point in the history 
* fix perf test

* rename

---------

Co-authored-by: Ryan Zhang <zhangryan@microsoft.com>
  • Loading branch information
@ryanzhang-oss
ryanzhang-oss and Ryan Zhang committed May 15, 2024
1 parent 98d542b commit 0362822
Showing 1 changed file with 26 additions and 40 deletions.
66 changes: 26 additions & 40 deletions hack/loadtest/util/placement.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"go.uber.org/atomic"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilrand "k8s.io/apimachinery/pkg/util/rand"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -112,7 +111,7 @@ func MeasureOnePlacement(ctx context.Context, hubClient client.Client, deadline,
crpCount.Inc()

klog.Infof("verify that the cluster resource placement `%s` is applied", crpName)
fleetSize, clusterNames = collectApplyMetrics(ctx, hubClient, deadline, interval, crpName, currency, fleetSize, clusterNames)
fleetSize, clusterNames = waitForCRPAvailable(ctx, hubClient, deadline, interval, crpName, currency, fleetSize, clusterNames)
if fleetSize == "0" {
return nil
}
Expand All @@ -130,13 +129,13 @@ func MeasureOnePlacement(ctx context.Context, hubClient client.Client, deadline,

// wait for the status of the CRP and make sure all conditions are all true
klog.Infof("verify cluster resource placement `%s` is updated", crpName)
waitForCrpToComplete(ctx, hubClient, deadline, interval, deletionStartTime, crpName, currency, fleetSize)
waitForCrpToCompleteUpdate(ctx, hubClient, deadline, interval, deletionStartTime, crpName, currency, fleetSize)
}
return hubClient.Delete(ctx, crp)
}

// collect the crp apply metrics
func collectApplyMetrics(ctx context.Context, hubClient client.Client, deadline, pollInterval time.Duration, crpName string, currency string, fleetSize string, clusterNames ClusterNames) (string, ClusterNames) {
// waitForCRPAvailable waits for the CRP to be available
func waitForCRPAvailable(ctx context.Context, hubClient client.Client, deadline, pollInterval time.Duration, crpName string, currency string, fleetSize string, clusterNames ClusterNames) (string, ClusterNames) {
startTime := time.Now()
var crp v1beta1.ClusterResourcePlacement
var err error
Expand All @@ -146,28 +145,21 @@ func collectApplyMetrics(ctx context.Context, hubClient client.Client, deadline,
defer timer.Stop()

for {
if err = hubClient.Get(ctx, types.NamespacedName{Name: crpName, Namespace: ""}, &crp); err != nil {
klog.ErrorS(err, "failed to get crp", "crp", crpName)
}
cond := crp.GetCondition(string(v1beta1.ClusterResourcePlacementAppliedConditionType))
select {
case <-timer.C:
// Deadline has been reached
if condition.IsConditionStatusFalse(cond, 1) {
// failed
klog.Infof("the cluster resource placement `%s` failed", crpName)
LoadTestApplyCountMetric.WithLabelValues(currency, fleetSize, "failed").Inc()
applyFailCount.Inc()
return fleetSize, clusterNames
}
// timeout
// Deadline has been reached, timeout
klog.Infof("the cluster resource placement `%s` timeout", crpName)
LoadTestApplyCountMetric.WithLabelValues(currency, fleetSize, "timeout").Inc()
applyTimeoutCount.Inc()
return fleetSize, clusterNames
case <-ticker.C:
// Interval for CRP status check
if condition.IsConditionStatusTrue(cond, 1) {
if err = hubClient.Get(ctx, types.NamespacedName{Name: crpName, Namespace: ""}, &crp); err != nil {
klog.ErrorS(err, "failed to get crp", "crp", crpName)
continue
}
cond := crp.GetCondition(string(v1beta1.ClusterResourcePlacementAvailableConditionType))
if condition.IsConditionStatusTrue(cond, crp.Generation) {
// succeeded
klog.Infof("the cluster resource placement `%s` succeeded", crpName)
endTime := time.Since(startTime)
Expand All @@ -180,11 +172,10 @@ func collectApplyMetrics(ctx context.Context, hubClient client.Client, deadline,
LoadTestApplyCountMetric.WithLabelValues(currency, fleetSize, "succeed").Inc()
applySuccessCount.Inc()
return fleetSize, clusterNames
} else if cond == nil || cond.Status == metav1.ConditionUnknown {
} else if condition.IsConditionStatusFalse(cond, crp.Generation) {
klog.Infof("the cluster resource placement `%s` failed with condition %+v. trying again.", crpName, cond)
} else {
klog.V(2).Infof("the cluster resource placement `%s` is pending", crpName)
} else if condition.IsConditionStatusFalse(cond, 1) {
klog.Infof("the cluster resource placement `%s` failed. trying again.", crpName)
continue
}
}
}
Expand All @@ -206,12 +197,13 @@ func resourcesDeletedCheck(ctx context.Context, hubClient client.Client, deadlin
case <-timer.C:
// Deadline has been reached
// timeout
klog.V(3).Infof("the cluster resource placement `%s` delete timeout", crpName)
klog.Infof("the cluster resource placement `%s` delete timeout", crpName)
return
case <-ticker.C:
// Interval for CRP status check
if err := hubClient.Get(ctx, types.NamespacedName{Name: crpName, Namespace: ""}, &crp); err != nil {
klog.ErrorS(err, "failed to get crp", "crp", crpName)
continue
}
// the only thing it still selects are namespace and crd
if len(crp.Status.SelectedResources) != 2 {
Expand Down Expand Up @@ -239,7 +231,7 @@ func resourcesDeletedCheck(ctx context.Context, hubClient client.Client, deadlin
}

// check crp updated/completed before deletion
func waitForCrpToComplete(ctx context.Context, hubClient client.Client, deadline, pollInterval time.Duration, deletionStartTime time.Time, crpName string, currency string, fleetSize string) {
func waitForCrpToCompleteUpdate(ctx context.Context, hubClient client.Client, deadline, pollInterval time.Duration, deletionStartTime time.Time, crpName string, currency string, fleetSize string) {
var crp v1beta1.ClusterResourcePlacement
var err error
timer := time.NewTimer(deadline)
Expand All @@ -257,35 +249,30 @@ func waitForCrpToComplete(ctx context.Context, hubClient client.Client, deadline
scheduledCond := crp.GetCondition(string(v1beta1.ClusterResourcePlacementScheduledConditionType))
select {
case <-timer.C:
// Deadline has been reached
if condition.IsConditionStatusFalse(appliedCond, 1) {
// failed
klog.V(3).Infof("the cluster resource placement `%s` failed", crpName)
LoadTestUpdateCountMetric.WithLabelValues(currency, fleetSize, "failed").Inc()
updateFailCount.Inc()
return
}
klog.V(3).Infof("the cluster resource placement `%s` timeout", crpName)
LoadTestUpdateCountMetric.WithLabelValues(currency, fleetSize, "timeout").Inc()
updateTimeoutCount.Inc()
return
case <-ticker.C:
// Interval for CRP status check
if condition.IsConditionStatusTrue(appliedCond, 1) && condition.IsConditionStatusTrue(synchronizedCond, 1) && condition.IsConditionStatusTrue(scheduledCond, 1) {
if err = hubClient.Get(ctx, types.NamespacedName{Name: crpName, Namespace: ""}, &crp); err != nil {
klog.ErrorS(err, "failed to get crp", "crp", crpName)
continue
}
if condition.IsConditionStatusTrue(appliedCond, crp.Generation) && condition.IsConditionStatusTrue(synchronizedCond, crp.Generation) && condition.IsConditionStatusTrue(scheduledCond, crp.Generation) {
// succeeded
klog.V(3).Infof("the cluster resource placement `%s` succeeded", crpName)
var endTime = time.Since(deletionStartTime).Seconds()
updateQuantile.Observe(endTime)
UpdateLatencyCountMetric.WithLabelValues(currency, fleetSize, strconv.FormatFloat(endTime, 'f', 3, 64)).Inc()
LoadTestUpdateCountMetric.WithLabelValues(currency, fleetSize, "succeed").Inc()
updateSuccessCount.Inc()

return
} else if appliedCond == nil || appliedCond.Status == metav1.ConditionUnknown || synchronizedCond == nil || synchronizedCond.Status == metav1.ConditionUnknown || scheduledCond == nil || scheduledCond.Status == metav1.ConditionUnknown {
klog.V(3).Infof("the cluster resource placement `%s` is pending", crpName)
} else if condition.IsConditionStatusFalse(appliedCond, 1) {
} else if condition.IsConditionStatusFalse(appliedCond, crp.Generation) {
// failed
klog.V(2).Infof("the cluster resource placement `%s` failed. try again", crpName)
klog.Infof("the cluster resource placement `%s` failed. try again", crpName)
} else {
klog.V(2).Infof("the cluster resource placement `%s` is pending", crpName)
}
}
}
Expand All @@ -294,6 +281,5 @@ func waitForCrpToComplete(ctx context.Context, hubClient client.Client, deadline
func PrintTestMetrics() {
klog.Infof("CRP count %d", crpCount.Load())
klog.InfoS("Placement apply result", "total applySuccessCount", applySuccessCount.Load(), "applyFailCount", applyFailCount.Load(), "applyTimeoutCount", applyTimeoutCount.Load())

klog.InfoS("Placement update result", "total updateSuccessCount", updateSuccessCount.Load(), "updateFailCount", updateFailCount.Load(), "updateTimeoutCount", updateTimeoutCount.Load())
}

0 comments on commit 0362822

Please sign in to comment.