From febc722fd43b41fc7b39c6d327ed00158ff27156 Mon Sep 17 00:00:00 2001 From: Richard Hagen Date: Thu, 5 Sep 2024 12:30:02 +0200 Subject: [PATCH 1/4] Bugfix: Use BatchJobName instead of JobName (#669) --- api/environments/job_handler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/environments/job_handler.go b/api/environments/job_handler.go index 06926f72..e7cc4604 100644 --- a/api/environments/job_handler.go +++ b/api/environments/job_handler.go @@ -113,7 +113,7 @@ func (eh EnvironmentHandler) RestartJob(ctx context.Context, appName, envName, j if err != nil { return err } - if _, err = findJobInRadixBatch(radixBatch, jobName); err != nil { + if _, err = findJobInRadixBatch(radixBatch, batchJobName); err != nil { return err } return jobSchedulerBatch.RestartRadixBatchJob(ctx, eh.accounts.UserAccount.RadixClient, radixBatch, batchJobName) @@ -282,11 +282,11 @@ func getActiveRadixDeployment(appName string, envName string, radixDeploymentMap return nil, fmt.Errorf("no active deployment found for the app %s, environment %s", appName, envName) } -func findJobInRadixBatch(radixBatch *radixv1.RadixBatch, jobName string) (*radixv1.RadixBatchJob, error) { - if job, ok := slice.FindFirst(radixBatch.Spec.Jobs, func(job radixv1.RadixBatchJob) bool { return job.Name == jobName }); ok { +func findJobInRadixBatch(radixBatch *radixv1.RadixBatch, batchJobName string) (*radixv1.RadixBatchJob, error) { + if job, ok := slice.FindFirst(radixBatch.Spec.Jobs, func(job radixv1.RadixBatchJob) bool { return job.Name == batchJobName }); ok { return &job, nil } - return nil, jobNotFoundError(jobName) + return nil, jobNotFoundError(batchJobName) } func (eh EnvironmentHandler) getDeploymentMapAndDeployJobComponents(ctx context.Context, appName string, envName string, jobComponentName string, radixBatch *radixv1.RadixBatch) (map[string]radixv1.RadixDeployment, *radixv1.RadixDeployJobComponent, *radixv1.RadixDeployJobComponent, error) { From 94a410aef702a53dee854ee3ef67110b39e7adf3 Mon Sep 17 00:00:00 2001 From: Elsa Mayra Irgens Date: Tue, 10 Sep 2024 09:49:00 +0200 Subject: [PATCH 2/4] Switch back to run on arm --- radixconfig.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radixconfig.yaml b/radixconfig.yaml index b6a37fa1..ee3b9454 100644 --- a/radixconfig.yaml +++ b/radixconfig.yaml @@ -24,7 +24,7 @@ spec: port: 9090 publicPort: http runtime: - architecture: amd64 + architecture: arm64 monitoring: true monitoringConfig: portName: metrics From b1de696fd718b91e3920719ac7c0e7df413495f0 Mon Sep 17 00:00:00 2001 From: Richard Hagen Date: Wed, 11 Sep 2024 07:59:43 +0200 Subject: [PATCH 3/4] Use Replica Override to manually scale a component (#665) * Use Replica Override to manually scale a component * parse component status * make readable * handle missing k8s deployment * simplify code * use component pods instead of all pods * Deprecate and add reset-scale endpoints * refactor component status * reafctor environment, aux resources, component spec * bugfixes * bugfixesuse status from component, not auxiliar service * upadte tests * Update api/deployments/models/component_deployment.go Co-authored-by: Sergey Smolnikov * Update api/environments/component_handler.go Co-authored-by: Sergey Smolnikov * Update api/deployments/models/component_status.go Co-authored-by: Sergey Smolnikov * Update api/deployments/models/component_status.go Co-authored-by: Sergey Smolnikov * Update api/deployments/models/component_status.go Co-authored-by: Sergey Smolnikov * Update api/deployments/models/component_status.go Co-authored-by: Sergey Smolnikov * fix panic in tests * fix lint bugs * Update api/environments/environment_handler.go Co-authored-by: Sergey Smolnikov * Update api/environments/environment_handler.go Co-authored-by: Sergey Smolnikov * remove unused function * Add documentation * return err if deployment not found * Update api/utils/predicate/kubernetes.go Co-authored-by: Sergey Smolnikov * update swagger * remove outdated null check * remove outdated test, introduce WithComponentStatuserFunc * Test component actions with status * revert go 1.23 * cleanup unused code * update radix-operator --------- Co-authored-by: Sergey Smolnikov --- .github/workflows/pr.yaml | 2 +- Makefile | 2 +- api/deployments/deployment_handler.go | 17 +- .../mock/deployment_handler_mock.go | 15 - api/deployments/models/component_builder.go | 3 + .../models/component_deployment.go | 10 +- api/deployments/models/component_status.go | 58 ++- .../models/component_status_test.go | 73 ++++ api/environments/component_handler.go | 116 +++--- api/environments/component_spec.go | 202 ++--------- api/environments/component_spec_test.go | 219 ------------ api/environments/deployment_updater.go | 10 +- api/environments/environment_controller.go | 175 ++++++++- .../environment_controller_test.go | 333 ++---------------- api/environments/environment_handler.go | 60 +++- api/environments/models/environment_errors.go | 6 +- api/environments/utils.go | 24 -- api/kubequery/radixapplication_test.go | 3 +- api/kubequery/radixdeployment.go | 30 ++ api/kubequery/radixjob_test.go | 3 +- api/kubequery/radixregistration_test.go | 3 +- api/models/auxiliary_resource.go | 16 +- api/models/component.go | 126 +------ api/utils/owner/verify_generation.go | 22 ++ api/utils/predicate/kubernetes.go | 9 +- go.mod | 51 +-- go.sum | 107 +++--- swaggerui/html/swagger.json | 161 ++++++++- 28 files changed, 801 insertions(+), 1055 deletions(-) create mode 100644 api/deployments/models/component_status_test.go delete mode 100644 api/environments/component_spec_test.go delete mode 100644 api/environments/utils.go create mode 100644 api/utils/owner/verify_generation.go diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8d927c94..ac3b20f2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,7 +25,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.59.1 + version: v1.60.3 test: name: Unit Test diff --git a/Makefile b/Makefile index 147637f8..573f4abe 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ ifndef HAS_SWAGGER go install github.com/go-swagger/go-swagger/cmd/swagger@v0.31.0 endif ifndef HAS_GOLANGCI_LINT - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.58.2 + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.3 endif ifndef HAS_MOCKGEN go install github.com/golang/mock/mockgen@v1.6.0 diff --git a/api/deployments/deployment_handler.go b/api/deployments/deployment_handler.go index 9d4a906d..8823a2a2 100644 --- a/api/deployments/deployment_handler.go +++ b/api/deployments/deployment_handler.go @@ -13,7 +13,7 @@ import ( "github.com/equinor/radix-api/models" "github.com/equinor/radix-common/utils/slice" "github.com/equinor/radix-operator/pkg/apis/kube" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" operatorUtils "github.com/equinor/radix-operator/pkg/apis/utils" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" "k8s.io/apimachinery/pkg/api/errors" @@ -28,7 +28,6 @@ type DeployHandler interface { GetDeploymentsForApplicationEnvironment(ctx context.Context, appName, environment string, latest bool) ([]*deploymentModels.DeploymentSummary, error) GetComponentsForDeploymentName(ctx context.Context, appName, deploymentID string) ([]*deploymentModels.Component, error) GetComponentsForDeployment(ctx context.Context, appName, deploymentName, envName string) ([]*deploymentModels.Component, error) - GetLatestDeploymentForApplicationEnvironment(ctx context.Context, appName, environment string) (*deploymentModels.DeploymentSummary, error) GetDeploymentsForPipelineJob(context.Context, string, string) ([]*deploymentModels.DeploymentSummary, error) GetJobComponentDeployments(context.Context, string, string, string) ([]*deploymentModels.DeploymentItem, error) } @@ -64,6 +63,7 @@ func (deploy *deployHandler) GetLogs(ctx context.Context, appName, podName strin return log, nil } + return nil, deploymentModels.NonExistingPod(appName, podName) } @@ -198,7 +198,7 @@ func (deploy *deployHandler) GetDeploymentWithName(ctx context.Context, appName, return dep, nil } -func (deploy *deployHandler) getRadixDeploymentRadixJob(ctx context.Context, appName string, rd *v1.RadixDeployment) (*v1.RadixJob, error) { +func (deploy *deployHandler) getRadixDeploymentRadixJob(ctx context.Context, appName string, rd *radixv1.RadixDeployment) (*radixv1.RadixJob, error) { jobName := rd.GetLabels()[kube.RadixJobNameLabel] radixJob, err := kubequery.GetRadixJob(ctx, deploy.accounts.UserAccount.RadixClient, appName, jobName) if err != nil { @@ -211,7 +211,6 @@ func (deploy *deployHandler) getRadixDeploymentRadixJob(ctx context.Context, app } func (deploy *deployHandler) getEnvironmentNames(ctx context.Context, appName string) ([]string, error) { - radixlabels.ForApplicationName(appName).AsSelector() labelSelector := radixlabels.ForApplicationName(appName).AsSelector() reList, err := deploy.accounts.ServiceAccount.RadixClient.RadixV1().RadixEnvironments().List(ctx, metav1.ListOptions{LabelSelector: labelSelector.String()}) @@ -219,7 +218,7 @@ func (deploy *deployHandler) getEnvironmentNames(ctx context.Context, appName st return nil, err } - return slice.Map(reList.Items, func(re v1.RadixEnvironment) string { + return slice.Map(reList.Items, func(re radixv1.RadixEnvironment) string { return re.Spec.EnvName }), nil } @@ -239,7 +238,7 @@ func (deploy *deployHandler) getDeployments(ctx context.Context, appName string, rdLabelSelector = rdLabelSelector.Add(*jobNameLabel) } - var radixDeploymentList []v1.RadixDeployment + var radixDeploymentList []radixv1.RadixDeployment namespaces := slice.Map(environments, func(env string) string { return operatorUtils.GetEnvironmentNamespace(appName, env) }) for _, ns := range namespaces { rdList, err := deploy.accounts.UserAccount.RadixClient.RadixV1().RadixDeployments(ns).List(ctx, metav1.ListOptions{LabelSelector: rdLabelSelector.String()}) @@ -250,7 +249,7 @@ func (deploy *deployHandler) getDeployments(ctx context.Context, appName string, } appNamespace := operatorUtils.GetAppNamespace(appName) - radixJobMap := make(map[string]*v1.RadixJob) + radixJobMap := make(map[string]*radixv1.RadixJob) if jobName != "" { radixJob, err := deploy.accounts.UserAccount.RadixClient.RadixV1().RadixJobs(appNamespace).Get(ctx, jobName, metav1.GetOptions{}) @@ -278,7 +277,7 @@ func (deploy *deployHandler) getDeployments(ctx context.Context, appName string, rds := sortRdsByActiveFromDesc(radixDeploymentList) var deploymentSummaries []*deploymentModels.DeploymentSummary for _, rd := range rds { - if latest && rd.Status.Condition == v1.DeploymentInactive { + if latest && rd.Status.Condition == radixv1.DeploymentInactive { continue } @@ -298,7 +297,7 @@ func (deploy *deployHandler) getDeployments(ctx context.Context, appName string, return deploymentSummaries, nil } -func sortRdsByActiveFromDesc(rds []v1.RadixDeployment) []v1.RadixDeployment { +func sortRdsByActiveFromDesc(rds []radixv1.RadixDeployment) []radixv1.RadixDeployment { sort.Slice(rds, func(i, j int) bool { if rds[j].Status.ActiveFrom.IsZero() { return true diff --git a/api/deployments/mock/deployment_handler_mock.go b/api/deployments/mock/deployment_handler_mock.go index 59e36e79..5618d231 100644 --- a/api/deployments/mock/deployment_handler_mock.go +++ b/api/deployments/mock/deployment_handler_mock.go @@ -127,21 +127,6 @@ func (mr *MockDeployHandlerMockRecorder) GetJobComponentDeployments(arg0, arg1, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetJobComponentDeployments", reflect.TypeOf((*MockDeployHandler)(nil).GetJobComponentDeployments), arg0, arg1, arg2, arg3) } -// GetLatestDeploymentForApplicationEnvironment mocks base method. -func (m *MockDeployHandler) GetLatestDeploymentForApplicationEnvironment(ctx context.Context, appName, environment string) (*models.DeploymentSummary, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLatestDeploymentForApplicationEnvironment", ctx, appName, environment) - ret0, _ := ret[0].(*models.DeploymentSummary) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetLatestDeploymentForApplicationEnvironment indicates an expected call of GetLatestDeploymentForApplicationEnvironment. -func (mr *MockDeployHandlerMockRecorder) GetLatestDeploymentForApplicationEnvironment(ctx, appName, environment interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLatestDeploymentForApplicationEnvironment", reflect.TypeOf((*MockDeployHandler)(nil).GetLatestDeploymentForApplicationEnvironment), ctx, appName, environment) -} - // GetLogs mocks base method. func (m *MockDeployHandler) GetLogs(ctx context.Context, appName, podName string, sinceTime *time.Time, logLines *int64, previousLog bool) (io.ReadCloser, error) { m.ctrl.T.Helper() diff --git a/api/deployments/models/component_builder.go b/api/deployments/models/component_builder.go index 61f2aef6..deb1a02a 100644 --- a/api/deployments/models/component_builder.go +++ b/api/deployments/models/component_builder.go @@ -54,6 +54,7 @@ type componentBuilder struct { gitTags string resources *radixv1.ResourceRequirements runtime *radixv1.Runtime + replicasOverride *int } func (b *componentBuilder) WithStatus(status ComponentStatus) ComponentBuilder { @@ -99,6 +100,7 @@ func (b *componentBuilder) WithComponent(component radixv1.RadixCommonDeployComp b.commitID = component.GetEnvironmentVariables()[defaults.RadixCommitHashEnvironmentVariable] b.gitTags = component.GetEnvironmentVariables()[defaults.RadixGitTagsEnvironmentVariable] b.runtime = component.GetRuntime() + b.replicasOverride = component.GetReplicasOverride() ports := []Port{} if component.GetPorts() != nil { @@ -252,6 +254,7 @@ func (b *componentBuilder) BuildComponent() (*Component, error) { Variables: variables, Replicas: b.podNames, ReplicaList: b.replicaSummaryList, + ReplicasOverride: b.replicasOverride, SchedulerPort: b.schedulerPort, ScheduledJobPayloadPath: b.scheduledJobPayloadPath, AuxiliaryResource: b.auxResource, diff --git a/api/deployments/models/component_deployment.go b/api/deployments/models/component_deployment.go index a3467f39..ae95f58b 100644 --- a/api/deployments/models/component_deployment.go +++ b/api/deployments/models/component_deployment.go @@ -85,6 +85,14 @@ type Component struct { // required: false ReplicaList []ReplicaSummary `json:"replicaList"` + // Set if manual control of replicas is in place. Not set means automatic control, 0 means stopped and >= 1 is manually scaled. + // + // required: false + // example: 5 + // Extensions: + // x-nullable: true + ReplicasOverride *int `json:"replicasOverride"` + // HorizontalScaling defines horizontal scaling summary for this component // // required: false @@ -585,7 +593,7 @@ func getReplicaType(pod corev1.Pod) ReplicaType { switch { case pod.GetLabels()[kube.RadixPodIsJobSchedulerLabel] == "true": return JobManager - case pod.GetLabels()[kube.RadixPodIsJobAuxObjectLabel] == "true": + case pod.GetLabels()[kube.RadixAuxiliaryComponentTypeLabel] == kube.RadixJobTypeManagerAux: return JobManagerAux case pod.GetLabels()[kube.RadixAuxiliaryComponentTypeLabel] == "oauth": return OAuth2 diff --git a/api/deployments/models/component_status.go b/api/deployments/models/component_status.go index e7838382..a5347606 100644 --- a/api/deployments/models/component_status.go +++ b/api/deployments/models/component_status.go @@ -1,6 +1,15 @@ package models -import appsv1 "k8s.io/api/apps/v1" +import ( + "github.com/equinor/radix-api/api/utils/owner" + commonutils "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/pointers" + operatordefaults "github.com/equinor/radix-operator/pkg/apis/defaults" + "github.com/equinor/radix-operator/pkg/apis/kube" + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/rs/zerolog/log" + appsv1 "k8s.io/api/apps/v1" +) // ComponentStatus Enumeration of the statuses of component type ComponentStatus int @@ -31,15 +40,46 @@ func (p ComponentStatus) String() string { return [...]string{"Stopped", "Consistent", "Reconciling", "Restarting", "Outdated"}[p] } -func ComponentStatusFromDeployment(deployment *appsv1.Deployment) ComponentStatus { - status := ConsistentComponent +type ComponentStatuserFunc func(component radixv1.RadixCommonDeployComponent, kd *appsv1.Deployment, rd *radixv1.RadixDeployment) ComponentStatus + +func ComponentStatusFromDeployment(component radixv1.RadixCommonDeployComponent, kd *appsv1.Deployment, rd *radixv1.RadixDeployment) ComponentStatus { + if kd == nil || kd.GetName() == "" { + return ComponentReconciling + } + replicasUnavailable := kd.Status.UnavailableReplicas + replicasReady := kd.Status.ReadyReplicas + replicas := pointers.Val(kd.Spec.Replicas) + + if isComponentRestarting(component, rd) { + return ComponentRestarting + } + + if !owner.VerifyCorrectObjectGeneration(rd, kd, kube.RadixDeploymentObservedGeneration) { + return ComponentOutdated + } - switch { - case deployment.Status.Replicas == 0: - status = StoppedComponent - case deployment.Status.UnavailableReplicas > 0: - status = ComponentReconciling + if replicas == 0 { + return StoppedComponent } - return status + // Check if component is scaling up or down + if replicasUnavailable > 0 || replicas < replicasReady { + return ComponentReconciling + } + + return ConsistentComponent +} + +func isComponentRestarting(component radixv1.RadixCommonDeployComponent, rd *radixv1.RadixDeployment) bool { + restarted := component.GetEnvironmentVariables()[operatordefaults.RadixRestartEnvironmentVariable] + if restarted == "" { + return false + } + restartedTime, err := commonutils.ParseTimestamp(restarted) + if err != nil { + log.Logger.Warn().Err(err).Msgf("unable to parse restarted time %v, component: %s", restarted, component.GetName()) + return false + } + reconciledTime := rd.Status.Reconciled + return reconciledTime.IsZero() || restartedTime.After(reconciledTime.Time) } diff --git a/api/deployments/models/component_status_test.go b/api/deployments/models/component_status_test.go new file mode 100644 index 00000000..29c592a8 --- /dev/null +++ b/api/deployments/models/component_status_test.go @@ -0,0 +1,73 @@ +package models_test + +import ( + "testing" + "time" + + "github.com/equinor/radix-api/api/deployments/models" + radixutils "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/pointers" + operatordefaults "github.com/equinor/radix-operator/pkg/apis/defaults" + "github.com/equinor/radix-operator/pkg/apis/kube" + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestNoKubeDeployments_IsReconciling(t *testing.T) { + status := models.ComponentStatusFromDeployment(&radixv1.RadixDeployComponent{}, nil, nil) + assert.Equal(t, models.ComponentReconciling, status) +} + +func TestKubeDeploymentsWithRestartLabel_IsRestarting(t *testing.T) { + status := models.ComponentStatusFromDeployment( + &radixv1.RadixDeployComponent{EnvironmentVariables: map[string]string{operatordefaults.RadixRestartEnvironmentVariable: radixutils.FormatTimestamp(time.Now())}}, + createKubeDeployment(0), + &radixv1.RadixDeployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 2}, + Status: radixv1.RadixDeployStatus{Reconciled: metav1.NewTime(time.Now().Add(-10 * time.Minute))}, + }) + + assert.Equal(t, models.ComponentRestarting, status) +} + +func TestKubeDeploymentsWithoutReplicas_IsStopped(t *testing.T) { + status := models.ComponentStatusFromDeployment( + &radixv1.RadixDeployComponent{}, + createKubeDeployment(0), + &radixv1.RadixDeployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 1}, + }) + assert.Equal(t, models.StoppedComponent, status) +} + +func TestKubeDeployment_IsConsistent(t *testing.T) { + status := models.ComponentStatusFromDeployment( + &radixv1.RadixDeployComponent{}, + createKubeDeployment(1), + &radixv1.RadixDeployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 1}, + }) + assert.Equal(t, models.ConsistentComponent, status) +} + +func TestKubeDeployment_IsOutdated(t *testing.T) { + status := models.ComponentStatusFromDeployment( + &radixv1.RadixDeployComponent{}, + createKubeDeployment(1), + &radixv1.RadixDeployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 2}, + }) + assert.Equal(t, models.ComponentOutdated, status) +} + +func createKubeDeployment(replicas int32) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "helloworld", + Annotations: map[string]string{kube.RadixDeploymentObservedGeneration: "1"}, + OwnerReferences: []metav1.OwnerReference{{Controller: pointers.Ptr(true)}}}, + Spec: appsv1.DeploymentSpec{Replicas: pointers.Ptr[int32](replicas)}, + } +} diff --git a/api/environments/component_handler.go b/api/environments/component_handler.go index 2f659548..58e7f9a9 100644 --- a/api/environments/component_handler.go +++ b/api/environments/component_handler.go @@ -7,8 +7,11 @@ import ( deploymentModels "github.com/equinor/radix-api/api/deployments/models" environmentModels "github.com/equinor/radix-api/api/environments/models" + "github.com/equinor/radix-api/api/kubequery" "github.com/equinor/radix-api/api/utils/labelselector" + "github.com/equinor/radix-common/net/http" radixutils "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/defaults" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" operatorUtils "github.com/equinor/radix-operator/pkg/apis/utils" @@ -25,10 +28,29 @@ const ( maxScaleReplicas = 20 ) -// StopComponent Stops a component -func (eh EnvironmentHandler) StopComponent(ctx context.Context, appName, envName, componentName string, ignoreComponentStatusError bool) error { +// ScaleComponent Scale a component replicas +func (eh EnvironmentHandler) ScaleComponent(ctx context.Context, appName, envName, componentName string, replicas int) error { + if replicas < 0 { + return environmentModels.CannotScaleComponentToNegativeReplicas(appName, envName, componentName) + } + if replicas > maxScaleReplicas { + return environmentModels.CannotScaleComponentToMoreThanMaxReplicas(appName, envName, componentName, maxScaleReplicas) + } + log.Ctx(ctx).Info().Msgf("Scaling component %s, %s to %d replicas", componentName, appName, replicas) + updater, err := eh.getRadixCommonComponentUpdater(ctx, appName, envName, componentName) + if err != nil { + return err + } + componentStatus := updater.getComponentStatus() + if !radixutils.ContainsString(validaStatusesToScaleComponent, componentStatus) { + return environmentModels.CannotScaleComponent(appName, envName, componentName, componentStatus) + } + return eh.patchRadixDeploymentWithReplicas(ctx, updater, &replicas) +} - log.Ctx(ctx).Info().Msgf("Stopping component %s, %s", componentName, appName) +// ResetScaledComponent Starts a component +func (eh EnvironmentHandler) ResetScaledComponent(ctx context.Context, appName, envName, componentName string, ignoreComponentStatusError bool) error { + log.Ctx(ctx).Info().Msgf("Resetting manually scaled component %s, %s", componentName, appName) updater, err := eh.getRadixCommonComponentUpdater(ctx, appName, envName, componentName) if err != nil { return err @@ -36,19 +58,19 @@ func (eh EnvironmentHandler) StopComponent(ctx context.Context, appName, envName if updater.getComponentToPatch().GetType() == v1.RadixComponentTypeJob { return environmentModels.JobComponentCanOnlyBeRestarted() } - componentStatus := updater.getComponentStatus() - if strings.EqualFold(componentStatus, deploymentModels.StoppedComponent.String()) { + if updater.getComponentToPatch().GetReplicasOverride() == nil { if ignoreComponentStatusError { return nil } - return environmentModels.CannotStopComponent(appName, componentName, componentStatus) + return environmentModels.CannotResetScaledComponent(appName, componentName) } - return eh.patchRadixDeploymentWithZeroReplicas(ctx, updater) + return eh.patchRadixDeploymentWithReplicas(ctx, updater, nil) } -// StartComponent Starts a component -func (eh EnvironmentHandler) StartComponent(ctx context.Context, appName, envName, componentName string, ignoreComponentStatusError bool) error { - log.Ctx(ctx).Info().Msgf("Starting component %s, %s", componentName, appName) +// StopComponent Stops a component +func (eh EnvironmentHandler) StopComponent(ctx context.Context, appName, envName, componentName string, ignoreComponentStatusError bool) error { + + log.Ctx(ctx).Info().Msgf("Stopping component %s, %s", componentName, appName) updater, err := eh.getRadixCommonComponentUpdater(ctx, appName, envName, componentName) if err != nil { return err @@ -57,13 +79,13 @@ func (eh EnvironmentHandler) StartComponent(ctx context.Context, appName, envNam return environmentModels.JobComponentCanOnlyBeRestarted() } componentStatus := updater.getComponentStatus() - if !strings.EqualFold(componentStatus, deploymentModels.StoppedComponent.String()) { + if strings.EqualFold(componentStatus, deploymentModels.StoppedComponent.String()) { if ignoreComponentStatusError { return nil } - return environmentModels.CannotStartComponent(appName, componentName, componentStatus) + return environmentModels.CannotStopComponent(appName, componentName, componentStatus) } - return eh.patchRadixDeploymentWithReplicasFromConfig(ctx, updater) + return eh.patchRadixDeploymentWithReplicas(ctx, updater, pointers.Ptr(0)) } // RestartComponent Restarts a component @@ -74,7 +96,7 @@ func (eh EnvironmentHandler) RestartComponent(ctx context.Context, appName, envN return err } componentStatus := updater.getComponentStatus() - if !strings.EqualFold(componentStatus, deploymentModels.ConsistentComponent.String()) { + if strings.EqualFold(componentStatus, deploymentModels.StoppedComponent.String()) { if ignoreComponentStatusError { return nil } @@ -87,12 +109,15 @@ func (eh EnvironmentHandler) RestartComponent(ctx context.Context, appName, envN func (eh EnvironmentHandler) RestartComponentAuxiliaryResource(ctx context.Context, appName, envName, componentName, auxType string) error { log.Ctx(ctx).Info().Msgf("Restarting auxiliary resource %s for component %s, %s", auxType, componentName, appName) - deploySummary, err := eh.deployHandler.GetLatestDeploymentForApplicationEnvironment(ctx, appName, envName) + radixDeployment, err := kubequery.GetLatestRadixDeployment(ctx, eh.accounts.UserAccount.RadixClient, appName, envName) if err != nil { return err } + if radixDeployment == nil { + return http.ValidationError(v1.KindRadixDeployment, "no radix deployments found") + } - componentsDto, err := eh.deployHandler.GetComponentsForDeployment(ctx, appName, deploySummary.Name, envName) + componentsDto, err := eh.deployHandler.GetComponentsForDeployment(ctx, appName, radixDeployment.Name, envName) if err != nil { return err } @@ -129,32 +154,12 @@ func (eh EnvironmentHandler) RestartComponentAuxiliaryResource(ctx context.Conte return eh.patchDeploymentForRestart(ctx, &deploymentList.Items[0]) } -// ScaleComponent Scale a component replicas -func (eh EnvironmentHandler) ScaleComponent(ctx context.Context, appName, envName, componentName string, replicas int) error { - if replicas < 0 { - return environmentModels.CannotScaleComponentToNegativeReplicas(appName, envName, componentName) - } - if replicas > maxScaleReplicas { - return environmentModels.CannotScaleComponentToMoreThanMaxReplicas(appName, envName, componentName, maxScaleReplicas) - } - log.Ctx(ctx).Info().Msgf("Scaling component %s, %s to %d replicas", componentName, appName, replicas) - updater, err := eh.getRadixCommonComponentUpdater(ctx, appName, envName, componentName) - if err != nil { - return err - } - componentStatus := updater.getComponentStatus() - if !radixutils.ContainsString(validaStatusesToScaleComponent, componentStatus) { - return environmentModels.CannotScaleComponent(appName, envName, componentName, componentStatus) - } - return eh.patchRadixDeploymentWithReplicas(ctx, updater, replicas) -} - func canDeploymentBeRestarted(deployment *appsv1.Deployment) bool { if deployment == nil { return false } - return deploymentModels.ComponentStatusFromDeployment(deployment) == deploymentModels.ConsistentComponent + return deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 0 } func (eh EnvironmentHandler) patchDeploymentForRestart(ctx context.Context, deployment *appsv1.Deployment) error { @@ -175,14 +180,6 @@ func (eh EnvironmentHandler) patchDeploymentForRestart(ctx context.Context, depl }) } -func getReplicasForComponentInEnvironment(environmentConfig v1.RadixCommonEnvironmentConfig) (*int, error) { - if environmentConfig != nil { - return environmentConfig.GetReplicas(), nil - } - - return nil, nil -} - func (eh EnvironmentHandler) patch(ctx context.Context, namespace, name string, oldJSON, newJSON []byte) error { patchBytes, err := jsonPatch.CreateMergePatch(oldJSON, newJSON) if err != nil { @@ -199,25 +196,9 @@ func (eh EnvironmentHandler) patch(ctx context.Context, namespace, name string, return nil } -func (eh EnvironmentHandler) patchRadixDeploymentWithReplicasFromConfig(ctx context.Context, updater radixDeployCommonComponentUpdater) error { +func (eh EnvironmentHandler) patchRadixDeploymentWithReplicas(ctx context.Context, updater radixDeployCommonComponentUpdater, replicas *int) error { return eh.commit(ctx, updater, func(updater radixDeployCommonComponentUpdater) error { - newReplica := 1 - replicas, err := getReplicasForComponentInEnvironment(updater.getEnvironmentConfig()) - if err != nil { - return err - } - if replicas != nil { - newReplica = *replicas - } - updater.setReplicasToComponent(&newReplica) - updater.setUserMutationTimestampAnnotation(radixutils.FormatTimestamp(time.Now())) - return nil - }) -} - -func (eh EnvironmentHandler) patchRadixDeploymentWithReplicas(ctx context.Context, updater radixDeployCommonComponentUpdater, replicas int) error { - return eh.commit(ctx, updater, func(updater radixDeployCommonComponentUpdater) error { - updater.setReplicasToComponent(&replicas) + updater.setReplicasOverrideToComponent(replicas) updater.setUserMutationTimestampAnnotation(radixutils.FormatTimestamp(time.Now())) return nil }) @@ -235,12 +216,3 @@ func (eh EnvironmentHandler) patchRadixDeploymentWithTimestampInEnvVar(ctx conte return nil }) } - -func (eh EnvironmentHandler) patchRadixDeploymentWithZeroReplicas(ctx context.Context, updater radixDeployCommonComponentUpdater) error { - return eh.commit(ctx, updater, func(updater radixDeployCommonComponentUpdater) error { - newReplica := 0 - updater.setReplicasToComponent(&newReplica) - updater.setUserMutationTimestampAnnotation(radixutils.FormatTimestamp(time.Now())) - return nil - }) -} diff --git a/api/environments/component_spec.go b/api/environments/component_spec.go index f10488e5..de701d9c 100644 --- a/api/environments/component_spec.go +++ b/api/environments/component_spec.go @@ -2,75 +2,66 @@ package environments import ( "context" - "strings" deploymentModels "github.com/equinor/radix-api/api/deployments/models" "github.com/equinor/radix-api/api/kubequery" "github.com/equinor/radix-api/api/models" "github.com/equinor/radix-api/api/utils/event" - "github.com/equinor/radix-api/api/utils/labelselector" - radixutils "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-api/api/utils/predicate" "github.com/equinor/radix-common/utils/slice" "github.com/equinor/radix-operator/pkg/apis/defaults" - "github.com/equinor/radix-operator/pkg/apis/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" crdUtils "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" - "k8s.io/client-go/kubernetes" ) // getComponentStateFromSpec Returns a component with the current state -func getComponentStateFromSpec( - ctx context.Context, - kubeClient kubernetes.Interface, - appName string, - deployment *deploymentModels.DeploymentSummary, - deploymentStatus v1.RadixDeployStatus, - environmentConfig v1.RadixCommonEnvironmentConfig, - component v1.RadixCommonDeployComponent, - hpas []autoscalingv2.HorizontalPodAutoscaler, - scaledObjects []v1alpha1.ScaledObject, -) (*deploymentModels.Component, error) { +func (eh EnvironmentHandler) getComponentStateFromSpec(ctx context.Context, rd *v1.RadixDeployment, component v1.RadixCommonDeployComponent, hpas []autoscalingv2.HorizontalPodAutoscaler, scaledObjects []v1alpha1.ScaledObject) (*deploymentModels.Component, error) { var componentPodNames []string var environmentVariables map[string]string var replicaSummaryList []deploymentModels.ReplicaSummary var auxResource deploymentModels.AuxiliaryResource var horizontalScalingSummary *deploymentModels.HorizontalScalingSummary + deployments, err := kubequery.GetDeploymentsForEnvironment(ctx, eh.accounts.UserAccount.Client, rd.Spec.AppName, rd.Spec.Environment) + if err != nil { + return nil, err + } + pods, err := kubequery.GetPodsForEnvironmentComponents(ctx, eh.accounts.UserAccount.Client, rd.Spec.AppName, rd.Spec.Environment) + if err != nil { + return nil, err + } - envNs := crdUtils.GetEnvironmentNamespace(appName, deployment.Environment) status := deploymentModels.ConsistentComponent - if deployment.ActiveTo == "" { + if rd.Status.ActiveTo.IsZero() { // current active deployment - we get existing pods - componentPods, err := getComponentPodsByNamespace(ctx, kubeClient, envNs, component.GetName()) + componentPods, err := getComponentPodsByNamespace(pods, component.GetName()) if err != nil { return nil, err } + componentPodNames = getPodNames(componentPods) environmentVariables = getRadixEnvironmentVariables(componentPods) - eventList, err := kubequery.GetEventsForEnvironment(ctx, kubeClient, appName, deployment.Environment) + eventList, err := kubequery.GetEventsForEnvironment(ctx, eh.accounts.UserAccount.Client, rd.Spec.AppName, rd.Spec.Environment) if err != nil { return nil, err } lastEventWarnings := event.ConvertToEventWarnings(eventList) replicaSummaryList = getReplicaSummaryList(componentPods, lastEventWarnings) - auxResource, err = getAuxiliaryResources(ctx, kubeClient, appName, component, envNs) + auxResource, err = getAuxiliaryResources(pods, deployments, rd, component) if err != nil { return nil, err } - status, err = getStatusOfActiveDeployment(component, - deploymentStatus, environmentConfig, componentPods) - if err != nil { - return nil, err - } + kd, _ := slice.FindFirst(deployments, predicate.IsDeploymentForComponent(rd.Spec.AppName, component.GetName())) + status = eh.ComponentStatuser(component, &kd, rd) } componentBuilder := deploymentModels.NewComponentBuilder() @@ -83,7 +74,7 @@ func getComponentStateFromSpec( } if component.GetType() == v1.RadixComponentTypeComponent { - horizontalScalingSummary = models.GetHpaSummary(appName, component.GetName(), hpas, scaledObjects) + horizontalScalingSummary = models.GetHpaSummary(rd.Spec.AppName, component.GetName(), hpas, scaledObjects) } return componentBuilder. @@ -105,16 +96,15 @@ func getPodNames(pods []corev1.Pod) []string { return names } -func getComponentPodsByNamespace(ctx context.Context, client kubernetes.Interface, envNs, componentName string) ([]corev1.Pod, error) { +func getComponentPodsByNamespace(allPods []corev1.Pod, componentName string) ([]corev1.Pod, error) { var componentPods []corev1.Pod - pods, err := client.CoreV1().Pods(envNs).List(ctx, metav1.ListOptions{ - LabelSelector: getLabelSelectorForComponentPods(componentName).String(), + + selector := getLabelSelectorForComponentPods(componentName) + pods := slice.FindAll(allPods, func(pod corev1.Pod) bool { + return selector.Matches(labels.Set(pod.Labels)) }) - if err != nil { - return nil, err - } - for _, pod := range pods.Items { + for _, pod := range pods { pod := pod // A previous version of the job-scheduler added the "radix-job-type" label to job pods. @@ -140,51 +130,6 @@ func getLabelSelectorForComponentPods(componentName string) labels.Selector { return labels.NewSelector().Add(*componentNameRequirement, *notJobAuxRequirement) } -func runningReplicaDiffersFromConfig(environmentConfig v1.RadixCommonEnvironmentConfig, actualPods []corev1.Pod) bool { - actualPodsLength := len(actualPods) - if radixutils.IsNil(environmentConfig) { - return actualPodsLength != deployment.DefaultReplicas - } - // No HPA config - if environmentConfig.GetHorizontalScaling() == nil { - if environmentConfig.GetReplicas() != nil { - return actualPodsLength != *environmentConfig.GetReplicas() - } - return actualPodsLength != deployment.DefaultReplicas - } - // With HPA config - if environmentConfig.GetReplicas() != nil && *environmentConfig.GetReplicas() == 0 { - return actualPodsLength != *environmentConfig.GetReplicas() - } - if environmentConfig.GetHorizontalScaling().MinReplicas != nil { - return actualPodsLength < int(*environmentConfig.GetHorizontalScaling().MinReplicas) || - actualPodsLength > int(environmentConfig.GetHorizontalScaling().MaxReplicas) - } - return actualPodsLength < deployment.DefaultReplicas || - actualPodsLength > int(environmentConfig.GetHorizontalScaling().MaxReplicas) -} - -func runningReplicaDiffersFromSpec(component v1.RadixCommonDeployComponent, actualPods []corev1.Pod) bool { - actualPodsLength := len(actualPods) - // No HPA config - if component.GetHorizontalScaling() == nil { - if component.GetReplicas() != nil { - return actualPodsLength != *component.GetReplicas() - } - return actualPodsLength != deployment.DefaultReplicas - } - // With HPA config - if component.GetReplicas() != nil && *component.GetReplicas() == 0 { - return actualPodsLength != *component.GetReplicas() - } - if component.GetHorizontalScaling().MinReplicas != nil { - return actualPodsLength < int(*component.GetHorizontalScaling().MinReplicas) || - actualPodsLength > int(component.GetHorizontalScaling().MaxReplicas) - } - return actualPodsLength < deployment.DefaultReplicas || - actualPodsLength > int(component.GetHorizontalScaling().MaxReplicas) -} - func getRadixEnvironmentVariables(pods []corev1.Pod) map[string]string { radixEnvironmentVariables := make(map[string]string) @@ -207,9 +152,9 @@ func getReplicaSummaryList(pods []corev1.Pod, lastEventWarnings event.LastEventW }) } -func getAuxiliaryResources(ctx context.Context, kubeClient kubernetes.Interface, appName string, component v1.RadixCommonDeployComponent, envNamespace string) (auxResource deploymentModels.AuxiliaryResource, err error) { +func getAuxiliaryResources(podList []corev1.Pod, deploymentList []appsv1.Deployment, deployment *v1.RadixDeployment, component v1.RadixCommonDeployComponent) (auxResource deploymentModels.AuxiliaryResource, err error) { if auth := component.GetAuthentication(); component.IsPublic() && auth != nil && auth.OAuth2 != nil { - auxResource.OAuth2, err = getOAuth2AuxiliaryResource(ctx, kubeClient, appName, component.GetName(), envNamespace) + auxResource.OAuth2, err = getOAuth2AuxiliaryResource(podList, deploymentList, deployment, component) if err != nil { return } @@ -218,9 +163,9 @@ func getAuxiliaryResources(ctx context.Context, kubeClient kubernetes.Interface, return } -func getOAuth2AuxiliaryResource(ctx context.Context, kubeClient kubernetes.Interface, appName, componentName, envNamespace string) (*deploymentModels.OAuth2AuxiliaryResource, error) { +func getOAuth2AuxiliaryResource(podList []corev1.Pod, deploymentList []appsv1.Deployment, deployment *v1.RadixDeployment, component v1.RadixCommonDeployComponent) (*deploymentModels.OAuth2AuxiliaryResource, error) { var oauth2Resource deploymentModels.OAuth2AuxiliaryResource - oauthDeployment, err := getAuxiliaryResourceDeployment(ctx, kubeClient, appName, componentName, envNamespace, defaults.OAuthProxyAuxiliaryComponentType) + oauthDeployment, err := getAuxiliaryResourceDeployment(podList, deploymentList, deployment, component, defaults.OAuthProxyAuxiliaryComponentType) if err != nil { return nil, err } @@ -231,93 +176,18 @@ func getOAuth2AuxiliaryResource(ctx context.Context, kubeClient kubernetes.Inter return &oauth2Resource, nil } -func getAuxiliaryResourceDeployment(ctx context.Context, kubeClient kubernetes.Interface, appName, componentName, envNamespace, auxType string) (*deploymentModels.AuxiliaryResourceDeployment, error) { +func getAuxiliaryResourceDeployment(podList []corev1.Pod, deploymentList []appsv1.Deployment, rd *v1.RadixDeployment, component v1.RadixCommonDeployComponent, auxType string) (*deploymentModels.AuxiliaryResourceDeployment, error) { var auxResourceDeployment deploymentModels.AuxiliaryResourceDeployment - selector := labelselector.ForAuxiliaryResource(appName, componentName, auxType).String() - deployments, err := kubeClient.AppsV1().Deployments(envNamespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) - if err != nil { - return nil, err - } - if len(deployments.Items) == 0 { + kd, ok := slice.FindFirst(deploymentList, predicate.IsDeploymentForAuxComponent(rd.Spec.AppName, component.GetName(), auxType)) + if !ok { auxResourceDeployment.Status = deploymentModels.ComponentReconciling.String() return &auxResourceDeployment, nil } - deployment := deployments.Items[0] - - pods, err := kubeClient.CoreV1().Pods(envNamespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) - if err != nil { - return nil, err - } - auxResourceDeployment.ReplicaList = getReplicaSummaryList(pods.Items, nil) - auxResourceDeployment.Status = deploymentModels.ComponentStatusFromDeployment(&deployment).String() - return &auxResourceDeployment, nil -} - -func runningReplicaIsOutdated(component v1.RadixCommonDeployComponent, actualPods []corev1.Pod) bool { - switch component.GetType() { - case v1.RadixComponentTypeComponent: - return runningComponentReplicaIsOutdated(component, actualPods) - case v1.RadixComponentTypeJob: - return false - default: - return false - } -} - -func runningComponentReplicaIsOutdated(component v1.RadixCommonDeployComponent, actualPods []corev1.Pod) bool { - // Check if running component's image is not the same as active deployment image tag and that active rd image is equal to 'starting' component image tag - componentIsInconsistent := false - for _, pod := range actualPods { - if pod.DeletionTimestamp != nil { - // Pod is in termination phase - continue - } - for _, container := range pod.Spec.Containers { - if container.Image != component.GetImage() { - // Container is running an outdated image - componentIsInconsistent = true - } - } - } - - return componentIsInconsistent -} -func getStatusOfActiveDeployment( - component v1.RadixCommonDeployComponent, - deploymentStatus v1.RadixDeployStatus, - environmentConfig v1.RadixCommonEnvironmentConfig, - pods []corev1.Pod) (deploymentModels.ComponentStatus, error) { + pods := slice.FindAll(podList, predicate.IsPodForAuxComponent(rd.Spec.AppName, rd.Spec.Environment, auxType)) - if component.GetType() == v1.RadixComponentTypeComponent { - if runningReplicaDiffersFromConfig(environmentConfig, pods) && - !runningReplicaDiffersFromSpec(component, pods) && - len(pods) == 0 { - return deploymentModels.StoppedComponent, nil - } - if runningReplicaDiffersFromSpec(component, pods) { - return deploymentModels.ComponentReconciling, nil - } - } else if component.GetType() == v1.RadixComponentTypeJob { - if len(pods) == 0 { - return deploymentModels.StoppedComponent, nil - } - } - if runningReplicaIsOutdated(component, pods) { - return deploymentModels.ComponentOutdated, nil - } - restarted := component.GetEnvironmentVariables()[defaults.RadixRestartEnvironmentVariable] - if strings.EqualFold(restarted, "") { - return deploymentModels.ConsistentComponent, nil - } - restartedTime, err := radixutils.ParseTimestamp(restarted) - if err != nil { - return deploymentModels.ConsistentComponent, err - } - reconciledTime := deploymentStatus.Reconciled - if reconciledTime.IsZero() || restartedTime.After(reconciledTime.Time) { - return deploymentModels.ComponentRestarting, nil - } - return deploymentModels.ConsistentComponent, nil + auxResourceDeployment.ReplicaList = getReplicaSummaryList(pods, nil) + auxResourceDeployment.Status = deploymentModels.ComponentStatusFromDeployment(component, &kd, rd).String() + return &auxResourceDeployment, nil } diff --git a/api/environments/component_spec_test.go b/api/environments/component_spec_test.go deleted file mode 100644 index 5d8374c6..00000000 --- a/api/environments/component_spec_test.go +++ /dev/null @@ -1,219 +0,0 @@ -package environments - -import ( - "testing" - "time" - - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestRunningReplicaDiffersFromConfig_NoHPA(t *testing.T) { - // Test replicas 2, pods 3 - replicas := 2 - raEnvironmentConfig := &v1.RadixEnvironmentConfig{ - Replicas: &replicas, - } - actualPods := []corev1.Pod{{}, {}, {}} - isDifferent := runningReplicaDiffersFromConfig(raEnvironmentConfig, actualPods) - assert.True(t, isDifferent) - - // Test replicas 2, pods 2 - actualPods = []corev1.Pod{{}, {}} - isDifferent = runningReplicaDiffersFromConfig(raEnvironmentConfig, actualPods) - assert.False(t, isDifferent) - - // Test replicas 0, pods 2 - replicas = 0 - isDifferent = runningReplicaDiffersFromConfig(raEnvironmentConfig, actualPods) - assert.True(t, isDifferent) - - // Test replicas nil, pods 2 - raEnvironmentConfig.Replicas = nil - isDifferent = runningReplicaDiffersFromConfig(raEnvironmentConfig, actualPods) - assert.True(t, isDifferent) - - // Test RadixEnvironmentConfig nil - raEnvironmentConfig = nil - isDifferent = runningReplicaDiffersFromConfig(raEnvironmentConfig, actualPods) - assert.True(t, isDifferent) -} -func TestRunningReplicaDiffersFromConfig_WithHPA(t *testing.T) { - // Test replicas 0, pods 3, minReplicas 2, maxReplicas 6 - replicas := 0 - minReplicas := int32(2) - raEnvironmentConfig := &v1.RadixEnvironmentConfig{ - Replicas: &replicas, - HorizontalScaling: &v1.RadixHorizontalScaling{ - MinReplicas: &minReplicas, - MaxReplicas: 6, - }, - } - actualPods := []corev1.Pod{{}, {}, {}} - isDifferent := runningReplicaDiffersFromConfig(raEnvironmentConfig, actualPods) - assert.True(t, isDifferent) - - // Test replicas 4, pods 3, minReplicas 2, maxReplicas 6 - replicas = 4 - isDifferent = runningReplicaDiffersFromConfig(raEnvironmentConfig, actualPods) - assert.False(t, isDifferent) - - // Test replicas 4, pods 1, minReplicas 2, maxReplicas 6 - actualPods = []corev1.Pod{{}} - isDifferent = runningReplicaDiffersFromConfig(raEnvironmentConfig, actualPods) - assert.True(t, isDifferent) - - // Test replicas 4, pods 1, minReplicas nil, maxReplicas 6 - raEnvironmentConfig.HorizontalScaling.MinReplicas = nil - isDifferent = runningReplicaDiffersFromConfig(raEnvironmentConfig, actualPods) - assert.False(t, isDifferent) -} - -func TestRunningReplicaDiffersFromSpec_NoHPA(t *testing.T) { - // Test replicas 0, pods 1 - replicas := 0 - rdComponent := v1.RadixDeployComponent{ - Replicas: &replicas, - } - actualPods := []corev1.Pod{{}} - isDifferent := runningReplicaDiffersFromSpec(&rdComponent, actualPods) - assert.True(t, isDifferent) - - // Test replicas 1, pods 1 - replicas = 1 - isDifferent = runningReplicaDiffersFromSpec(&rdComponent, actualPods) - assert.False(t, isDifferent) - - // Test replicas nil, pods 1 - rdComponent.Replicas = nil - isDifferent = runningReplicaDiffersFromSpec(&rdComponent, actualPods) - assert.False(t, isDifferent) -} - -func TestRunningReplicaDiffersFromSpec_WithHPA(t *testing.T) { - // Test replicas 0, pods 1, minReplicas 2, maxReplicas 6 - replicas := 0 - minReplicas := int32(2) - rdComponent := v1.RadixDeployComponent{ - Replicas: &replicas, - HorizontalScaling: &v1.RadixHorizontalScaling{ - MinReplicas: &minReplicas, - MaxReplicas: 6, - }, - } - actualPods := []corev1.Pod{{}} - isDifferent := runningReplicaDiffersFromSpec(&rdComponent, actualPods) - assert.True(t, isDifferent) - - // Test replicas 1, pods 1, minReplicas 2, maxReplicas 6 - replicas = 1 - isDifferent = runningReplicaDiffersFromSpec(&rdComponent, actualPods) - assert.True(t, isDifferent) - - // Test replicas 1, pods 3, minReplicas 2, maxReplicas 6 - actualPods = []corev1.Pod{{}, {}, {}} - isDifferent = runningReplicaDiffersFromSpec(&rdComponent, actualPods) - assert.False(t, isDifferent) - - // Test replicas 1, pods 3, minReplicas nil, maxReplicas 6 - rdComponent.HorizontalScaling.MinReplicas = nil - isDifferent = runningReplicaDiffersFromSpec(&rdComponent, actualPods) - assert.False(t, isDifferent) -} - -func TestRunningReplicaOutdatedImage(t *testing.T) { - // Test replicas 0, pods 1, minReplicas 2, maxReplicas 6 - replicas := 0 - minReplicas := int32(2) - rdComponent := v1.RadixDeployComponent{ - Image: "not-outdated", - Replicas: &replicas, - HorizontalScaling: &v1.RadixHorizontalScaling{ - MinReplicas: &minReplicas, - MaxReplicas: 6, - }, - } - - actualPods := []corev1.Pod{ - { - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - Image: "outdated", - }, - }, - }, - }, - } - - isOutdated := runningReplicaIsOutdated(&rdComponent, actualPods) - assert.True(t, isOutdated) - -} - -func TestRunningReplicaNotOutdatedImage_(t *testing.T) { - // Test replicas 0, pods 1, minReplicas 2, maxReplicas 6 - replicas := 0 - minReplicas := int32(2) - rdComponent := v1.RadixDeployComponent{ - Image: "not-outdated", - Replicas: &replicas, - HorizontalScaling: &v1.RadixHorizontalScaling{ - MinReplicas: &minReplicas, - MaxReplicas: 6, - }, - } - - actualPods := []corev1.Pod{ - { - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - Image: "not-outdated", - }, - }, - }, - }, - } - - isOutdated := runningReplicaIsOutdated(&rdComponent, actualPods) - assert.False(t, isOutdated) -} -func TestRunningReplicaNotOutdatedImage_TerminatingPod(t *testing.T) { - // Test replicas 0, pods 1, minReplicas 2, maxReplicas 6 - replicas := 0 - minReplicas := int32(2) - rdComponent := v1.RadixDeployComponent{ - Image: "not-outdated", - Replicas: &replicas, - HorizontalScaling: &v1.RadixHorizontalScaling{ - MinReplicas: &minReplicas, - MaxReplicas: 6, - }, - } - - actualPods := []corev1.Pod{ - { - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Now(), - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - Image: "not-outdated", - }, - }, - }, - }, - } - - isOutdated := runningReplicaIsOutdated(&rdComponent, actualPods) - assert.False(t, isOutdated) -} diff --git a/api/environments/deployment_updater.go b/api/environments/deployment_updater.go index bc9106a8..aac423da 100644 --- a/api/environments/deployment_updater.go +++ b/api/environments/deployment_updater.go @@ -13,7 +13,7 @@ type radixDeployCommonComponentUpdater interface { getComponentStatus() string getRadixDeployment() *v1.RadixDeployment getEnvironmentConfig() v1.RadixCommonEnvironmentConfig - setReplicasToComponent(replicas *int) + setReplicasOverrideToComponent(replicas *int) setUserMutationTimestampAnnotation(timestamp string) } @@ -44,8 +44,8 @@ func (updater *radixDeployComponentUpdater) setEnvironmentVariablesToComponent(e updater.base.radixDeployment.Spec.Components[updater.base.componentIndex].SetEnvironmentVariables(envVars) } -func (updater *radixDeployComponentUpdater) setReplicasToComponent(replicas *int) { - updater.base.radixDeployment.Spec.Components[updater.base.componentIndex].Replicas = replicas +func (updater *radixDeployComponentUpdater) setReplicasOverrideToComponent(replicas *int) { + updater.base.radixDeployment.Spec.Components[updater.base.componentIndex].ReplicasOverride = replicas } func (updater *radixDeployComponentUpdater) setUserMutationTimestampAnnotation(timestamp string) { @@ -76,8 +76,8 @@ func (updater *radixDeployJobComponentUpdater) setUserMutationTimestampAnnotatio updater.base.radixDeployment.Annotations[lastUserMutationAnnotation] = timestamp } -func (updater *radixDeployJobComponentUpdater) setReplicasToComponent(replicas *int) { - //job component has always 1 replica +func (updater *radixDeployJobComponentUpdater) setReplicasOverrideToComponent(replicas *int) { + // job component has always 1 replica } func (updater *radixDeployJobComponentUpdater) getComponentStatus() string { diff --git a/api/environments/environment_controller.go b/api/environments/environment_controller.go index a20626a0..6c0abf53 100644 --- a/api/environments/environment_controller.go +++ b/api/environments/environment_controller.go @@ -71,7 +71,12 @@ func (c *environmentController) GetRoutes() models.Routes { models.Route{ Path: rootPath + "/environments/{envName}/components/{componentName}/start", Method: "POST", - HandlerFunc: c.StartComponent, + HandlerFunc: c.ResetScaledComponent, + }, + models.Route{ + Path: rootPath + "/environments/{envName}/components/{componentName}/reset-scale", + Method: "POST", + HandlerFunc: c.ResetScaledComponent, }, models.Route{ Path: rootPath + "/environments/{envName}/components/{componentName}/restart", @@ -91,7 +96,12 @@ func (c *environmentController) GetRoutes() models.Routes { models.Route{ Path: rootPath + "/environments/{envName}/start", Method: "POST", - HandlerFunc: c.StartEnvironment, + HandlerFunc: c.ResetManuallyStoppedComponentsInEnvironment, + }, + models.Route{ + Path: rootPath + "/environments/{envName}/reset-scale", + Method: "POST", + HandlerFunc: c.ResetManuallyStoppedComponentsInEnvironment, }, models.Route{ Path: rootPath + "/environments/{envName}/restart", @@ -106,7 +116,12 @@ func (c *environmentController) GetRoutes() models.Routes { models.Route{ Path: rootPath + "/start", Method: "POST", - HandlerFunc: c.StartApplication, + HandlerFunc: c.ResetManuallyScaledComponentsInApplication, + }, + models.Route{ + Path: rootPath + "/reset-scale", + Method: "POST", + HandlerFunc: c.ResetManuallyScaledComponentsInApplication, }, models.Route{ Path: rootPath + "/restart", @@ -571,11 +586,65 @@ func (c *environmentController) StopComponent(accounts models.Accounts, w http.R c.JSONResponse(w, r, "Success") } +// ResetScaledComponent reset manually scaled component and resumes normal operation +func (c *environmentController) ResetScaledComponent(accounts models.Accounts, w http.ResponseWriter, r *http.Request) { + // swagger:operation POST /applications/{appName}/environments/{envName}/components/{componentName}/reset-scale component resetScaledComponent + // --- + // summary: Reset manually scaled component and resumes normal operation + // parameters: + // - name: appName + // in: path + // description: Name of application + // type: string + // required: true + // - name: envName + // in: path + // description: Name of environment + // type: string + // required: true + // - name: componentName + // in: path + // description: Name of component + // type: string + // required: true + // - name: Impersonate-User + // in: header + // description: Works only with custom setup of cluster. Allow impersonation of test users (Required if Impersonate-Group is set) + // type: string + // required: false + // - name: Impersonate-Group + // in: header + // description: Works only with custom setup of cluster. Allow impersonation of a comma-seperated list of test groups (Required if Impersonate-User is set) + // type: string + // required: false + // responses: + // "200": + // description: "Component started ok" + // "401": + // description: "Unauthorized" + // "404": + // description: "Not found" + appName := mux.Vars(r)["appName"] + envName := mux.Vars(r)["envName"] + componentName := mux.Vars(r)["componentName"] + + environmentHandler := c.environmentHandlerFactory(accounts) + err := environmentHandler.ResetScaledComponent(r.Context(), appName, envName, componentName, false) + + if err != nil { + c.ErrorResponse(w, r, err) + return + } + + c.JSONResponse(w, r, "Success") +} + // StartComponent Starts job func (c *environmentController) StartComponent(accounts models.Accounts, w http.ResponseWriter, r *http.Request) { // swagger:operation POST /applications/{appName}/environments/{envName}/components/{componentName}/start component startComponent // --- - // summary: Start component + // summary: Deprecated Start component. Use reset-scale instead. This does the same thing, but naming is wrong. This endpoint will be removed after 1. september 2025. + // deprecated: true // parameters: // - name: appName // in: path @@ -614,7 +683,7 @@ func (c *environmentController) StartComponent(accounts models.Accounts, w http. componentName := mux.Vars(r)["componentName"] environmentHandler := c.environmentHandlerFactory(accounts) - err := environmentHandler.StartComponent(r.Context(), appName, envName, componentName, false) + err := environmentHandler.ResetScaledComponent(r.Context(), appName, envName, componentName, false) if err != nil { c.ErrorResponse(w, r, err) @@ -728,11 +797,59 @@ func (c *environmentController) StopEnvironment(accounts models.Accounts, w http c.JSONResponse(w, r, "Success") } +// ResetManuallyStoppedComponentsInEnvironment Reset all manually scaled component and resumes normal operation in environment +func (c *environmentController) ResetManuallyStoppedComponentsInEnvironment(accounts models.Accounts, w http.ResponseWriter, r *http.Request) { + // swagger:operation POST /applications/{appName}/environments/{envName}/reset-scale environment resetManuallyScaledComponentsInEnvironment + // --- + // summary: Reset all manually scaled component and resumes normal operation in environment + // parameters: + // - name: appName + // in: path + // description: Name of application + // type: string + // required: true + // - name: envName + // in: path + // description: Name of environment + // type: string + // required: true + // - name: Impersonate-User + // in: header + // description: Works only with custom setup of cluster. Allow impersonation of test users (Required if Impersonate-Group is set) + // type: string + // required: false + // - name: Impersonate-Group + // in: header + // description: Works only with custom setup of cluster. Allow impersonation of a comma-seperated list of test groups (Required if Impersonate-User is set) + // type: string + // required: false + // responses: + // "200": + // description: "Environment started ok" + // "401": + // description: "Unauthorized" + // "404": + // description: "Not found" + appName := mux.Vars(r)["appName"] + envName := mux.Vars(r)["envName"] + + environmentHandler := c.environmentHandlerFactory(accounts) + err := environmentHandler.ResetManuallyStoppedComponentsInEnvironment(r.Context(), appName, envName) + + if err != nil { + c.ErrorResponse(w, r, err) + return + } + + c.JSONResponse(w, r, "Success") +} + // StartEnvironment Starts all components in the environment func (c *environmentController) StartEnvironment(accounts models.Accounts, w http.ResponseWriter, r *http.Request) { // swagger:operation POST /applications/{appName}/environments/{envName}/start environment startEnvironment // --- - // summary: Start all components in the environment + // summary: Deprecated. Use reset-scale instead that does the same thing, but with better naming. This method will be removed after 1. september 2025. + // deprecated: true // parameters: // - name: appName // in: path @@ -765,7 +882,7 @@ func (c *environmentController) StartEnvironment(accounts models.Accounts, w htt envName := mux.Vars(r)["envName"] environmentHandler := c.environmentHandlerFactory(accounts) - err := environmentHandler.StartEnvironment(r.Context(), appName, envName) + err := environmentHandler.ResetManuallyStoppedComponentsInEnvironment(r.Context(), appName, envName) if err != nil { c.ErrorResponse(w, r, err) @@ -869,11 +986,53 @@ func (c *environmentController) StopApplication(accounts models.Accounts, w http c.JSONResponse(w, r, "Success") } +// ResetManuallyScaledComponentsInApplication Resets and resumes normal opperation for all manually stopped components in all environments of the application +func (c *environmentController) ResetManuallyScaledComponentsInApplication(accounts models.Accounts, w http.ResponseWriter, r *http.Request) { + // swagger:operation POST /applications/{appName}/reset-scale application resetManuallyScaledComponentsInApplication + // --- + // summary: Resets and resumes normal opperation for all manually stopped components in all environments of the application + // parameters: + // - name: appName + // in: path + // description: Name of application + // type: string + // required: true + // - name: Impersonate-User + // in: header + // description: Works only with custom setup of cluster. Allow impersonation of test users (Required if Impersonate-Group is set) + // type: string + // required: false + // - name: Impersonate-Group + // in: header + // description: Works only with custom setup of cluster. Allow impersonation of a comma-seperated list of test groups (Required if Impersonate-User is set) + // type: string + // required: false + // responses: + // "200": + // description: "Application started ok" + // "401": + // description: "Unauthorized" + // "404": + // description: "Not found" + appName := mux.Vars(r)["appName"] + + environmentHandler := c.environmentHandlerFactory(accounts) + err := environmentHandler.StartApplication(r.Context(), appName) + + if err != nil { + c.ErrorResponse(w, r, err) + return + } + + c.JSONResponse(w, r, "Success") +} + // StartApplication Starts all components in all environments of the application func (c *environmentController) StartApplication(accounts models.Accounts, w http.ResponseWriter, r *http.Request) { // swagger:operation POST /applications/{appName}/start application startApplication // --- - // summary: Start all components in all environments of the application + // summary: Deprecated. Use reset scale that does the same thing instead. This will be removed after 1. september 2025. + // deprecated: true // parameters: // - name: appName // in: path diff --git a/api/environments/environment_controller_test.go b/api/environments/environment_controller_test.go index 0e97d1b9..8b978eba 100644 --- a/api/environments/environment_controller_test.go +++ b/api/environments/environment_controller_test.go @@ -25,6 +25,7 @@ import ( radixhttp "github.com/equinor/radix-common/net/http" radixutils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/numbers" + "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-common/utils/slice" operatordefaults "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/kube" @@ -43,7 +44,6 @@ import ( "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" kubefake "k8s.io/client-go/kubernetes/fake" @@ -450,155 +450,40 @@ func setupGetDeploymentsTest(t *testing.T, commonTestUtils *commontest.Utils, ap require.NoError(t, err) } -func TestRestartComponent_ApplicationWithDeployment_EnvironmentConsistent(t *testing.T) { - zeroReplicas := 0 - stoppedComponent, startedComponent := "stoppedComponent", "startedComponent" - - // Setup - commonTestUtils, environmentControllerTestUtils, _, client, radixclient, _, _, _, _ := setupTest(t, nil) - rd, _ := createRadixDeploymentWithReplicas(commonTestUtils, anyAppName, anyEnvironment, []ComponentCreatorStruct{ - {name: stoppedComponent, number: 0}, - {name: startedComponent, number: 1}, - }) - - t.Run("Component Restart Succeeds", func(t *testing.T) { - component := findComponentInDeployment(rd, startedComponent) - assert.True(t, *component.Replicas > zeroReplicas) - - // Emulate a started component - _, err := createComponentPod(client, rd.GetNamespace(), startedComponent) - require.NoError(t, err) - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/components/%s/restart", anyAppName, anyEnvironment, startedComponent)) - response := <-responseChannel - assert.Equal(t, http.StatusOK, response.Code) +func TestComponentStatusActions(t *testing.T) { - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - component = findComponentInDeployment(updatedRd, startedComponent) - assert.True(t, *component.Replicas > zeroReplicas) - assert.NotEmpty(t, component.EnvironmentVariables[operatordefaults.RadixRestartEnvironmentVariable]) - }) - - t.Run("Component Restart Fails", func(t *testing.T) { - component := findComponentInDeployment(rd, stoppedComponent) - assert.True(t, *component.Replicas == zeroReplicas) - - // Emulate a stopped component - err := deleteComponentPod(client, rd.GetNamespace(), stoppedComponent) - require.NoError(t, err) - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/components/%s/start", anyAppName, anyEnvironment, stoppedComponent)) - response := <-responseChannel - assert.Equal(t, http.StatusOK, response.Code) - - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - component = findComponentInDeployment(updatedRd, stoppedComponent) - assert.True(t, *component.Replicas > zeroReplicas) - - responseChannel = environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/components/%s/restart", anyAppName, anyEnvironment, stoppedComponent)) - response = <-responseChannel - // Since pods are not appearing out of nowhere with kubernetes-fake, the component will be in - // a reconciling state and cannot be restarted - assert.Equal(t, http.StatusBadRequest, response.Code) - - errorResponse, _ := controllertest.GetErrorResponse(response) - expectedError := environmentModels.CannotRestartComponent(anyAppName, stoppedComponent, deploymentModels.ComponentReconciling.String()) - assert.Equal(t, (expectedError.(*radixhttp.Error)).Message, errorResponse.Message) - }) -} - -func TestStartComponent_ApplicationWithDeployment_EnvironmentConsistent(t *testing.T) { - zeroReplicas := 0 - stoppedComponent1, stoppedComponent2 := "stoppedComponent1", "stoppedComponent2" - - // Setup - commonTestUtils, environmentControllerTestUtils, _, client, radixclient, _, _, _, _ := setupTest(t, nil) - rd, _ := createRadixDeploymentWithReplicas(commonTestUtils, anyAppName, anyEnvironment, []ComponentCreatorStruct{ - {name: stoppedComponent1, number: 0}, - {name: stoppedComponent2, number: 0}, - }) - - t.Run("Component Start Succeeds", func(t *testing.T) { - component := findComponentInDeployment(rd, stoppedComponent1) - assert.True(t, *component.Replicas == zeroReplicas) - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/components/%s/start", anyAppName, anyEnvironment, stoppedComponent1)) - response := <-responseChannel - assert.Equal(t, http.StatusOK, response.Code) - - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - component = findComponentInDeployment(updatedRd, stoppedComponent1) - assert.True(t, *component.Replicas > zeroReplicas) - }) - - t.Run("Component Start Fails", func(t *testing.T) { - component := findComponentInDeployment(rd, stoppedComponent2) - assert.True(t, *component.Replicas == zeroReplicas) - - // Create pod - _, err := createComponentPod(client, rd.GetNamespace(), stoppedComponent2) - require.NoError(t, err) - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/components/%s/start", anyAppName, anyEnvironment, stoppedComponent2)) - response := <-responseChannel - // Since pods are not appearing out of nowhere with kubernetes-fake, the component will be in - // a reconciling state and cannot be started - assert.Equal(t, http.StatusBadRequest, response.Code) - - errorResponse, _ := controllertest.GetErrorResponse(response) - expectedError := environmentModels.CannotStartComponent(anyAppName, stoppedComponent2, deploymentModels.ComponentReconciling.String()) - assert.Equal(t, (expectedError.(*radixhttp.Error)).Message, errorResponse.Message) + scenarios := []ComponentCreatorStruct{ + {scenarioName: "Stop unstopped component", number: 1, name: "comp1", action: "stop", status: deploymentModels.ConsistentComponent, expectedStatus: http.StatusOK}, + {scenarioName: "Stop stopped component", number: 1, name: "comp2", action: "stop", status: deploymentModels.StoppedComponent, expectedStatus: http.StatusBadRequest}, + {scenarioName: "Start stopped component", number: 1, name: "comp3", action: "start", status: deploymentModels.StoppedComponent, expectedStatus: http.StatusOK}, + {scenarioName: "Start started component", number: 1, name: "comp4", action: "start", status: deploymentModels.ConsistentComponent, expectedStatus: http.StatusOK}, + {scenarioName: "Restart started component", number: 1, name: "comp5", action: "restart", status: deploymentModels.ConsistentComponent, expectedStatus: http.StatusOK}, + {scenarioName: "Restart stopped component", number: 1, name: "comp6", action: "restart", status: deploymentModels.StoppedComponent, expectedStatus: http.StatusBadRequest}, + {scenarioName: "Reset manually scaled component", number: 1, name: "comp7", action: "reset-scale", status: deploymentModels.StoppedComponent, expectedStatus: http.StatusOK}, + } - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - component = findComponentInDeployment(updatedRd, stoppedComponent2) - assert.True(t, *component.Replicas == zeroReplicas) - }) -} + // Mock Status + statuser := func(component v1.RadixCommonDeployComponent, kd *appsv1.Deployment, rd *v1.RadixDeployment) deploymentModels.ComponentStatus { + for _, scenario := range scenarios { + if scenario.name == component.GetName() { + return scenario.status + } + } -func TestStopComponent_ApplicationWithDeployment_EnvironmentConsistent(t *testing.T) { - zeroReplicas := 0 - runningComponent, stoppedComponent := "runningComp", "stoppedComponent" + panic("unknown component! ") + } // Setup - commonTestUtils, environmentControllerTestUtils, _, _, radixclient, _, _, _, _ := setupTest(t, nil) - rd, _ := createRadixDeploymentWithReplicas(commonTestUtils, anyAppName, anyEnvironment, []ComponentCreatorStruct{ - {name: runningComponent, number: 3}, - {name: stoppedComponent, number: 0}, - }) + commonTestUtils, environmentControllerTestUtils, _, _, _, _, _, _, _ := setupTest(t, []EnvironmentHandlerOptions{WithComponentStatuserFunc(statuser)}) + _, _ = createRadixDeploymentWithReplicas(commonTestUtils, anyAppName, anyEnvironment, scenarios) - // Test - t.Run("Stop Component Succeeds", func(t *testing.T) { - component := findComponentInDeployment(rd, runningComponent) - assert.True(t, *component.Replicas > zeroReplicas) - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/components/%s/stop", anyAppName, anyEnvironment, runningComponent)) - response := <-responseChannel - // Since pods are not appearing out of nowhere with kubernetes-fake, the component will be in - // a reconciling state because number of replicas in spec > 0. Therefore it can be stopped - assert.Equal(t, http.StatusOK, response.Code) - - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - component = findComponentInDeployment(updatedRd, runningComponent) - assert.True(t, *component.Replicas == zeroReplicas) - }) - - t.Run("Stop Component Fails", func(t *testing.T) { - component := findComponentInDeployment(rd, stoppedComponent) - assert.True(t, *component.Replicas == zeroReplicas) - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/components/%s/stop", anyAppName, anyEnvironment, stoppedComponent)) - response := <-responseChannel - // The component is in a stopped state since replicas in spec = 0, and therefore cannot be stopped again - assert.Equal(t, http.StatusBadRequest, response.Code) - - errorResponse, _ := controllertest.GetErrorResponse(response) - expectedError := environmentModels.CannotStopComponent(anyAppName, stoppedComponent, deploymentModels.StoppedComponent.String()) - assert.Equal(t, (expectedError.(*radixhttp.Error)).Message, errorResponse.Message) - - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - component = findComponentInDeployment(updatedRd, stoppedComponent) - assert.True(t, *component.Replicas == zeroReplicas) - }) + for _, scenario := range scenarios { + t.Run(scenario.scenarioName, func(t *testing.T) { + responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/components/%s/%s", anyAppName, anyEnvironment, scenario.name, scenario.action)) + response := <-responseChannel + assert.Equal(t, scenario.expectedStatus, response.Code) + }) + } } func TestRestartEnvrionment_ApplicationWithDeployment_EnvironmentConsistent(t *testing.T) { @@ -656,114 +541,6 @@ func TestRestartEnvrionment_ApplicationWithDeployment_EnvironmentConsistent(t *t }) } -func TestStartEnvrionment_ApplicationWithDeployment_EnvironmentConsistent(t *testing.T) { - zeroReplicas := 0 - - // Setup - commonTestUtils, environmentControllerTestUtils, _, _, radixclient, _, _, _, _ := setupTest(t, nil) - - // Test - t.Run("Start Environment", func(t *testing.T) { - envName := "fullyStoppedEnv" - rd, _ := createRadixDeploymentWithReplicas(commonTestUtils, anyAppName, envName, []ComponentCreatorStruct{ - {name: "stoppedComponent1", number: 0}, - {name: "stoppedComponent2", number: 0}, - }) - for _, comp := range rd.Spec.Components { - assert.True(t, *comp.Replicas == zeroReplicas) - } - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/start", anyAppName, envName)) - response := <-responseChannel - assert.Equal(t, http.StatusOK, response.Code) - - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - for _, comp := range updatedRd.Spec.Components { - assert.True(t, *comp.Replicas > zeroReplicas) - } - }) - - t.Run("Start Environment with running component", func(t *testing.T) { - envName := "partiallyRunningEnv" - rd, _ := createRadixDeploymentWithReplicas(commonTestUtils, anyAppName, envName, []ComponentCreatorStruct{ - {name: "stoppedComponent", number: 0}, - {name: "runningComponent", number: 7}, - }) - replicaCount := 0 - for _, comp := range rd.Spec.Components { - replicaCount += *comp.Replicas - } - assert.True(t, replicaCount > zeroReplicas) - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/start", anyAppName, envName)) - response := <-responseChannel - assert.Equal(t, http.StatusOK, response.Code) - - errorResponse, _ := controllertest.GetErrorResponse(response) - assert.Nil(t, errorResponse) - - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - updatedReplicaCount := 0 - for _, comp := range updatedRd.Spec.Components { - updatedReplicaCount += *comp.Replicas - } - assert.True(t, updatedReplicaCount > replicaCount) - }) -} - -func TestStopEnvrionment_ApplicationWithDeployment_EnvironmentConsistent(t *testing.T) { - zeroReplicas := 0 - - // Setup - commonTestUtils, environmentControllerTestUtils, _, _, radixclient, _, _, _, _ := setupTest(t, nil) - - // Test - t.Run("Stop Environment", func(t *testing.T) { - envName := "fullyRunningEnv" - rd, _ := createRadixDeploymentWithReplicas(commonTestUtils, anyAppName, envName, []ComponentCreatorStruct{ - {name: "runningComponent1", number: 3}, - {name: "runningComponent2", number: 7}, - }) - for _, comp := range rd.Spec.Components { - assert.True(t, *comp.Replicas > zeroReplicas) - } - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/stop", anyAppName, envName)) - response := <-responseChannel - assert.Equal(t, http.StatusOK, response.Code) - - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - for _, comp := range updatedRd.Spec.Components { - assert.True(t, *comp.Replicas == zeroReplicas) - } - }) - - t.Run("Stop Environment with stopped component", func(t *testing.T) { - envName := "partiallyRunningEnv" - rd, _ := createRadixDeploymentWithReplicas(commonTestUtils, anyAppName, envName, []ComponentCreatorStruct{ - {name: "stoppedComponent", number: 0}, - {name: "runningComponent", number: 7}, - }) - replicaCount := 0 - for _, comp := range rd.Spec.Components { - replicaCount += *comp.Replicas - } - assert.True(t, replicaCount > zeroReplicas) - - responseChannel := environmentControllerTestUtils.ExecuteRequest("POST", fmt.Sprintf("/api/v1/applications/%s/environments/%s/stop", anyAppName, envName)) - response := <-responseChannel - assert.Equal(t, http.StatusOK, response.Code) - - errorResponse, _ := controllertest.GetErrorResponse(response) - assert.Nil(t, errorResponse) - - updatedRd, _ := radixclient.RadixV1().RadixDeployments(rd.GetNamespace()).Get(context.Background(), rd.GetName(), metav1.GetOptions{}) - for _, comp := range updatedRd.Spec.Components { - assert.True(t, *comp.Replicas == zeroReplicas) - } - }) -} - func TestCreateEnvironment(t *testing.T) { // Setup commonTestUtils, environmentControllerTestUtils, _, _, _, _, _, _, _ := setupTest(t, nil) @@ -2896,8 +2673,12 @@ func initHandler(client kubernetes.Interface, } type ComponentCreatorStruct struct { - name string - number int + name string + number int + action string + status deploymentModels.ComponentStatus + expectedStatus int + scenarioName string } func createRadixDeploymentWithReplicas(tu *commontest.Utils, appName, envName string, components []ComponentCreatorStruct) (*v1.RadixDeployment, error) { @@ -2908,7 +2689,8 @@ func createRadixDeploymentWithReplicas(tu *commontest.Utils, appName, envName st operatorutils. NewDeployComponentBuilder(). WithName(component.name). - WithReplicas(numbers.IntPtr(component.number)), + WithReplicas(pointers.Ptr(component.number)). + WithReplicasOverride(pointers.Ptr(component.number)), ) } @@ -2925,44 +2707,6 @@ func createRadixDeploymentWithReplicas(tu *commontest.Utils, appName, envName st return rd, err } -func createComponentPod(kubeclient kubernetes.Interface, namespace, componentName string) (*corev1.Pod, error) { - podSpec := getPodSpec(componentName) - return kubeclient.CoreV1().Pods(namespace).Create(context.Background(), podSpec, metav1.CreateOptions{}) -} - -func deleteComponentPod(kubeclient kubernetes.Interface, namespace, componentName string) error { - err := kubeclient.CoreV1().Pods(namespace).Delete(context.Background(), getComponentPodName(componentName), metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - return err - } - return nil -} - -func findComponentInDeployment(rd *v1.RadixDeployment, componentName string) *v1.RadixDeployComponent { - for _, comp := range rd.Spec.Components { - if comp.Name == componentName { - return &comp - } - } - - return nil -} - -func getPodSpec(componentName string) *corev1.Pod { - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: getComponentPodName(componentName), - Labels: map[string]string{ - kube.RadixComponentLabel: componentName, - }, - }, - } -} - -func getComponentPodName(componentName string) string { - return fmt.Sprintf("%s-pod", componentName) -} - func contains(secrets []secretModels.Secret, name string) bool { for _, secret := range secrets { if secret.Name == name { @@ -2980,8 +2724,7 @@ func assertBatchDeleted(t *testing.T, rc radixclient.Interface, ns, batchName st require.Nil(t, err) } else { // not deletable - require.Nil(t, updatedBatch) - require.NotNil(t, err) + require.Error(t, err) } } diff --git a/api/environments/environment_handler.go b/api/environments/environment_handler.go index 100d8e74..0665ed7e 100644 --- a/api/environments/environment_handler.go +++ b/api/environments/environment_handler.go @@ -18,6 +18,7 @@ import ( "github.com/equinor/radix-api/api/utils/predicate" "github.com/equinor/radix-api/api/utils/tlsvalidation" "github.com/equinor/radix-api/models" + "github.com/equinor/radix-common/net/http" radixutils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/slice" deployUtils "github.com/equinor/radix-operator/pkg/apis/deployment" @@ -57,6 +58,12 @@ func WithTLSValidator(validator tlsvalidation.Validator) EnvironmentHandlerOptio } } +func WithComponentStatuserFunc(statuser deploymentModels.ComponentStatuserFunc) EnvironmentHandlerOptions { + return func(eh *EnvironmentHandler) { + eh.ComponentStatuser = statuser + } +} + // EnvironmentHandlerFactory defines a factory function for EnvironmentHandler type EnvironmentHandlerFactory func(accounts models.Accounts) EnvironmentHandler @@ -75,10 +82,11 @@ func NewEnvironmentHandlerFactory(opts ...EnvironmentHandlerOptions) Environment // EnvironmentHandler Instance variables type EnvironmentHandler struct { - deployHandler deployments.DeployHandler - eventHandler events.EventHandler - accounts models.Accounts - tlsValidator tlsvalidation.Validator + deployHandler deployments.DeployHandler + eventHandler events.EventHandler + accounts models.Accounts + tlsValidator tlsvalidation.Validator + ComponentStatuser deploymentModels.ComponentStatuserFunc } var validaStatusesToScaleComponent []string @@ -89,7 +97,9 @@ var validaStatusesToScaleComponent []string func Init(opts ...EnvironmentHandlerOptions) EnvironmentHandler { validaStatusesToScaleComponent = []string{deploymentModels.ConsistentComponent.String(), deploymentModels.StoppedComponent.String()} - eh := EnvironmentHandler{} + eh := EnvironmentHandler{ + ComponentStatuser: deploymentModels.ComponentStatusFromDeployment, + } for _, opt := range opts { opt(&eh) @@ -304,10 +314,13 @@ func (eh EnvironmentHandler) GetAuxiliaryResourcePodLog(ctx context.Context, app // StopEnvironment Stops all components in the environment func (eh EnvironmentHandler) StopEnvironment(ctx context.Context, appName, envName string) error { - _, radixDeployment, err := eh.getRadixDeployment(ctx, appName, envName) + radixDeployment, err := kubequery.GetLatestRadixDeployment(ctx, eh.accounts.UserAccount.RadixClient, appName, envName) if err != nil { return err } + if radixDeployment == nil { + return http.ValidationError(v1.KindRadixDeployment, "no radix deployments found") + } log.Ctx(ctx).Info().Msgf("Stopping components in environment %s, %s", envName, appName) for _, deployComponent := range radixDeployment.Spec.Components { @@ -319,18 +332,22 @@ func (eh EnvironmentHandler) StopEnvironment(ctx context.Context, appName, envNa return nil } -// StartEnvironment Starts all components in the environment -func (eh EnvironmentHandler) StartEnvironment(ctx context.Context, appName, envName string) error { - _, radixDeployment, err := eh.getRadixDeployment(ctx, appName, envName) +// ResetManuallyStoppedComponentsInEnvironment Starts all components in the environment +func (eh EnvironmentHandler) ResetManuallyStoppedComponentsInEnvironment(ctx context.Context, appName, envName string) error { + radixDeployment, err := kubequery.GetLatestRadixDeployment(ctx, eh.accounts.UserAccount.RadixClient, appName, envName) if err != nil { return err } + if radixDeployment == nil { + return http.ValidationError(v1.KindRadixDeployment, "no radix deployments found") + } log.Ctx(ctx).Info().Msgf("Starting components in environment %s, %s", envName, appName) for _, deployComponent := range radixDeployment.Spec.Components { - err := eh.StartComponent(ctx, appName, envName, deployComponent.GetName(), true) - if err != nil { - return err + if override := deployComponent.GetReplicasOverride(); override != nil && *override == 0 { + if err := eh.ResetScaledComponent(ctx, appName, envName, deployComponent.GetName(), true); err != nil { + return err + } } } return nil @@ -338,10 +355,13 @@ func (eh EnvironmentHandler) StartEnvironment(ctx context.Context, appName, envN // RestartEnvironment Restarts all components in the environment func (eh EnvironmentHandler) RestartEnvironment(ctx context.Context, appName, envName string) error { - _, radixDeployment, err := eh.getRadixDeployment(ctx, appName, envName) + radixDeployment, err := kubequery.GetLatestRadixDeployment(ctx, eh.accounts.UserAccount.RadixClient, appName, envName) if err != nil { return err } + if radixDeployment == nil { + return http.ValidationError(v1.KindRadixDeployment, "no radix deployments found") + } log.Ctx(ctx).Info().Msgf("Restarting components in environment %s, %s", envName, appName) for _, deployComponent := range radixDeployment.Spec.Components { @@ -377,7 +397,7 @@ func (eh EnvironmentHandler) StartApplication(ctx context.Context, appName strin } log.Ctx(ctx).Info().Msgf("Starting components in the application %s", appName) for _, environmentName := range environmentNames { - err := eh.StartEnvironment(ctx, appName, environmentName) + err := eh.ResetManuallyStoppedComponentsInEnvironment(ctx, appName, environmentName) if err != nil { return err } @@ -402,10 +422,13 @@ func (eh EnvironmentHandler) RestartApplication(ctx context.Context, appName str } func (eh EnvironmentHandler) getRadixCommonComponentUpdater(ctx context.Context, appName, envName, componentName string) (radixDeployCommonComponentUpdater, error) { - deploymentSummary, rd, err := eh.getRadixDeployment(ctx, appName, envName) + rd, err := kubequery.GetLatestRadixDeployment(ctx, eh.accounts.UserAccount.RadixClient, appName, envName) if err != nil { return nil, err } + if rd == nil { + return nil, http.ValidationError(v1.KindRadixDeployment, "no radix deployments found") + } baseUpdater := &baseComponentUpdater{ appName: appName, envName: envName, @@ -437,9 +460,12 @@ func (eh EnvironmentHandler) getRadixCommonComponentUpdater(ctx context.Context, baseUpdater.componentIndex = componentIndex baseUpdater.componentToPatch = componentToPatch - ra, _ := kubequery.GetRadixApplication(ctx, eh.accounts.UserAccount.RadixClient, appName) + ra, err := kubequery.GetRadixApplication(ctx, eh.accounts.UserAccount.RadixClient, appName) + if err != nil { + return nil, err + } baseUpdater.environmentConfig = utils.GetComponentEnvironmentConfig(ra, envName, componentName) - baseUpdater.componentState, err = getComponentStateFromSpec(ctx, eh.accounts.UserAccount.Client, appName, deploymentSummary, rd.Status, baseUpdater.environmentConfig, componentToPatch, hpas, scalers) + baseUpdater.componentState, err = eh.getComponentStateFromSpec(ctx, rd, componentToPatch, hpas, scalers) if err != nil { return nil, err } diff --git a/api/environments/models/environment_errors.go b/api/environments/models/environment_errors.go index 19bca05d..04bee012 100644 --- a/api/environments/models/environment_errors.go +++ b/api/environments/models/environment_errors.go @@ -32,9 +32,9 @@ func CannotStopComponent(appName, componentName, state string) error { return radixhttp.ValidationError("Radix Application Component", fmt.Sprintf("Component %s for app %s cannot be stopped when in %s state", componentName, appName, strings.ToLower(state))) } -// CannotStartComponent Component cannot be started -func CannotStartComponent(appName, componentName, state string) error { - return radixhttp.ValidationError("Radix Application Component", fmt.Sprintf("Component %s for app %s cannot be started when in %s state", componentName, appName, strings.ToLower(state))) +// CannotResetScaledComponent Component cannot be started +func CannotResetScaledComponent(appName, componentName string) error { + return radixhttp.ValidationError("Radix Application Component", fmt.Sprintf("Component %s for app %s cannot be reset when not manually scaled", componentName, appName)) } // CannotRestartComponent Component cannot be restarted diff --git a/api/environments/utils.go b/api/environments/utils.go deleted file mode 100644 index 7f13f81e..00000000 --- a/api/environments/utils.go +++ /dev/null @@ -1,24 +0,0 @@ -package environments - -import ( - "context" - - deploymentModels "github.com/equinor/radix-api/api/deployments/models" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" - operatorutils "github.com/equinor/radix-operator/pkg/apis/utils" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func (eh EnvironmentHandler) getRadixDeployment(ctx context.Context, appName, envName string) (*deploymentModels.DeploymentSummary, *v1.RadixDeployment, error) { - envNs := operatorutils.GetEnvironmentNamespace(appName, envName) - deploymentSummary, err := eh.deployHandler.GetLatestDeploymentForApplicationEnvironment(ctx, appName, envName) - if err != nil { - return nil, nil, err - } - - radixDeployment, err := eh.accounts.UserAccount.RadixClient.RadixV1().RadixDeployments(envNs).Get(ctx, deploymentSummary.Name, metav1.GetOptions{}) - if err != nil { - return nil, nil, err - } - return deploymentSummary, radixDeployment, nil -} diff --git a/api/kubequery/radixapplication_test.go b/api/kubequery/radixapplication_test.go index 2989ee09..321aea3a 100644 --- a/api/kubequery/radixapplication_test.go +++ b/api/kubequery/radixapplication_test.go @@ -23,7 +23,6 @@ func Test_GetRadixApplication(t *testing.T) { assert.Equal(t, &matched, actual) // Get non-existing RA (wrong namespace) - actual, err = GetRadixApplication(context.Background(), client, "app2") + _, err = GetRadixApplication(context.Background(), client, "app2") assert.True(t, errors.IsNotFound(err)) - assert.Nil(t, actual) } diff --git a/api/kubequery/radixdeployment.go b/api/kubequery/radixdeployment.go index b9d97ad6..f6b85c61 100644 --- a/api/kubequery/radixdeployment.go +++ b/api/kubequery/radixdeployment.go @@ -2,6 +2,7 @@ package kubequery import ( "context" + "sort" "github.com/equinor/radix-common/utils/slice" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" @@ -73,3 +74,32 @@ func GetRadixDeploymentByName(ctx context.Context, radixClient radixclient.Inter ns := operatorUtils.GetEnvironmentNamespace(appName, envName) return radixClient.RadixV1().RadixDeployments(ns).Get(ctx, deploymentName, metav1.GetOptions{}) } + +// GetLatestRadixDeployment returns the last active Radix Deployment found in environment, will be nil if not found +func GetLatestRadixDeployment(ctx context.Context, radixClient radixclient.Interface, appName, envName string) (*radixv1.RadixDeployment, error) { + ns := operatorUtils.GetEnvironmentNamespace(appName, envName) + rdList, err := radixClient.RadixV1().RadixDeployments(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + if len(rdList.Items) == 0 { + return nil, nil + } + + rds := sortRdsByActiveFromDesc(rdList.Items) + return &rds[0], nil +} + +func sortRdsByActiveFromDesc(rds []radixv1.RadixDeployment) []radixv1.RadixDeployment { + sort.Slice(rds, func(i, j int) bool { + if rds[j].Status.ActiveFrom.IsZero() { + return true + } + + if rds[i].Status.ActiveFrom.IsZero() { + return false + } + return rds[j].Status.ActiveFrom.Before(&rds[i].Status.ActiveFrom) + }) + return rds +} diff --git a/api/kubequery/radixjob_test.go b/api/kubequery/radixjob_test.go index 62a6d600..5ac3120e 100644 --- a/api/kubequery/radixjob_test.go +++ b/api/kubequery/radixjob_test.go @@ -37,7 +37,6 @@ func Test_GetRadixJob(t *testing.T) { assert.Equal(t, &matched, actual) // Get non-existing RJ - actual, err = GetRadixJob(context.Background(), client, "app2", "unmatched") + _, err = GetRadixJob(context.Background(), client, "app2", "unmatched") assert.True(t, errors.IsNotFound(err)) - assert.Nil(t, actual) } diff --git a/api/kubequery/radixregistration_test.go b/api/kubequery/radixregistration_test.go index 500e2682..5e94c4d5 100644 --- a/api/kubequery/radixregistration_test.go +++ b/api/kubequery/radixregistration_test.go @@ -22,7 +22,6 @@ func Test_GetRadixRegistration(t *testing.T) { assert.Equal(t, &matched, actual) // Get non-existing RR - actual, err = GetRadixRegistration(context.Background(), client, "anyapp") + _, err = GetRadixRegistration(context.Background(), client, "anyapp") assert.True(t, errors.IsNotFound(err)) - assert.Nil(t, actual) } diff --git a/api/models/auxiliary_resource.go b/api/models/auxiliary_resource.go index 19a3b08b..456412c2 100644 --- a/api/models/auxiliary_resource.go +++ b/api/models/auxiliary_resource.go @@ -10,30 +10,30 @@ import ( corev1 "k8s.io/api/core/v1" ) -func getAuxiliaryResources(appName string, component radixv1.RadixCommonDeployComponent, deploymentList []appsv1.Deployment, podList []corev1.Pod, eventWarnings map[string]string) deploymentModels.AuxiliaryResource { +func getAuxiliaryResources(rd *radixv1.RadixDeployment, component radixv1.RadixCommonDeployComponent, deploymentList []appsv1.Deployment, podList []corev1.Pod, eventWarnings map[string]string) deploymentModels.AuxiliaryResource { var auxResource deploymentModels.AuxiliaryResource if auth := component.GetAuthentication(); component.IsPublic() && auth != nil && auth.OAuth2 != nil { - auxResource.OAuth2 = getOAuth2AuxiliaryResource(appName, component.GetName(), deploymentList, podList, eventWarnings) + auxResource.OAuth2 = getOAuth2AuxiliaryResource(rd, component, deploymentList, podList, eventWarnings) } return auxResource } -func getOAuth2AuxiliaryResource(appName, componentName string, deploymentList []appsv1.Deployment, podList []corev1.Pod, eventWarnings map[string]string) *deploymentModels.OAuth2AuxiliaryResource { +func getOAuth2AuxiliaryResource(rd *radixv1.RadixDeployment, component radixv1.RadixCommonDeployComponent, deploymentList []appsv1.Deployment, podList []corev1.Pod, eventWarnings map[string]string) *deploymentModels.OAuth2AuxiliaryResource { return &deploymentModels.OAuth2AuxiliaryResource{ - Deployment: getAuxiliaryResourceDeployment(appName, componentName, operatordefaults.OAuthProxyAuxiliaryComponentType, deploymentList, podList, eventWarnings), + Deployment: getAuxiliaryResourceDeployment(rd, component, operatordefaults.OAuthProxyAuxiliaryComponentType, deploymentList, podList, eventWarnings), } } -func getAuxiliaryResourceDeployment(appName, componentName, auxType string, deploymentList []appsv1.Deployment, podList []corev1.Pod, eventWarnings map[string]string) deploymentModels.AuxiliaryResourceDeployment { +func getAuxiliaryResourceDeployment(rd *radixv1.RadixDeployment, component radixv1.RadixCommonDeployComponent, auxType string, deploymentList []appsv1.Deployment, podList []corev1.Pod, eventWarnings map[string]string) deploymentModels.AuxiliaryResourceDeployment { var auxResourceDeployment deploymentModels.AuxiliaryResourceDeployment - auxDeployments := slice.FindAll(deploymentList, predicate.IsDeploymentForAuxComponent(appName, componentName, auxType)) + auxDeployments := slice.FindAll(deploymentList, predicate.IsDeploymentForAuxComponent(rd.Spec.AppName, component.GetName(), auxType)) if len(auxDeployments) == 0 { auxResourceDeployment.Status = deploymentModels.ComponentReconciling.String() return auxResourceDeployment } deployment := auxDeployments[0] - auxPods := slice.FindAll(podList, predicate.IsPodForAuxComponent(appName, componentName, auxType)) + auxPods := slice.FindAll(podList, predicate.IsPodForAuxComponent(rd.Spec.AppName, component.GetName(), auxType)) auxResourceDeployment.ReplicaList = BuildReplicaSummaryList(auxPods, eventWarnings) - auxResourceDeployment.Status = deploymentModels.ComponentStatusFromDeployment(&deployment).String() + auxResourceDeployment.Status = deploymentModels.ComponentStatusFromDeployment(component, &deployment, rd).String() return auxResourceDeployment } diff --git a/api/models/component.go b/api/models/component.go index 6f2ef9dd..23113a6f 100644 --- a/api/models/component.go +++ b/api/models/component.go @@ -9,19 +9,14 @@ import ( cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" cmmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" deploymentModels "github.com/equinor/radix-api/api/deployments/models" - "github.com/equinor/radix-api/api/utils" "github.com/equinor/radix-api/api/utils/event" "github.com/equinor/radix-api/api/utils/predicate" "github.com/equinor/radix-api/api/utils/tlsvalidation" - commonutils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/slice" - operatordefaults "github.com/equinor/radix-operator/pkg/apis/defaults" - operatordeployment "github.com/equinor/radix-operator/pkg/apis/deployment" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" operatorutils "github.com/equinor/radix-operator/pkg/apis/utils" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" "github.com/kedacore/keda/v2/apis/keda/v1alpha1" - "github.com/rs/zerolog/log" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" @@ -65,12 +60,17 @@ func buildComponent( WithExternalDNS(getComponentExternalDNS(ra.Name, radixComponent, secretList, certs, certRequests, tlsValidator)) componentPods := slice.FindAll(podList, predicate.IsPodForComponent(ra.Name, radixComponent.GetName())) + var kd *appsv1.Deployment + if depl, ok := slice.FindFirst(deploymentList, predicate.IsDeploymentForComponent(ra.Name, radixComponent.GetName())); ok { + kd = &depl + } + if rd.Status.ActiveTo.IsZero() { builder.WithPodNames(slice.Map(componentPods, func(pod corev1.Pod) string { return pod.Name })) builder.WithRadixEnvironmentVariables(getRadixEnvironmentVariables(componentPods)) builder.WithReplicaSummaryList(BuildReplicaSummaryList(componentPods, lastEventWarnings)) - builder.WithStatus(getComponentStatus(radixComponent, ra, rd, componentPods)) - builder.WithAuxiliaryResource(getAuxiliaryResources(ra.Name, radixComponent, deploymentList, podList, lastEventWarnings)) + builder.WithStatus(deploymentModels.ComponentStatusFromDeployment(radixComponent, kd, rd)) + builder.WithAuxiliaryResource(getAuxiliaryResources(rd, radixComponent, deploymentList, podList, lastEventWarnings)) } // TODO: Use radixComponent.GetType() instead? @@ -235,118 +235,6 @@ func certificateRequestConditionReady(condition cmv1.CertificateRequestCondition return condition.Type == cmv1.CertificateRequestConditionReady } -func getComponentStatus(component radixv1.RadixCommonDeployComponent, ra *radixv1.RadixApplication, rd *radixv1.RadixDeployment, pods []corev1.Pod) deploymentModels.ComponentStatus { - environmentConfig := utils.GetComponentEnvironmentConfig(ra, rd.Spec.Environment, component.GetName()) - if component.GetType() == radixv1.RadixComponentTypeComponent { - if runningReplicaDiffersFromConfig(environmentConfig, pods) && - !runningReplicaDiffersFromSpec(component, pods) && - len(pods) == 0 { - return deploymentModels.StoppedComponent - } - if runningReplicaDiffersFromSpec(component, pods) { - return deploymentModels.ComponentReconciling - } - } else if component.GetType() == radixv1.RadixComponentTypeJob { - if len(pods) == 0 { - return deploymentModels.StoppedComponent - } - } - if runningReplicaIsOutdated(component, pods) { - return deploymentModels.ComponentOutdated - } - restarted := component.GetEnvironmentVariables()[operatordefaults.RadixRestartEnvironmentVariable] - if strings.EqualFold(restarted, "") { - return deploymentModels.ConsistentComponent - } - restartedTime, err := commonutils.ParseTimestamp(restarted) - if err != nil { - // TODO: How should we handle invalid value for restarted time? - - log.Logger.Warn().Err(err).Msgf("unable to parse restarted time %v", restarted) - return deploymentModels.ConsistentComponent - } - reconciledTime := rd.Status.Reconciled - if reconciledTime.IsZero() || restartedTime.After(reconciledTime.Time) { - return deploymentModels.ComponentRestarting - } - return deploymentModels.ConsistentComponent -} - -func runningReplicaDiffersFromConfig(environmentConfig radixv1.RadixCommonEnvironmentConfig, actualPods []corev1.Pod) bool { - actualPodsLength := len(actualPods) - if commonutils.IsNil(environmentConfig) { - return actualPodsLength != operatordeployment.DefaultReplicas - } - // No HPA config - if environmentConfig.GetHorizontalScaling() == nil { - if environmentConfig.GetReplicas() != nil { - return actualPodsLength != *environmentConfig.GetReplicas() - } - return actualPodsLength != operatordeployment.DefaultReplicas - } - // With HPA config - if environmentConfig.GetReplicas() != nil && *environmentConfig.GetReplicas() == 0 { - return actualPodsLength != *environmentConfig.GetReplicas() - } - if environmentConfig.GetHorizontalScaling().MinReplicas != nil { - return actualPodsLength < int(*environmentConfig.GetHorizontalScaling().MinReplicas) || - actualPodsLength > int(environmentConfig.GetHorizontalScaling().MaxReplicas) - } - return actualPodsLength < operatordeployment.DefaultReplicas || - actualPodsLength > int(environmentConfig.GetHorizontalScaling().MaxReplicas) -} - -func runningReplicaDiffersFromSpec(component radixv1.RadixCommonDeployComponent, actualPods []corev1.Pod) bool { - actualPodsLength := len(actualPods) - // No HPA config - if component.GetHorizontalScaling() == nil { - if component.GetReplicas() != nil { - return actualPodsLength != *component.GetReplicas() - } - return actualPodsLength != operatordeployment.DefaultReplicas - } - // With HPA config - if component.GetReplicas() != nil && *component.GetReplicas() == 0 { - return actualPodsLength != *component.GetReplicas() - } - if component.GetHorizontalScaling().MinReplicas != nil { - return actualPodsLength < int(*component.GetHorizontalScaling().MinReplicas) || - actualPodsLength > int(component.GetHorizontalScaling().MaxReplicas) - } - return actualPodsLength < operatordeployment.DefaultReplicas || - actualPodsLength > int(component.GetHorizontalScaling().MaxReplicas) -} - -func runningReplicaIsOutdated(component radixv1.RadixCommonDeployComponent, actualPods []corev1.Pod) bool { - switch component.GetType() { - case radixv1.RadixComponentTypeComponent: - return runningComponentReplicaIsOutdated(component, actualPods) - case radixv1.RadixComponentTypeJob: - return false - default: - return false - } -} - -func runningComponentReplicaIsOutdated(component radixv1.RadixCommonDeployComponent, actualPods []corev1.Pod) bool { - // Check if running component's image is not the same as active deployment image tag and that active rd image is equal to 'starting' component image tag - componentIsInconsistent := false - for _, pod := range actualPods { - if pod.DeletionTimestamp != nil { - // Pod is in termination phase - continue - } - for _, container := range pod.Spec.Containers { - if container.Image != component.GetImage() { - // Container is running an outdated image - componentIsInconsistent = true - } - } - } - - return componentIsInconsistent -} - func getRadixEnvironmentVariables(pods []corev1.Pod) map[string]string { radixEnvironmentVariables := make(map[string]string) diff --git a/api/utils/owner/verify_generation.go b/api/utils/owner/verify_generation.go new file mode 100644 index 00000000..a7ef364c --- /dev/null +++ b/api/utils/owner/verify_generation.go @@ -0,0 +1,22 @@ +package owner + +import ( + "strconv" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VerifyCorrectObjectGeneration Returns true if owner is controller of object and annotation matches owner generation +func VerifyCorrectObjectGeneration(owner metav1.Object, object metav1.Object, annotation string) bool { + controller := metav1.GetControllerOf(object) + if controller == nil { + return false + } + + if controller.UID != owner.GetUID() { + return false + } + + observedGeneration, ok := object.GetAnnotations()[annotation] + return ok && observedGeneration == strconv.Itoa(int(owner.GetGeneration())) +} diff --git a/api/utils/predicate/kubernetes.go b/api/utils/predicate/kubernetes.go index cc3bbb3c..b0654c9e 100644 --- a/api/utils/predicate/kubernetes.go +++ b/api/utils/predicate/kubernetes.go @@ -20,12 +20,19 @@ func IsAppAliasIngress(ingress networkingv1.Ingress) bool { } func IsPodForComponent(appName, componentName string) func(corev1.Pod) bool { - selector := labels.SelectorFromSet(radixlabels.Merge(radixlabels.ForApplicationName(appName), radixlabels.ForComponentName(componentName))) + selector := labelselector.ForComponent(appName, componentName).AsSelector() return func(pod corev1.Pod) bool { return selector.Matches(labels.Set(pod.Labels)) } } +func IsDeploymentForComponent(appName, componentName string) func(appsv1.Deployment) bool { + selector := labelselector.ForComponent(appName, componentName).AsSelector() + return func(deployment appsv1.Deployment) bool { + return selector.Matches(labels.Set(deployment.Labels)) + } +} + func IsPodForAuxComponent(appName, componentName, auxType string) func(corev1.Pod) bool { selector := labelselector.ForAuxiliaryResource(appName, componentName, auxType).AsSelector() return func(pod corev1.Pod) bool { diff --git a/go.mod b/go.mod index 07e35409..563f096e 100644 --- a/go.mod +++ b/go.mod @@ -6,39 +6,39 @@ toolchain go1.22.5 require ( github.com/cert-manager/cert-manager v1.15.0 - github.com/equinor/radix-common v1.9.3 + github.com/equinor/radix-common v1.9.4 github.com/equinor/radix-job-scheduler v1.11.0 - github.com/equinor/radix-operator v1.58.1 + github.com/equinor/radix-operator v1.58.3 github.com/evanphx/json-patch/v5 v5.9.0 github.com/felixge/httpsnoop v1.0.4 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.1 - github.com/kedacore/keda/v2 v2.13.1 + github.com/kedacore/keda/v2 v2.15.1 github.com/marstr/guid v1.1.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.2 - github.com/prometheus/client_golang v1.19.1 + github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.0 + github.com/prometheus/client_golang v1.20.2 github.com/rs/cors v1.11.0 github.com/rs/xid v1.5.0 github.com/rs/zerolog v1.33.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.18.2 + github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/tektoncd/pipeline v0.55.0 github.com/urfave/negroni/v3 v3.1.0 golang.org/x/sync v0.8.0 - k8s.io/api v0.30.2 - k8s.io/apimachinery v0.30.2 - k8s.io/client-go v0.30.2 - knative.dev/pkg v0.0.0-20240116073220-b488e7be5902 - sigs.k8s.io/secrets-store-csi-driver v1.4.0 + k8s.io/api v0.31.0 + k8s.io/apimachinery v0.31.0 + k8s.io/client-go v0.31.0 + knative.dev/pkg v0.0.0-20240805063731-c88d5dad9653 + sigs.k8s.io/secrets-store-csi-driver v1.4.5 ) require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect - dario.cat/mergo v1.0.0 // indirect + dario.cat/mergo v1.0.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blendle/zapdriver v1.3.1 // indirect @@ -47,9 +47,9 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/elnormous/contenttype v1.0.4 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect - github.com/evanphx/json-patch v5.9.0+incompatible // indirect - github.com/expr-lang/expr v1.15.8 // indirect + github.com/expr-lang/expr v1.16.9 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -59,7 +59,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/cel-go v0.18.2 // indirect + github.com/google/cel-go v0.20.1 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-containerregistry v0.16.1 // indirect @@ -72,6 +72,7 @@ require ( github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -83,10 +84,10 @@ require ( github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.2 // indirect + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.53.0 // indirect - github.com/prometheus/procfs v0.15.0 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -96,6 +97,7 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect @@ -108,20 +110,21 @@ require ( golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.6.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/api v0.181.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect - google.golang.org/grpc v1.64.1 // indirect + google.golang.org/api v0.190.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect + google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.30.2 // indirect + k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - sigs.k8s.io/controller-runtime v0.18.5 // indirect + sigs.k8s.io/controller-runtime v0.19.0 // indirect sigs.k8s.io/gateway-api v1.1.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 0e89b5a7..a34541a0 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d h contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -67,8 +67,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudevents/sdk-go/v2 v2.14.0 h1:Nrob4FwVgi5L4tV9lhjzZcjYqFVyJzsA56CwPaPfv6s= -github.com/cloudevents/sdk-go/v2 v2.14.0/go.mod h1:xDmKfzNjM8gBvjaF8ijFjM1VYOVUEeUfapHMUX1T5To= +github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= +github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -83,24 +83,26 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/equinor/radix-common v1.9.3 h1:dLKFzYy8/XyEG9Zygi0rMWIYGCddai/ILwUqjBiGYxQ= -github.com/equinor/radix-common v1.9.3/go.mod h1:+g0Wj0D40zz29DjNkYKVmCVeYy4OsFWKI7Qi9rA6kpY= +github.com/equinor/radix-common v1.9.4 h1:ErSnB2tqlRwaQuQdaA0qzsReDtHDgubcvqRO098ncEw= +github.com/equinor/radix-common v1.9.4/go.mod h1:+g0Wj0D40zz29DjNkYKVmCVeYy4OsFWKI7Qi9rA6kpY= github.com/equinor/radix-job-scheduler v1.11.0 h1:8wCmXOVl/1cto8q2WJQEE06Cw68/QmfoifYVR49vzkY= github.com/equinor/radix-job-scheduler v1.11.0/go.mod h1:yPXn3kDcMY0Z3kBkosjuefsdY1x2g0NlBeybMmHz5hc= -github.com/equinor/radix-operator v1.58.1 h1:Wb/UOP1m4wUdWCL/gynPcnf6axz01Z24fBvK2DRL5m0= -github.com/equinor/radix-operator v1.58.1/go.mod h1:zCdAiP/wxyvlUO4qGoJuLW3O+ZSt9kTyHMnjmsR3fCU= +github.com/equinor/radix-operator v1.58.3 h1:F4YhNkQ4uRONP125OTfG8hdy9PiyKlOWVO8/p2NIi70= +github.com/equinor/radix-operator v1.58.3/go.mod h1:DTPXOxU3uHPvji7qBGSK1b03iXROpX3l94kYjcOHkPM= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/expr-lang/expr v1.15.8 h1:FL8+d3rSSP4tmK9o+vKfSMqqpGL8n15pEPiHcnBpxoI= -github.com/expr-lang/expr v1.15.8/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= +github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI= +github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -127,7 +129,6 @@ github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDsl github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -171,8 +172,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/cel-go v0.18.2 h1:L0B6sNBSVmt0OyECi8v6VOS74KOc9W/tLiWKfZABvf4= -github.com/google/cel-go v0.18.2/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= +github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= +github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -242,12 +243,14 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kedacore/keda/v2 v2.13.1 h1:8y4Mp4iWyiqHoedVT3q2g5xvWDe494TRH3sUCZPpn/o= -github.com/kedacore/keda/v2 v2.13.1/go.mod h1:AZTRgxWpK5/6pq+DqJ15y3Bl/C8sl9C7tUVF4phzGDQ= +github.com/kedacore/keda/v2 v2.15.1 h1:Kb3woYuCeCPICH037vTIcUopXgOYpdP2qa+CmHgV3SE= +github.com/kedacore/keda/v2 v2.15.1/go.mod h1:2umVEoNgklKt0+q+7BEEbrSgxqh+KPjyh6vnKXt3sls= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -258,6 +261,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -300,10 +305,10 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.2 h1:6UsAv+jAevuGO2yZFU/BukV4o9NKnFMOuoouSA4G0ns= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.2/go.mod h1:XYrdZw5dW12Cjkt4ndbeNZZTBp4UCHtW0ccR9+sTtPU= -github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.2 h1:71GOmhZFA2/17maXqCcuJEzpJDyqPty8SpEOGZWyVec= -github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.2/go.mod h1:Sv6XsfGGkR9gKnhP92F5dNXEpsSePn0W+7JwYP0NVkc= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.0 h1:tRwEFYFg+To2TGnibGl8dHBCh8Z/BVNKnXj2O5Za/2M= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.0/go.mod h1:Rd8YnCqz+2FYsiGmE2DMlaLjQRB4v2jFNnzCt9YY4IM= +github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.0 h1:bJhRd6R4kaYBZpH7cBrzbJpEKJjHx8cbVW1n3dxYnag= +github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.0/go.mod h1:Nu6G9XLApnqXqunMwMYulcHlaxRwoveH4p4WnZsBHD8= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -311,8 +316,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= +github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -325,16 +330,16 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= -github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= -github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -364,8 +369,8 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= -github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -390,6 +395,8 @@ github.com/tektoncd/pipeline v0.55.0 h1:RUfqSC/J1dMrdfu1ThJreHojwGXcWc8P131el/c+ github.com/tektoncd/pipeline v0.55.0/go.mod h1:fFbFAhyNwsPQpitrwhi+Wp0Xse2EkIE1LtGKC08rVqo= github.com/urfave/negroni/v3 v3.1.0 h1:lzmuxGSpnJCT/ujgIAjkU3+LW3NX8alCglO/L6KjIGQ= github.com/urfave/negroni/v3 v3.1.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -649,8 +656,8 @@ google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.181.0 h1:rPdjwnWgiPPOJx3IcSAQ2III5aX5tCer6wMpa/xmZi4= -google.golang.org/api v0.181.0/go.mod h1:MnQ+M0CFsfUwA5beZ+g/vCBCPXvtmZwRz2qzZk8ih1k= +google.golang.org/api v0.190.0 h1:ASM+IhLY1zljNdLu19W1jTmU6A+gMk6M46Wlur61s+Q= +google.golang.org/api v0.190.0/go.mod h1:QIr6I9iedBLnfqoD6L6Vze1UvS5Hzj5r2aUBOaZnLHo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -688,10 +695,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291 h1:4HZJ3Xv1cmrJ+0aFo304Zn79ur1HMxptAE7aCPNLSqc= -google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f h1:b1Ln/PG8orm0SsBbHZWke8dDp2lrCD4jSmfglFpTZbk= +google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf h1:liao9UHurZLtiEwBgT9LMOnKYsHze6eA6w1KQCMVN2Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -705,8 +712,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -730,6 +737,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= @@ -753,33 +762,33 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= -k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= -k8s.io/apiextensions-apiserver v0.30.2 h1:l7Eue2t6QiLHErfn2vwK4KgF4NeDgjQkCXtEbOocKIE= -k8s.io/apiextensions-apiserver v0.30.2/go.mod h1:lsJFLYyK40iguuinsb3nt+Sj6CmodSI4ACDLep1rgjw= -k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= -k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= -k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= +k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= +k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= +k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= +k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= +k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= +k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 h1:1Wof1cGQgA5pqgo8MxKPtf+qN6Sh/0JzznmeGPm1HnE= k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8/go.mod h1:Os6V6dZwLNii3vxFpxcNaTmH8LJJBkOTg1N0tOA0fvA= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -knative.dev/pkg v0.0.0-20240116073220-b488e7be5902 h1:H6+JJN23fhwYWCHY1339sY6uhIyoUwDy1a8dN233fdk= -knative.dev/pkg v0.0.0-20240116073220-b488e7be5902/go.mod h1:NYk8mMYoLkO7CQWnNkti4YGGnvLxN6MIDbUvtgeo0C0= +knative.dev/pkg v0.0.0-20240805063731-c88d5dad9653 h1:VHUW124ZpkDn4EnIzMuGWvGuJte3ISIoHMmEw2kx0zU= +knative.dev/pkg v0.0.0-20240805063731-c88d5dad9653/go.mod h1:H+5rS2GEWpAZzrmQoXOEVq/1M77LLMhR7+4jZBMOQ24= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.18.5 h1:nTHio/W+Q4aBlQMgbnC5hZb4IjIidyrizMai9P6n4Rk= -sigs.k8s.io/controller-runtime v0.18.5/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= +sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= +sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM= sigs.k8s.io/gateway-api v1.1.0/go.mod h1:ZH4lHrL2sDi0FHZ9jjneb8kKnGzFWyrTya35sWUTrRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/secrets-store-csi-driver v1.4.0 h1:R9JVcKOs11fEuiOLlH1BWMeyb6WYzvElRVkq1BWJkr4= -sigs.k8s.io/secrets-store-csi-driver v1.4.0/go.mod h1:RjFTqJzIV6/howouY0llU0iMbldSEt3nc2MGFOL6gko= +sigs.k8s.io/secrets-store-csi-driver v1.4.5 h1:ta4aiNbl2EAWKrn3hFnN4Nll8mC6cgBP9+O4XdzorHM= +sigs.k8s.io/secrets-store-csi-driver v1.4.5/go.mod h1:0/wMVOv8qLx7YNVMGU+Sh7S4D6TH6GhyEpouo28OTUU= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/swaggerui/html/swagger.json b/swaggerui/html/swagger.json index 52325b6a..156c8ccf 100644 --- a/swaggerui/html/swagger.json +++ b/swaggerui/html/swagger.json @@ -2070,6 +2070,61 @@ } } }, + "/applications/{appName}/environments/{envName}/components/{componentName}/reset-scale": { + "post": { + "tags": [ + "component" + ], + "summary": "Reset manually scaled component and resumes normal operation", + "operationId": "resetScaledComponent", + "parameters": [ + { + "type": "string", + "description": "Name of application", + "name": "appName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Name of environment", + "name": "envName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Name of component", + "name": "componentName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Works only with custom setup of cluster. Allow impersonation of test users (Required if Impersonate-Group is set)", + "name": "Impersonate-User", + "in": "header" + }, + { + "type": "string", + "description": "Works only with custom setup of cluster. Allow impersonation of a comma-seperated list of test groups (Required if Impersonate-User is set)", + "name": "Impersonate-Group", + "in": "header" + } + ], + "responses": { + "200": { + "description": "Component started ok" + }, + "401": { + "description": "Unauthorized" + }, + "404": { + "description": "Not found" + } + } + } + }, "/applications/{appName}/environments/{envName}/components/{componentName}/restart": { "post": { "tags": [ @@ -2367,8 +2422,9 @@ "tags": [ "component" ], - "summary": "Start component", + "summary": "Deprecated Start component. Use reset-scale instead. This does the same thing, but naming is wrong. This endpoint will be removed after 1. september 2025.", "operationId": "startComponent", + "deprecated": true, "parameters": [ { "type": "string", @@ -3599,6 +3655,54 @@ } } }, + "/applications/{appName}/environments/{envName}/reset-scale": { + "post": { + "tags": [ + "environment" + ], + "summary": "Reset all manually scaled component and resumes normal operation in environment", + "operationId": "resetManuallyScaledComponentsInEnvironment", + "parameters": [ + { + "type": "string", + "description": "Name of application", + "name": "appName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Name of environment", + "name": "envName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Works only with custom setup of cluster. Allow impersonation of test users (Required if Impersonate-Group is set)", + "name": "Impersonate-User", + "in": "header" + }, + { + "type": "string", + "description": "Works only with custom setup of cluster. Allow impersonation of a comma-seperated list of test groups (Required if Impersonate-User is set)", + "name": "Impersonate-Group", + "in": "header" + } + ], + "responses": { + "200": { + "description": "Environment started ok" + }, + "401": { + "description": "Unauthorized" + }, + "404": { + "description": "Not found" + } + } + } + }, "/applications/{appName}/environments/{envName}/restart": { "post": { "tags": [ @@ -3655,8 +3759,9 @@ "tags": [ "environment" ], - "summary": "Start all components in the environment", + "summary": "Deprecated. Use reset-scale instead that does the same thing, but with better naming. This method will be removed after 1. september 2025.", "operationId": "startEnvironment", + "deprecated": true, "parameters": [ { "type": "string", @@ -4875,6 +4980,47 @@ } } }, + "/applications/{appName}/reset-scale": { + "post": { + "tags": [ + "application" + ], + "summary": "Resets and resumes normal opperation for all manually stopped components in all environments of the application", + "operationId": "resetManuallyScaledComponentsInApplication", + "parameters": [ + { + "type": "string", + "description": "Name of application", + "name": "appName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Works only with custom setup of cluster. Allow impersonation of test users (Required if Impersonate-Group is set)", + "name": "Impersonate-User", + "in": "header" + }, + { + "type": "string", + "description": "Works only with custom setup of cluster. Allow impersonation of a comma-seperated list of test groups (Required if Impersonate-User is set)", + "name": "Impersonate-Group", + "in": "header" + } + ], + "responses": { + "200": { + "description": "Application started ok" + }, + "401": { + "description": "Unauthorized" + }, + "404": { + "description": "Not found" + } + } + } + }, "/applications/{appName}/restart": { "post": { "tags": [ @@ -4921,8 +5067,9 @@ "tags": [ "application" ], - "summary": "Start all components in all environments of the application", + "summary": "Deprecated. Use reset scale that does the same thing instead. This will be removed after 1. september 2025.", "operationId": "startApplication", + "deprecated": true, "parameters": [ { "type": "string", @@ -5634,6 +5781,14 @@ "server-78fc8857c4-asfa2" ] }, + "replicasOverride": { + "description": "Set if manual control of replicas is in place. Not set means automatic control, 0 means stopped and \u003e= 1 is manually scaled.", + "type": "integer", + "format": "int64", + "x-go-name": "ReplicasOverride", + "x-nullable": true, + "example": 5 + }, "resources": { "$ref": "#/definitions/ResourceRequirements" }, From 0732e20ca7f2fb06265c7035fd842968d5f52f56 Mon Sep 17 00:00:00 2001 From: Richard Hagen Date: Thu, 12 Sep 2024 08:39:48 +0200 Subject: [PATCH 4/4] Allow scaling reconciling component, block scaling jobs (#672) --- api/environments/component_handler.go | 8 ++++---- api/environments/environment_handler.go | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/api/environments/component_handler.go b/api/environments/component_handler.go index 58e7f9a9..a2a8e56c 100644 --- a/api/environments/component_handler.go +++ b/api/environments/component_handler.go @@ -36,15 +36,15 @@ func (eh EnvironmentHandler) ScaleComponent(ctx context.Context, appName, envNam if replicas > maxScaleReplicas { return environmentModels.CannotScaleComponentToMoreThanMaxReplicas(appName, envName, componentName, maxScaleReplicas) } - log.Ctx(ctx).Info().Msgf("Scaling component %s, %s to %d replicas", componentName, appName, replicas) updater, err := eh.getRadixCommonComponentUpdater(ctx, appName, envName, componentName) if err != nil { return err } - componentStatus := updater.getComponentStatus() - if !radixutils.ContainsString(validaStatusesToScaleComponent, componentStatus) { - return environmentModels.CannotScaleComponent(appName, envName, componentName, componentStatus) + if updater.getComponentToPatch().GetType() == v1.RadixComponentTypeJob { + return environmentModels.JobComponentCanOnlyBeRestarted() } + + log.Ctx(ctx).Info().Msgf("Scaling component %s, %s to %d replicas", componentName, appName, replicas) return eh.patchRadixDeploymentWithReplicas(ctx, updater, &replicas) } diff --git a/api/environments/environment_handler.go b/api/environments/environment_handler.go index 0665ed7e..696e8726 100644 --- a/api/environments/environment_handler.go +++ b/api/environments/environment_handler.go @@ -89,14 +89,10 @@ type EnvironmentHandler struct { ComponentStatuser deploymentModels.ComponentStatuserFunc } -var validaStatusesToScaleComponent []string - // Init Constructor. // Use the WithAccounts configuration function to configure a 'ready to use' EnvironmentHandler. // EnvironmentHandlerOptions are processed in the sequence they are passed to this function. func Init(opts ...EnvironmentHandlerOptions) EnvironmentHandler { - validaStatusesToScaleComponent = []string{deploymentModels.ConsistentComponent.String(), deploymentModels.StoppedComponent.String()} - eh := EnvironmentHandler{ ComponentStatuser: deploymentModels.ComponentStatusFromDeployment, }