Skip to content

Commit

Permalink
fix: CDStageDeploy stuck in the running status (#131)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zmotso @SergK
zmotso authored and SergK committed Sep 23, 2024
1 parent e6e1e03 commit 65ead20
Show file tree
Hide file tree
Showing 6 changed files with 194 additions and 30 deletions.
9 changes: 9 additions & 0 deletions api/v1/labels.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package v1

const (
// CdPipelineLabel is a label that is used to store the name of the CD pipeline in the related resources.
CdPipelineLabel = "app.edp.epam.com/cdpipeline"

// CdStageLabel is a label that is used to store the name of the CD stage in the related resources.
CdStageLabel = "app.edp.epam.com/cdstage"
)
49 changes: 47 additions & 2 deletions controllers/cdstagedeploy/chain/resolve_status.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,23 @@ func (r *ResolveStatus) ServeRequest(ctx context.Context, stageDeploy *codebaseA
if allPipelineRunsCompleted(pipelineRun.Items) {
log.Info("All PipelineRuns have been completed.")

hasRunning, err := r.hasRunningCDStageDeploys(
ctx,
stageDeploy.Name,
stageDeploy.GetStageCRName(),
stageDeploy.Spec.Pipeline,
stageDeploy.Namespace,
)
if err != nil {
return fmt.Errorf("failed to check running CDStageDeploys: %w", err)
}

if hasRunning {
log.Info("Some CDStageDeploys are still running.")

return nil
}

stageDeploy.Status.Status = codebaseApi.CDStageDeployStatusPending

return nil
Expand Down Expand Up @@ -92,8 +109,8 @@ func (r *ResolveStatus) getRunningPipelines(ctx context.Context, stageDeploy *co
pipelineRun,
client.InNamespace(stageDeploy.Namespace),
client.MatchingLabels{
"app.edp.epam.com/cdpipeline": stageDeploy.Spec.Pipeline,
"app.edp.epam.com/cdstage": stageDeploy.GetStageCRName(),
codebaseApi.CdPipelineLabel: stageDeploy.Spec.Pipeline,
codebaseApi.CdStageLabel: stageDeploy.GetStageCRName(),
},
client.Limit(maxPipelineRuns),
); err != nil {
Expand All @@ -107,6 +124,34 @@ func (r *ResolveStatus) getRunningPipelines(ctx context.Context, stageDeploy *co
return pipelineRun, nil
}

func (r *ResolveStatus) hasRunningCDStageDeploys(
ctx context.Context,
currentCDStageDeployName, stage, pipeline, namespace string,
) (bool, error) {
cdStageDeploy := &codebaseApi.CDStageDeployList{}

if err := r.client.List(
ctx,
cdStageDeploy,
client.InNamespace(namespace),
client.MatchingLabels{
codebaseApi.CdPipelineLabel: pipeline,
codebaseApi.CdStageLabel: stage,
},
); err != nil {
return false, fmt.Errorf("failed to list CDStageDeploys: %w", err)
}

for i := range cdStageDeploy.Items {
if cdStageDeploy.Items[i].Name != currentCDStageDeployName &&
cdStageDeploy.Items[i].Status.Status == codebaseApi.CDStageDeployStatusRunning {
return true, nil
}
}

return false, nil
}

func allPipelineRunsCompleted(pipelineRuns []tektonpipelineApi.PipelineRun) bool {
for i := range pipelineRuns {
if pipelineRuns[i].Status.CompletionTime == nil {
Expand Down
120 changes: 114 additions & 6 deletions controllers/cdstagedeploy/chain/resolve_status_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestResolveStatus_ServeRequest(t *testing.T) {
wantStatus string
}{
{
name: "should skip failed CDStageDeploy",
name: "failed CDStageDeploy should be pending to retry",
stageDeploy: &codebaseApi.CDStageDeploy{
Status: codebaseApi.CDStageDeployStatus{
Status: codebaseApi.CDStageDeployStatusFailed,
Expand All @@ -44,7 +44,7 @@ func TestResolveStatus_ServeRequest(t *testing.T) {
wantStatus: codebaseApi.CDStageDeployStatusFailed,
},
{
name: "should skip pending CDStageDeploy",
name: "pending CDStageDeploy should be pending if no pipeline runs",
stageDeploy: &codebaseApi.CDStageDeploy{
Status: codebaseApi.CDStageDeployStatus{
Status: codebaseApi.CDStageDeployStatusPending,
Expand All @@ -57,7 +57,35 @@ func TestResolveStatus_ServeRequest(t *testing.T) {
wantStatus: codebaseApi.CDStageDeployStatusPending,
},
{
name: "have running pipeline runs",
name: "pending CDStageDeploy should be in queue if not all pipeline runs completed",
stageDeploy: &codebaseApi.CDStageDeploy{
Spec: codebaseApi.CDStageDeploySpec{
Pipeline: "app1",
Stage: "dev",
},
Status: codebaseApi.CDStageDeployStatus{
Status: codebaseApi.CDStageDeployStatusPending,
},
},
k8sClient: func(t *testing.T) client.Client {
piperun := &tektonpipelineApi.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Labels: map[string]string{
codebaseApi.CdPipelineLabel: "app1",
codebaseApi.CdStageLabel: "app1-dev",
},
},
}

return fake.NewClientBuilder().WithScheme(scheme).WithObjects(piperun).Build()
},
wantErr: require.NoError,
wantStatus: codebaseApi.CDStageDeployStatusInQueue,
},
{
name: "running CDStageDeploy should be running if pipeline run is running",
stageDeploy: &codebaseApi.CDStageDeploy{
Spec: codebaseApi.CDStageDeploySpec{
Pipeline: "app1",
Expand All @@ -73,8 +101,8 @@ func TestResolveStatus_ServeRequest(t *testing.T) {
Name: "test",
Namespace: "default",
Labels: map[string]string{
"app.edp.epam.com/cdpipeline": "app1",
"app.edp.epam.com/cdstage": "app1-dev",
codebaseApi.CdPipelineLabel: "app1",
codebaseApi.CdStageLabel: "app1-dev",
},
},
}
Expand All @@ -85,7 +113,7 @@ func TestResolveStatus_ServeRequest(t *testing.T) {
wantStatus: codebaseApi.CDStageDeployStatusRunning,
},
{
name: "all pipeline runs completed",
name: "running CDStageDeploy should be completed after all pipeline runs completed",
stageDeploy: &codebaseApi.CDStageDeploy{
Spec: codebaseApi.CDStageDeploySpec{
Pipeline: "app1",
Expand All @@ -101,6 +129,86 @@ func TestResolveStatus_ServeRequest(t *testing.T) {
wantErr: require.NoError,
wantStatus: codebaseApi.CDStageDeployStatusCompleted,
},
{
name: "queued CDStageDeploy should be pending after all pipeline runs completed",
stageDeploy: &codebaseApi.CDStageDeploy{
Spec: codebaseApi.CDStageDeploySpec{
Pipeline: "app1",
Stage: "dev",
},
Status: codebaseApi.CDStageDeployStatus{
Status: codebaseApi.CDStageDeployStatusInQueue,
},
},
k8sClient: func(t *testing.T) client.Client {
return fake.NewClientBuilder().WithScheme(scheme).Build()
},
wantErr: require.NoError,
wantStatus: codebaseApi.CDStageDeployStatusPending,
},
{
name: "queued CDStageDeploy should be queued if not all pipeline runs completed",
stageDeploy: &codebaseApi.CDStageDeploy{
Spec: codebaseApi.CDStageDeploySpec{
Pipeline: "app1",
Stage: "dev",
},
Status: codebaseApi.CDStageDeployStatus{
Status: codebaseApi.CDStageDeployStatusInQueue,
},
},
k8sClient: func(t *testing.T) client.Client {
piperun := &tektonpipelineApi.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Labels: map[string]string{
codebaseApi.CdPipelineLabel: "app1",
codebaseApi.CdStageLabel: "app1-dev",
},
},
}

return fake.NewClientBuilder().WithScheme(scheme).WithObjects(piperun).Build()
},
wantErr: require.NoError,
wantStatus: codebaseApi.CDStageDeployStatusInQueue,
},
{
name: "queued CDStageDeploy should be queued if another CDStageDeploy is running",
stageDeploy: &codebaseApi.CDStageDeploy{
Spec: codebaseApi.CDStageDeploySpec{
Pipeline: "app1",
Stage: "dev",
},
Status: codebaseApi.CDStageDeployStatus{
Status: codebaseApi.CDStageDeployStatusInQueue,
},
},
k8sClient: func(t *testing.T) client.Client {
stageDeploy := &codebaseApi.CDStageDeploy{
ObjectMeta: metav1.ObjectMeta{
Name: "running",
Namespace: "default",
Labels: map[string]string{
codebaseApi.CdPipelineLabel: "app1",
codebaseApi.CdStageLabel: "app1-dev",
},
},
Spec: codebaseApi.CDStageDeploySpec{
Pipeline: "app1",
Stage: "dev",
},
Status: codebaseApi.CDStageDeployStatus{
Status: codebaseApi.CDStageDeployStatusRunning,
},
}

return fake.NewClientBuilder().WithScheme(scheme).WithObjects(stageDeploy).Build()
},
wantErr: require.NoError,
wantStatus: codebaseApi.CDStageDeployStatusInQueue,
},
}

for _, tt := range tests {
Expand Down
8 changes: 4 additions & 4 deletions controllers/codebaseimagestream/chain/put_cd_stage_deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,8 @@ func (h PutCDStageDeploy) skipCDStageDeployCreation(ctx context.Context, envLabe
list,
client.InNamespace(namespace),
client.MatchingLabels{
"app.edp.epam.com/cdpipeline": env[0],
"app.edp.epam.com/cdstage": fmt.Sprintf("%s-%s", env[0], env[1]),
codebaseApi.CdPipelineLabel: env[0],
codebaseApi.CdStageLabel: fmt.Sprintf("%s-%s", env[0], env[1]),
},
); err != nil {
return false, fmt.Errorf("failed to get CDStageDeploys: %w", err)
Expand Down Expand Up @@ -196,8 +196,8 @@ func (h PutCDStageDeploy) create(ctx context.Context, command *cdStageDeployComm
}

stageDeploy.SetLabels(map[string]string{
"app.edp.epam.com/cdpipeline": command.Pipeline,
"app.edp.epam.com/cdstage": stageDeploy.GetStageCRName(),
codebaseApi.CdPipelineLabel: command.Pipeline,
codebaseApi.CdStageLabel: stageDeploy.GetStageCRName(),
})

stage := &pipelineApi.Stage{}
Expand Down
36 changes: 18 additions & 18 deletions controllers/codebaseimagestream/chain/put_cd_stage_deploy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ func TestPutCDStageDeploy_ServeRequest(t *testing.T) {
Name: "test-stage-deploy",
Namespace: "default",
Labels: map[string]string{
"app.edp.epam.com/cdpipeline": "ci",
"app.edp.epam.com/cdstage": "ci-prod",
codebaseApi.CdPipelineLabel: "ci",
codebaseApi.CdStageLabel: "ci-prod",
},
},
},
Expand All @@ -79,8 +79,8 @@ func TestPutCDStageDeploy_ServeRequest(t *testing.T) {
cdStageDeploys,
client.InNamespace("default"),
client.MatchingLabels{
"app.edp.epam.com/cdpipeline": "ci",
"app.edp.epam.com/cdstage": "ci-dev",
codebaseApi.CdPipelineLabel: "ci",
codebaseApi.CdStageLabel: "ci-dev",
}))
require.Len(t, cdStageDeploys.Items, 1)
},
Expand Down Expand Up @@ -116,8 +116,8 @@ func TestPutCDStageDeploy_ServeRequest(t *testing.T) {
Name: "test-stage-deploy",
Namespace: "default",
Labels: map[string]string{
"app.edp.epam.com/cdpipeline": "ci",
"app.edp.epam.com/cdstage": "ci-dev",
codebaseApi.CdPipelineLabel: "ci",
codebaseApi.CdStageLabel: "ci-dev",
},
},
},
Expand All @@ -132,8 +132,8 @@ func TestPutCDStageDeploy_ServeRequest(t *testing.T) {
cdStageDeploys,
client.InNamespace("default"),
client.MatchingLabels{
"app.edp.epam.com/cdpipeline": "ci",
"app.edp.epam.com/cdstage": "ci-dev",
codebaseApi.CdPipelineLabel: "ci",
codebaseApi.CdStageLabel: "ci-dev",
}))
require.Len(t, cdStageDeploys.Items, 2)
},
Expand Down Expand Up @@ -169,8 +169,8 @@ func TestPutCDStageDeploy_ServeRequest(t *testing.T) {
Name: "test-stage-deploy1",
Namespace: "default",
Labels: map[string]string{
"app.edp.epam.com/cdpipeline": "ci",
"app.edp.epam.com/cdstage": "ci-dev",
codebaseApi.CdPipelineLabel: "ci",
codebaseApi.CdStageLabel: "ci-dev",
},
},
},
Expand All @@ -179,8 +179,8 @@ func TestPutCDStageDeploy_ServeRequest(t *testing.T) {
Name: "test-stage-deploy2",
Namespace: "default",
Labels: map[string]string{
"app.edp.epam.com/cdpipeline": "ci",
"app.edp.epam.com/cdstage": "ci-dev",
codebaseApi.CdPipelineLabel: "ci",
codebaseApi.CdStageLabel: "ci-dev",
},
},
},
Expand All @@ -195,8 +195,8 @@ func TestPutCDStageDeploy_ServeRequest(t *testing.T) {
cdStageDeploys,
client.InNamespace("default"),
client.MatchingLabels{
"app.edp.epam.com/cdpipeline": "ci",
"app.edp.epam.com/cdstage": "ci-dev",
codebaseApi.CdPipelineLabel: "ci",
codebaseApi.CdStageLabel: "ci-dev",
}))
require.Len(t, cdStageDeploys.Items, 2)
},
Expand Down Expand Up @@ -366,8 +366,8 @@ func TestPutCDStageDeploy_ServeRequest(t *testing.T) {
Name: "test-stage-deploy",
Namespace: "default",
Labels: map[string]string{
"app.edp.epam.com/cdpipeline": "ci",
"app.edp.epam.com/cdstage": "ci-prod",
codebaseApi.CdPipelineLabel: "ci",
codebaseApi.CdStageLabel: "ci-prod",
},
},
},
Expand All @@ -382,8 +382,8 @@ func TestPutCDStageDeploy_ServeRequest(t *testing.T) {
cdStageDeploys,
client.InNamespace("default"),
client.MatchingLabels{
"app.edp.epam.com/cdpipeline": "ci",
"app.edp.epam.com/cdstage": "ci-dev",
codebaseApi.CdPipelineLabel: "ci",
codebaseApi.CdStageLabel: "ci-dev",
}))
require.Len(t, cdStageDeploys.Items, 0)
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/util/template_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,9 +205,11 @@ func TestCopyRpmPackageTemplates(t *testing.T) {

b, err := os.ReadFile(fmt.Sprintf("%s/test-application.spec", dest))
require.Contains(t, string(b), "test-application")
require.NoError(t, err)

b, err = os.ReadFile(fmt.Sprintf("%s/test-application.service", dest))
require.Contains(t, string(b), "test-application")
require.NoError(t, err)

_, err = os.Stat(fmt.Sprintf("%s/test-application.service", dest))
require.NoError(t, err)
Expand Down

0 comments on commit 65ead20

Please sign in to comment.