From e061ef223a93a28d88bb1238a0acd7d29f39f2b6 Mon Sep 17 00:00:00 2001 From: Nick Williams <65220492+Nickcw6@users.noreply.github.com> Date: Wed, 21 Aug 2024 05:18:23 +0100 Subject: [PATCH] fix(circleci-plugin): deployments matching job name (#7910) * fix(circleci-plugin): set job pipeline_id column to generated workflow id for correct cicd_deployments generation * refactor(circleci-plugin): rename IdGen function to better represent it deals with workflows * test(circleci-plugin): update cicd_tasks snapshot table to exepcted pipeline_id values --- .../e2e/snapshot_tables/cicd_tasks.csv | 18 +++++++++--------- .../plugins/circleci/tasks/job_converter.go | 7 ++++--- backend/plugins/circleci/tasks/shared.go | 2 +- .../circleci/tasks/workflow_converter.go | 2 +- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/backend/plugins/circleci/e2e/snapshot_tables/cicd_tasks.csv b/backend/plugins/circleci/e2e/snapshot_tables/cicd_tasks.csv index e58108926fa..722138f62b7 100644 --- a/backend/plugins/circleci/e2e/snapshot_tables/cicd_tasks.csv +++ b/backend/plugins/circleci/e2e/snapshot_tables/cicd_tasks.csv @@ -1,10 +1,10 @@ id,name,pipeline_id,result,status,original_status,original_result,type,environment,duration_sec,queued_duration_sec,created_date,queued_date,started_date,finished_date,cicd_scope_id -circleci:CircleciJob:1:6731159f-5275-4bfa-ba70-39d343d63814:ab8c3282-0e74-4a41-834e-152a71280bed,build,23622ee4-e150-4920-9d66-81533fa765a4,FAILURE,DONE,failed,,,PRODUCTION,3,,2023-03-25T17:52:20.000+00:00,,2023-03-25T17:52:20.000+00:00,2023-03-25T17:52:23.000+00:00,circleci:CircleciProject:1:abcd -circleci:CircleciJob:1:7370985a-9de3-4a47-acbc-e6a1fe8e5812:a00f80bc-f759-4900-97a5-2d121d80bde8,build,866e967d-f826-4470-aed6-fc0c92e98703,FAILURE,DONE,failed,,,PRODUCTION,16,,2023-03-25T17:56:27.000+00:00,,2023-03-25T17:56:27.000+00:00,2023-03-25T17:56:43.000+00:00,circleci:CircleciProject:1:abcd -circleci:CircleciJob:1:89054eb2-8e85-4f5c-9a93-66d753a0e970:c46092f9-6f82-4a52-8d8b-bd70d365dfc2,say-hello,625ca634-68fe-4515-91f0-7ba8af51dc99,SUCCESS,DONE,success,,,PRODUCTION,3,,2023-03-25T17:39:25.000+00:00,,2023-03-25T17:39:25.000+00:00,2023-03-25T17:39:28.000+00:00,circleci:CircleciProject:1:abcd -circleci:CircleciJob:1:8971a56b-5547-4824-94dd-07bb467524c5:7b96e45b-b10e-47a0-95d0-96580b88bdda,say-hello,87aad008-1ad5-486a-8174-fdeed846561a,SUCCESS,DONE,success,,,PRODUCTION,3,,2023-03-25T17:12:20.000+00:00,,2023-03-25T17:12:20.000+00:00,2023-03-25T17:12:23.000+00:00,circleci:CircleciProject:1:abcd -circleci:CircleciJob:1:8fe60291-68f7-40e2-acec-d99bff4da713:afde48dd-7319-4973-b3c8-e00308ff7667,say-hello,afef32b3-5ffe-48d2-8d9e-46dcedd82554,SUCCESS,DONE,success,,,PRODUCTION,2,,2023-03-25T17:12:20.000+00:00,,2023-03-25T17:12:20.000+00:00,2023-03-25T17:12:22.000+00:00,circleci:CircleciProject:1:abcd -circleci:CircleciJob:1:b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14:76c1f2cc-27ea-47aa-8167-48d2633abdba,build,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,FAILURE,DONE,failed,,,PRODUCTION,12,,2023-03-25T17:54:11.000+00:00,,2023-03-25T17:54:11.000+00:00,2023-03-25T17:54:23.000+00:00,circleci:CircleciProject:1:abcd -circleci:CircleciJob:1:c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943:a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b,build,2c45280f-7fb3-4025-b703-a547c4a94916,FAILURE,DONE,failed,,,PRODUCTION,3,,2023-03-25T17:50:22.000+00:00,,2023-03-25T17:50:22.000+00:00,2023-03-25T17:50:25.000+00:00,circleci:CircleciProject:1:abcd -circleci:CircleciJob:1:fc76deef-bcdd-4856-8e96-a8e2d1c5a85f:004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e,build,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,FAILURE,DONE,failed,,,PRODUCTION,13,,2023-03-25T18:06:15.000+00:00,,2023-03-25T18:06:15.000+00:00,2023-03-25T18:06:28.000+00:00,circleci:CircleciProject:1:abcd -circleci:CircleciJob:1:fd0bd4f5-264f-4e3c-a151-06153c018f78:2ff3594e-9da1-4306-aefa-77b72a97971e,build,7fcc1623-edcc-4a76-ad20-cd81aa83519f,SUCCESS,DONE,success,,,PRODUCTION,13,,2023-03-25T18:13:25.000+00:00,,2023-03-25T18:13:25.000+00:00,2023-03-25T18:13:38.000+00:00,circleci:CircleciProject:1:abcd +circleci:CircleciJob:1:6731159f-5275-4bfa-ba70-39d343d63814:ab8c3282-0e74-4a41-834e-152a71280bed,build,circleci:CircleciWorkflow:1:6731159f-5275-4bfa-ba70-39d343d63814,FAILURE,DONE,failed,,,PRODUCTION,3,,2023-03-25T17:52:20.000+00:00,,2023-03-25T17:52:20.000+00:00,2023-03-25T17:52:23.000+00:00,circleci:CircleciProject:1:abcd +circleci:CircleciJob:1:7370985a-9de3-4a47-acbc-e6a1fe8e5812:a00f80bc-f759-4900-97a5-2d121d80bde8,build,circleci:CircleciWorkflow:1:7370985a-9de3-4a47-acbc-e6a1fe8e5812,FAILURE,DONE,failed,,,PRODUCTION,16,,2023-03-25T17:56:27.000+00:00,,2023-03-25T17:56:27.000+00:00,2023-03-25T17:56:43.000+00:00,circleci:CircleciProject:1:abcd +circleci:CircleciJob:1:89054eb2-8e85-4f5c-9a93-66d753a0e970:c46092f9-6f82-4a52-8d8b-bd70d365dfc2,say-hello,circleci:CircleciWorkflow:1:89054eb2-8e85-4f5c-9a93-66d753a0e970,SUCCESS,DONE,success,,,PRODUCTION,3,,2023-03-25T17:39:25.000+00:00,,2023-03-25T17:39:25.000+00:00,2023-03-25T17:39:28.000+00:00,circleci:CircleciProject:1:abcd +circleci:CircleciJob:1:8971a56b-5547-4824-94dd-07bb467524c5:7b96e45b-b10e-47a0-95d0-96580b88bdda,say-hello,circleci:CircleciWorkflow:1:8971a56b-5547-4824-94dd-07bb467524c5,SUCCESS,DONE,success,,,PRODUCTION,3,,2023-03-25T17:12:20.000+00:00,,2023-03-25T17:12:20.000+00:00,2023-03-25T17:12:23.000+00:00,circleci:CircleciProject:1:abcd +circleci:CircleciJob:1:8fe60291-68f7-40e2-acec-d99bff4da713:afde48dd-7319-4973-b3c8-e00308ff7667,say-hello,circleci:CircleciWorkflow:1:8fe60291-68f7-40e2-acec-d99bff4da713,SUCCESS,DONE,success,,,PRODUCTION,2,,2023-03-25T17:12:20.000+00:00,,2023-03-25T17:12:20.000+00:00,2023-03-25T17:12:22.000+00:00,circleci:CircleciProject:1:abcd +circleci:CircleciJob:1:b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14:76c1f2cc-27ea-47aa-8167-48d2633abdba,build,circleci:CircleciWorkflow:1:b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,FAILURE,DONE,failed,,,PRODUCTION,12,,2023-03-25T17:54:11.000+00:00,,2023-03-25T17:54:11.000+00:00,2023-03-25T17:54:23.000+00:00,circleci:CircleciProject:1:abcd +circleci:CircleciJob:1:c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943:a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b,build,circleci:CircleciWorkflow:1:c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,FAILURE,DONE,failed,,,PRODUCTION,3,,2023-03-25T17:50:22.000+00:00,,2023-03-25T17:50:22.000+00:00,2023-03-25T17:50:25.000+00:00,circleci:CircleciProject:1:abcd +circleci:CircleciJob:1:fc76deef-bcdd-4856-8e96-a8e2d1c5a85f:004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e,build,circleci:CircleciWorkflow:1:fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,FAILURE,DONE,failed,,,PRODUCTION,13,,2023-03-25T18:06:15.000+00:00,,2023-03-25T18:06:15.000+00:00,2023-03-25T18:06:28.000+00:00,circleci:CircleciProject:1:abcd +circleci:CircleciJob:1:fd0bd4f5-264f-4e3c-a151-06153c018f78:2ff3594e-9da1-4306-aefa-77b72a97971e,build,circleci:CircleciWorkflow:1:fd0bd4f5-264f-4e3c-a151-06153c018f78,SUCCESS,DONE,success,,,PRODUCTION,13,,2023-03-25T18:13:25.000+00:00,,2023-03-25T18:13:25.000+00:00,2023-03-25T18:13:38.000+00:00,circleci:CircleciProject:1:abcd diff --git a/backend/plugins/circleci/tasks/job_converter.go b/backend/plugins/circleci/tasks/job_converter.go index 9d0f806b9bd..2cb8de1207c 100644 --- a/backend/plugins/circleci/tasks/job_converter.go +++ b/backend/plugins/circleci/tasks/job_converter.go @@ -18,6 +18,9 @@ limitations under the License. package tasks import ( + "reflect" + "time" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -25,8 +28,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/circleci/models" - "reflect" - "time" ) var ConvertJobsMeta = plugin.SubTaskMeta{ @@ -66,7 +67,7 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) errors.Error { }, CicdScopeId: getProjectIdGen().Generate(data.Options.ConnectionId, data.Project.Id), Name: userTool.Name, - PipelineId: userTool.PipelineId, + PipelineId: getWorkflowIdGen().Generate(data.Options.ConnectionId, userTool.WorkflowId), TaskDatesInfo: devops.TaskDatesInfo{ CreatedDate: createdAt, QueuedDate: userTool.QueuedAt.ToNullableTime(), diff --git a/backend/plugins/circleci/tasks/shared.go b/backend/plugins/circleci/tasks/shared.go index dd0cdf8c7f2..dd2c4944a1b 100644 --- a/backend/plugins/circleci/tasks/shared.go +++ b/backend/plugins/circleci/tasks/shared.go @@ -48,7 +48,7 @@ func getProjectIdGen() *didgen.DomainIdGenerator { return projectIdGen } -func getPipelineIdGen() *didgen.DomainIdGenerator { +func getWorkflowIdGen() *didgen.DomainIdGenerator { if pipelineIdGen == nil { pipelineIdGen = didgen.NewDomainIdGenerator(&models.CircleciWorkflow{}) } diff --git a/backend/plugins/circleci/tasks/workflow_converter.go b/backend/plugins/circleci/tasks/workflow_converter.go index 91806fad545..e1af8f96153 100644 --- a/backend/plugins/circleci/tasks/workflow_converter.go +++ b/backend/plugins/circleci/tasks/workflow_converter.go @@ -60,7 +60,7 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error { createdAt := userTool.CreatedAt.ToTime() pipeline := &devops.CICDPipeline{ DomainEntity: domainlayer.DomainEntity{ - Id: getPipelineIdGen().Generate(data.Options.ConnectionId, userTool.Id), + Id: getWorkflowIdGen().Generate(data.Options.ConnectionId, userTool.Id), }, Name: userTool.Name, DurationSec: userTool.DurationSec,