From 31bf915790ee70c3266f7112767bbf9e44071e30 Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 25 Aug 2023 14:08:31 +0200 Subject: [PATCH 01/22] Prepare for Apache migration --- .ci/jenkins/Jenkinsfile.build-operator-node | 2 +- .ci/jenkins/Jenkinsfile.nightly | 2 +- .ci/jenkins/Jenkinsfile.nightly.cloud | 2 +- .ci/jenkins/Jenkinsfile.prod.nightly | 6 +-- .ci/jenkins/Jenkinsfile.release | 2 +- .ci/jenkins/Jenkinsfile.release.cloud | 2 +- .ci/jenkins/Jenkinsfile.setup-branch | 2 +- .ci/jenkins/Jenkinsfile.setup-branch.cloud | 2 +- ...kinsfile.tools.update-jenkins-dependencies | 2 +- .ci/jenkins/config/branch.yaml | 33 ++++++------ .ci/jenkins/config/main.yaml | 50 +------------------ docs/jenkins.md | 4 +- dsl/seed/jenkinsfiles/Jenkinsfile.buildchain | 2 +- ...Jenkinsfile.environment.integration-branch | 2 +- .../jenkinsfiles/Jenkinsfile.remove.branches | 2 +- dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch | 4 +- dsl/seed/jenkinsfiles/Jenkinsfile.seed.main | 2 +- .../Jenkinsfile.tools.toggle-triggers | 2 +- .../Jenkinsfile.update-quarkus-platform | 2 +- dsl/seed/jobs/seed_job_branch.groovy | 2 +- dsl/seed/jobs/seed_job_main.groovy | 2 +- .../jenkins/jobdsl/KogitoJobTemplate.groovy | 2 +- .../jenkins/jobdsl/utils/SeedJobUtils.groovy | 2 +- 23 files changed, 45 insertions(+), 88 deletions(-) diff --git a/.ci/jenkins/Jenkinsfile.build-operator-node b/.ci/jenkins/Jenkinsfile.build-operator-node index 116500afb..44962508c 100644 --- a/.ci/jenkins/Jenkinsfile.build-operator-node +++ b/.ci/jenkins/Jenkinsfile.build-operator-node @@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library pipeline { agent { - label 'kie-rhel8-priority && !built-in' + label 'ubuntu' } options { diff --git a/.ci/jenkins/Jenkinsfile.nightly b/.ci/jenkins/Jenkinsfile.nightly index 3a41a7aaf..b0f215fcf 100644 --- a/.ci/jenkins/Jenkinsfile.nightly +++ b/.ci/jenkins/Jenkinsfile.nightly @@ -19,7 +19,7 @@ UNSTABLE_STAGES = [:] // Should be multibranch pipeline pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { diff --git a/.ci/jenkins/Jenkinsfile.nightly.cloud b/.ci/jenkins/Jenkinsfile.nightly.cloud index 65015d054..f3f5d0b81 100644 --- a/.ci/jenkins/Jenkinsfile.nightly.cloud +++ b/.ci/jenkins/Jenkinsfile.nightly.cloud @@ -25,7 +25,7 @@ promoteImageParamsPrefix = 'PROMOTE_IMAGE' // Should be multibranch pipeline pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { diff --git a/.ci/jenkins/Jenkinsfile.prod.nightly b/.ci/jenkins/Jenkinsfile.prod.nightly index 917cec3fb..d7ac44cfe 100644 --- a/.ci/jenkins/Jenkinsfile.prod.nightly +++ b/.ci/jenkins/Jenkinsfile.prod.nightly @@ -4,11 +4,11 @@ import org.kie.jenkins.MavenCommand pipeline { agent { - label 'kie-rhel8 && docker && kie-mem24g && !built-in' + label 'ubuntu' } tools { - maven 'kie-maven-3.8.7' - jdk 'kie-jdk11' + maven 'maven_3.8.6' + jdk 'jdk_11_latest' nodejs 'nodejs-16.2.0' } parameters { diff --git a/.ci/jenkins/Jenkinsfile.release b/.ci/jenkins/Jenkinsfile.release index cb4d2cb95..087a1de69 100644 --- a/.ci/jenkins/Jenkinsfile.release +++ b/.ci/jenkins/Jenkinsfile.release @@ -19,7 +19,7 @@ releaseProperties = [:] pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } // parameters { diff --git a/.ci/jenkins/Jenkinsfile.release.cloud b/.ci/jenkins/Jenkinsfile.release.cloud index e6df13c4f..c9cd756e7 100644 --- a/.ci/jenkins/Jenkinsfile.release.cloud +++ b/.ci/jenkins/Jenkinsfile.release.cloud @@ -26,7 +26,7 @@ promoteImageParamsPrefix = 'PROMOTE_IMAGE' pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } // parameters { diff --git a/.ci/jenkins/Jenkinsfile.setup-branch b/.ci/jenkins/Jenkinsfile.setup-branch index 3388cb097..c9474d736 100644 --- a/.ci/jenkins/Jenkinsfile.setup-branch +++ b/.ci/jenkins/Jenkinsfile.setup-branch @@ -12,7 +12,7 @@ UNSTABLE_STAGES = [:] pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { diff --git a/.ci/jenkins/Jenkinsfile.setup-branch.cloud b/.ci/jenkins/Jenkinsfile.setup-branch.cloud index 2f4aa42b2..f03831de0 100644 --- a/.ci/jenkins/Jenkinsfile.setup-branch.cloud +++ b/.ci/jenkins/Jenkinsfile.setup-branch.cloud @@ -16,7 +16,7 @@ examplesImagesParamsPrefix = 'EXAMPLES_IMAGES' pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { diff --git a/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies b/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies index 49195f668..1da1b984d 100644 --- a/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies +++ b/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies @@ -6,7 +6,7 @@ branchCreated = false pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index 8d907eb9f..e05dc2b96 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -1,3 +1,6 @@ +generation_config: + missing_environment: ignore + environments: native: env_vars: @@ -32,8 +35,8 @@ environments: quarkus-3: pull_request_default_check: false env_vars: - BUILD_MAVEN_TOOL: kie-maven-3.9.1 - BUILD_JDK_TOOL: kie-jdk17 + BUILD_MAVEN_TOOL: maven_3.9.3 + BUILD_JDK_TOOL: jdk_17_latest BUILD_MVN_OPTS: -Denforcer.skip ids: - quarkus @@ -101,20 +104,20 @@ buildchain_config: repository: kogito-pipelines file_path: .ci/pull-request-config.yaml maven: - settings_file_id: kogito_release_settings + settings_file_id: TO_DEFINE nexus: - release_url: https://repository.jboss.org/nexus - release_repository: jboss-releases-repository - staging_profile_url: https://repository.jboss.org/nexus/content/groups/kogito-public/ - staging_profile_id: 1976dd1c18392a - build_promotion_profile_id: ea49ccd6f174 + release_url: TO_DEFINE + release_repository: TO_DEFINE + staging_profile_url: TO_DEFINE + staging_profile_id: TO_DEFINE + build_promotion_profile_id: TO_DEFINE artifacts_repository: '' artifacts_upload_repository: - url: https://repository.jboss.org/nexus/content/repositories/snapshots/ - creds_id: kogito-snapshots-upload + url: TO_DEFINE + creds_id: TO_DEFINE quarkus_platform_repository: - url: https://repository.jboss.org/nexus/content/repositories/kogito-internal-repository/ - creds_id: kogito-internal-nexus-creds + url: TO_DEFINE + creds_id: TO_DEFINE cloud: image: registry_credentials_nightly: nightly_kogito @@ -125,7 +128,7 @@ cloud: jenkins: email_creds_id: KOGITO_CI_EMAIL_TO default_tools: - jdk: kie-jdk11 - maven: kie-maven-3.8.7 - sonar_jdk: kie-jdk17 + jdk: jdk_11_latest + maven: maven_3.8.6 + sonar_jdk: jdk_17_latest jobs_definition_file: .ci/jenkins/dsl/jobs.groovy diff --git a/.ci/jenkins/config/main.yaml b/.ci/jenkins/config/main.yaml index ce8ece56f..f0976f31f 100644 --- a/.ci/jenkins/config/main.yaml +++ b/.ci/jenkins/config/main.yaml @@ -19,54 +19,8 @@ ecosystem: - kie-docs.* git: branches: - - name: main + - name: apache_migration main_branch: true - seed: - branch: main - - name: 1.13.x - old_folder_structure: true - seed: - branch: 1.13.x - config_file: - path: dsl/seed/config/branch.yaml - jenkinsfile: dsl/seed/jobs/Jenkinsfile.seed.branch - - name: 1.13.x-blue - old_folder_structure: true - seed: - branch: 1.13.x-blue - config_file: - path: dsl/seed/config/branch.yaml - jenkinsfile: dsl/seed/jobs/Jenkinsfile.seed.branch - - name: 1.29.x - seed: - branch: seed-kogito-1.29.x - config_file: - path: dsl/config/branch.yaml - - name: 1.38.x-prod - seed: - branch: seed-kogito-1.38.x - config_file: - path: dsl/config/branch.yaml - - name: 1.40.x - seed: - branch: seed-kogito-1.40.x - config_file: - path: dsl/config/branch.yaml - - name: 1.40.x-prod - seed: - branch: seed-kogito-1.40.x - config_file: - path: dsl/config/branch.yaml - - name: 1.42.x - seed: - branch: seed-kogito-1.42.x - config_file: - path: dsl/config/branch.yaml - - name: 1.43.x - seed: - branch: seed-kogito-1.43.x - config_file: - path: dsl/config/branch.yaml seed: config_file: git: @@ -74,7 +28,7 @@ seed: author: name: kiegroup credentials_id: kie-ci3 - branch: main + branch: apache_migration path: .ci/jenkins/config/branch.yaml jenkinsfile: dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch jenkins: diff --git a/docs/jenkins.md b/docs/jenkins.md index 05c1c6697..33bab7278 100644 --- a/docs/jenkins.md +++ b/docs/jenkins.md @@ -199,8 +199,8 @@ cloud: jenkins: email_creds_id: KOGITO_CI_EMAIL_TO_PERSO default_tools: - jdk: kie-jdk11 - maven: kie-maven-3.8.7 + jdk: jdk_11_latest + maven: maven_3.8.6 ``` diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain index 79889115e..463a16d93 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain @@ -3,7 +3,7 @@ import org.kie.jenkins.MavenCommand import org.kie.jenkins.MavenSettingsUtils -agentLabel = "${env.ADDITIONAL_LABEL?.trim() ?: 'kie-rhel8 && docker && kie-mem16g'} && !built-in" +agentLabel = "${env.ADDITIONAL_LABEL?.trim() ?: 'ubuntu'} && !built-in" timeoutValue = env.ADDITIONAL_TIMEOUT?.trim() ?: '180' jdkTool = env.BUILD_JDK_TOOL sonarJdkTool = env.SONAR_JDK_TOOL diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch b/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch index 4b769bbd3..b766d8beb 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch @@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } environment { diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches b/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches index be29f3e9c..f42833b20 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches @@ -6,7 +6,7 @@ import org.kie.jenkins.MavenCommand pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options{ diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch index 5e8891c74..b589b496f 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch @@ -32,7 +32,7 @@ pipeline { } tools { - jdk 'kie-jdk11' + jdk 'jdk_11_latest' } options { @@ -309,7 +309,7 @@ void fillEnvProperties(Map envProperties, String envKeyPrefix, Map propsMap) { } if (value instanceof Map) { fillEnvProperties(envProperties, newKey, value as Map) - } else if ([List, net.sf.json.JSONArray].any { it.isAssignableFrom(value.getClass()) }) { + } else if (value instanceof List || value instanceof net.sf.json.JSONArray) { envProperties.put(newKey, (value as List).collect { it }.join(',')) } else { envProperties.put(newKey, value) diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main index 4a56f485c..7745c7ea1 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main @@ -20,7 +20,7 @@ pipeline { } tools { - jdk 'kie-jdk11' + jdk 'jdk_11_latest' } environment { diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers b/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers index e0f3e68f8..d63cd573d 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers @@ -5,7 +5,7 @@ import org.kie.jenkins.MavenCommand pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-platform b/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-platform index d318e9883..08a06e49f 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-platform +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-platform @@ -3,7 +3,7 @@ import org.jenkinsci.plugins.workflow.libs.Library import org.kie.jenkins.MavenCommand -AGENT_LABEL="kie-rhel8 && docker && kie-mem16g && !built-in" +AGENT_LABEL="ubuntu" quarkusPlatformRepo="quarkus-platform" branchCreated = false diff --git a/dsl/seed/jobs/seed_job_branch.groovy b/dsl/seed/jobs/seed_job_branch.groovy index c99c17743..976e6c935 100644 --- a/dsl/seed/jobs/seed_job_branch.groovy +++ b/dsl/seed/jobs/seed_job_branch.groovy @@ -76,7 +76,7 @@ pipelineJob("${GENERATION_BRANCH}/${JOB_NAME}") { env('SEED_AUTHOR_CREDS_ID', Utils.getSeedAuthorCredsId(this)) env('SEED_BRANCH', Utils.getSeedBranch(this)) - env('AGENT_LABEL', Utils.isProdEnvironment(this) ? 'kie-rhel8 && !built-in' : 'kie-rhel8-priority') + env('AGENT_LABEL', 'ubuntu') env('JENKINS_EMAIL_CREDS_ID', Utils.getJenkinsEmailCredsId(this)) } diff --git a/dsl/seed/jobs/seed_job_main.groovy b/dsl/seed/jobs/seed_job_main.groovy index c43c2d3be..4265c4396 100644 --- a/dsl/seed/jobs/seed_job_main.groovy +++ b/dsl/seed/jobs/seed_job_main.groovy @@ -83,7 +83,7 @@ pipelineJob('0-seed-job') { } environmentVariables { - env('AGENT_LABEL', Utils.isProdEnvironment(this) ? 'kie-rhel8 && !built-in' : 'kie-rhel8-priority') + env('AGENT_LABEL', 'ubuntu') env('JENKINS_EMAIL_CREDS_ID', Utils.getJenkinsEmailCredsId(this)) } diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy index e4b84a259..ab23b3432 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy @@ -44,7 +44,7 @@ class KogitoJobTemplate { String jobFolderName = '' def jobFolder = jobParams.job.folder if (jobFolder) { - if (![JenkinsFolder].any { it.isAssignableFrom(jobFolder.getClass()) }) { + if (!jobFolder instanceof JenkinsFolder) { throw new RuntimeException('Folder is not of type org.kie.jenkins.jobdsl.model.JenkinsFolder') } diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/SeedJobUtils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/SeedJobUtils.groovy index 8abdbb183..2f53892a6 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/SeedJobUtils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/SeedJobUtils.groovy @@ -38,7 +38,7 @@ class SeedJobUtils { env('JOB_RELATIVE_PATH_TO_TRIGGER', jobRelativePathToTrigger) env('LISTEN_TO_MODIFIED_PATHS', new groovy.json.JsonBuilder(pathsToListen).toString()) - env('AGENT_LABEL', Utils.isProdEnvironment(jenkinsScript) ? 'rhel8 && !built-in' : 'kie-rhel8-priority') + env('AGENT_LABEL', 'ubuntu') } definition { From 58271f5701f6b28d74a3568079f13d761a71320a Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 25 Aug 2023 14:09:14 +0200 Subject: [PATCH 02/22] test config --- .ci/jenkins/config/branch.yaml | 128 ++++++++++++++++++--------------- .ci/jenkins/config/main.yaml | 2 +- 2 files changed, 72 insertions(+), 58 deletions(-) diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index e05dc2b96..dbf1c7376 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -1,55 +1,69 @@ generation_config: missing_environment: ignore +job_types: + setup-branch: + disabled: true + nightly: + disabled: false + other: + disabled: true + pullrequest: + disabled: false + release: + disabled: true + tools: + disabled: true + environments: - native: - env_vars: - NATIVE: true - BUILD_MVN_OPTS_CURRENT: -Dnative -Dquarkus.native.container-build=true - ADDITIONAL_TIMEOUT: 720 - ids: - - native - native-lts: - env_vars: - NATIVE: true - BUILD_MVN_OPTS: -Dproductized -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral - BUILD_MVN_OPTS_CURRENT: -Dnative -Dquarkus.native.container-build=true - QUARKUS_BRANCH: '2.13' - ADDITIONAL_TIMEOUT: 720 - BUILD_ENVIRONMENT: quarkus-lts - ids: - - native - - prod - - lts - quarkus-main: - enabled: false - env_vars: - QUARKUS_BRANCH: main - ids: - - quarkus + # native: + # env_vars: + # NATIVE: true + # BUILD_MVN_OPTS_CURRENT: -Dnative -Dquarkus.native.container-build=true + # ADDITIONAL_TIMEOUT: 720 + # ids: + # - native + # native-lts: + # env_vars: + # NATIVE: true + # BUILD_MVN_OPTS: -Dproductized -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral + # BUILD_MVN_OPTS_CURRENT: -Dnative -Dquarkus.native.container-build=true + # QUARKUS_BRANCH: '2.13' + # ADDITIONAL_TIMEOUT: 720 + # BUILD_ENVIRONMENT: quarkus-lts + # ids: + # - native + # - prod + # - lts + # quarkus-main: + # enabled: false + # env_vars: + # QUARKUS_BRANCH: main + # ids: + # - quarkus quarkus-branch: env_vars: QUARKUS_BRANCH: '2.16' ids: - quarkus - quarkus-3: - pull_request_default_check: false - env_vars: - BUILD_MAVEN_TOOL: maven_3.9.3 - BUILD_JDK_TOOL: jdk_17_latest - BUILD_MVN_OPTS: -Denforcer.skip - ids: - - quarkus - - quarkus3 - quarkus-lts: - pull_request_default_check: false - env_vars: - BUILD_MVN_OPTS: -Dproductized -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral - QUARKUS_BRANCH: '2.13' - ids: - - quarkus - - lts - - prod + # quarkus-3: + # pull_request_default_check: false + # env_vars: + # BUILD_MAVEN_TOOL: maven_3.9.3 + # BUILD_JDK_TOOL: jdk_17_latest + # BUILD_MVN_OPTS: -Denforcer.skip + # ids: + # - quarkus + # - quarkus3 + # quarkus-lts: + # pull_request_default_check: false + # env_vars: + # BUILD_MVN_OPTS: -Dproductized -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral + # QUARKUS_BRANCH: '2.13' + # ids: + # - quarkus + # - lts + # - prod sonarcloud: auto_generation: false env_vars: @@ -66,16 +80,16 @@ disable: triggers: false repositories: - name: kogito-pipelines -- name: kogito-runtimes -- name: kogito-apps -- name: kogito-examples -- name: kogito-images -- name: kogito-operator -- name: kogito-serverless-operator -- name: kie-tools -- name: kogito-docs -- name: kie-docs - branch: main-kogito +# - name: kogito-runtimes +# - name: kogito-apps +# - name: kogito-examples +# - name: kogito-images +# - name: kogito-operator +# - name: kogito-serverless-operator +# - name: kie-tools +# - name: kogito-docs +# - name: kie-docs +# branch: main-kogito productized_repositories: - name: kogito-pipelines - name: kogito-runtimes @@ -120,13 +134,13 @@ maven: creds_id: TO_DEFINE cloud: image: - registry_credentials_nightly: nightly_kogito - registry_credentials_release: release_kogito + registry_credentials_nightly: tradisso_registry_token # to set back + registry_credentials_release: tradisso_registry_token # to set back registry: quay.io - namespace: kiegroup + namespace: tradisso # to set back latest_git_branch: main jenkins: - email_creds_id: KOGITO_CI_EMAIL_TO + email_creds_id: KOGITO_CI_EMAIL_TO_PERSO default_tools: jdk: jdk_11_latest maven: maven_3.8.6 diff --git a/.ci/jenkins/config/main.yaml b/.ci/jenkins/config/main.yaml index f0976f31f..07bf0dc17 100644 --- a/.ci/jenkins/config/main.yaml +++ b/.ci/jenkins/config/main.yaml @@ -32,4 +32,4 @@ seed: path: .ci/jenkins/config/branch.yaml jenkinsfile: dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch jenkins: - email_creds_id: KOGITO_CI_EMAIL_TO + email_creds_id: KOGITO_CI_EMAIL_TO_PERSO From e0f72c663ad979715a87ea52d8122ed5fe05ea49 Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 25 Aug 2023 14:13:37 +0200 Subject: [PATCH 03/22] test config --- .ci/jenkins/config/branch.yaml | 2 +- .ci/jenkins/config/main.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index dbf1c7376..e99892699 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -140,7 +140,7 @@ cloud: namespace: tradisso # to set back latest_git_branch: main jenkins: - email_creds_id: KOGITO_CI_EMAIL_TO_PERSO + email_creds_id: KOGITO_CI_EMAIL_TO default_tools: jdk: jdk_11_latest maven: maven_3.8.6 diff --git a/.ci/jenkins/config/main.yaml b/.ci/jenkins/config/main.yaml index 07bf0dc17..f0976f31f 100644 --- a/.ci/jenkins/config/main.yaml +++ b/.ci/jenkins/config/main.yaml @@ -32,4 +32,4 @@ seed: path: .ci/jenkins/config/branch.yaml jenkinsfile: dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch jenkins: - email_creds_id: KOGITO_CI_EMAIL_TO_PERSO + email_creds_id: KOGITO_CI_EMAIL_TO From f37a90c8a7f06a8c646aae93fc478de24497ceab Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 25 Aug 2023 14:15:50 +0200 Subject: [PATCH 04/22] test config --- .ci/jenkins/config/branch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index e99892699..c4e868409 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -118,7 +118,7 @@ buildchain_config: repository: kogito-pipelines file_path: .ci/pull-request-config.yaml maven: - settings_file_id: TO_DEFINE + settings_file_id: kogito_release_settings nexus: release_url: TO_DEFINE release_repository: TO_DEFINE From b281c9ff4c916a2adad905e125d796504a34e1e1 Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 25 Aug 2023 14:47:18 +0200 Subject: [PATCH 05/22] update --- .ci/jenkins/config/branch.yaml | 4 ++-- dsl/seed/jenkinsfiles/Jenkinsfile.buildchain | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index c4e868409..f1b9fd15b 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -118,7 +118,7 @@ buildchain_config: repository: kogito-pipelines file_path: .ci/pull-request-config.yaml maven: - settings_file_id: kogito_release_settings + settings_file_id: kie-release-settings nexus: release_url: TO_DEFINE release_repository: TO_DEFINE @@ -140,7 +140,7 @@ cloud: namespace: tradisso # to set back latest_git_branch: main jenkins: - email_creds_id: KOGITO_CI_EMAIL_TO + email_creds_id: KOGITO_CI_NOTIFICATION_EMAILS default_tools: jdk: jdk_11_latest maven: maven_3.8.6 diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain index 463a16d93..6978e0ee7 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain @@ -16,7 +16,7 @@ mavenDeployLocalDir = env.MAVEN_DEPLOY_LOCAL_DIR ?: '' buildChainType = env.BUILDCHAIN_TYPE?.trim() ?: 'cross_pr' buildChainProject = env.BUILDCHAIN_PROJECT?.trim() -defaultSettingsXmlId = isPRBuildChainType() || isFDBBuildChainType() ? 'kogito_pr_settings' : 'kogito_release_settings' +defaultSettingsXmlId = isPRBuildChainType() || isFDBBuildChainType() ? 'kie-pr-settings' : 'kie-release-settings' settingsXmlId = env.MAVEN_SETTINGS_FILE_ID ?: defaultSettingsXmlId enableSonarCloudAnalysis = env.ENABLE_SONARCLOUD ? env.ENABLE_SONARCLOUD.toBoolean() : false From 60009a955fb02b18ec3052535df19c8a406516a6 Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 25 Aug 2023 14:50:57 +0200 Subject: [PATCH 06/22] update --- .ci/jenkins/config/branch.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index f1b9fd15b..c7591c866 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -134,10 +134,9 @@ maven: creds_id: TO_DEFINE cloud: image: - registry_credentials_nightly: tradisso_registry_token # to set back - registry_credentials_release: tradisso_registry_token # to set back + registry_credentials: tradisso_registry_token # set to `kogito-quay-token` registry: quay.io - namespace: tradisso # to set back + namespace: tradisso # set to `kiegroup` after testing latest_git_branch: main jenkins: email_creds_id: KOGITO_CI_NOTIFICATION_EMAILS From b0dc35f5f5571a924a58e971a054ace1e6d52d48 Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 25 Aug 2023 14:51:29 +0200 Subject: [PATCH 07/22] correction --- .ci/jenkins/config/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/jenkins/config/main.yaml b/.ci/jenkins/config/main.yaml index f0976f31f..ca3320e4e 100644 --- a/.ci/jenkins/config/main.yaml +++ b/.ci/jenkins/config/main.yaml @@ -32,4 +32,4 @@ seed: path: .ci/jenkins/config/branch.yaml jenkinsfile: dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch jenkins: - email_creds_id: KOGITO_CI_EMAIL_TO + email_creds_id: KOGITO_CI_NOTIFICATION_EMAILS From 206b41ac1c888b39bd38dd26dc640cc464bd0ca2 Mon Sep 17 00:00:00 2001 From: radtriste Date: Tue, 29 Aug 2023 15:49:31 +0200 Subject: [PATCH 08/22] remove prod occurences --- .ci/jenkins/Jenkinsfile.prod.nightly | 269 ------- .ci/jenkins/config/branch.yaml | 32 - .ci/jenkins/config/main.yaml | 2 - .ci/jenkins/dsl/jobs.groovy | 2 +- .ci/nightly-build-config.yaml | 61 -- .ci/nightly-project-dependencies.yaml | 82 --- docs/jenkins.md | 2 - .../jenkinsfiles/Jenkinsfile.prod.prepare | 664 ------------------ .../jenkinsfiles/Jenkinsfile.release.prepare | 4 - dsl/seed/jenkinsfiles/Jenkinsfile.seed.main | 2 - dsl/seed/jobs/root_jobs.groovy | 60 -- .../org/kie/jenkins/jobdsl/Utils.groovy | 4 - .../kie/jenkins/jobdsl/JobScriptsSpec.groovy | 1 - 13 files changed, 1 insertion(+), 1184 deletions(-) delete mode 100644 .ci/jenkins/Jenkinsfile.prod.nightly delete mode 100644 .ci/nightly-build-config.yaml delete mode 100644 .ci/nightly-project-dependencies.yaml delete mode 100644 dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare diff --git a/.ci/jenkins/Jenkinsfile.prod.nightly b/.ci/jenkins/Jenkinsfile.prod.nightly deleted file mode 100644 index d7ac44cfe..000000000 --- a/.ci/jenkins/Jenkinsfile.prod.nightly +++ /dev/null @@ -1,269 +0,0 @@ -@Library('jenkins-pipeline-shared-libraries')_ - -import org.kie.jenkins.MavenCommand - -pipeline { - agent { - label 'ubuntu' - } - tools { - maven 'maven_3.8.6' - jdk 'jdk_11_latest' - nodejs 'nodejs-16.2.0' - } - parameters { - string(description: 'The UMB message version', name: 'UMB_VERSION', defaultValue: 'main') - string(description: 'The product version, if not provided the kogito-runtimes main branch one will be use', name: 'PRODUCT_VERSION') - string(description: 'The drools product version, if not provided the drools main branch one will be used', name: 'DROOLS_PRODUCT_VERSION') - string(description: 'The config repository branch', name: 'CONFIG_BRANCH', defaultValue: 'master') - } - options { - buildDiscarder logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10') - } - environment { - DATE_TIME_SUFFIX_FORMAT = 'yyyyMMddHH' - } - stages { - stage('Initialize') { - steps { - script { - // Fetch versions from pom if not provided - env.PRODUCT_VERSION = "${PRODUCT_VERSION ?: parseVersionFromPom('kiegroup/kogito-runtimes', getBuildBranch())}" - env.DROOLS_PRODUCT_VERSION = "${DROOLS_PRODUCT_VERSION ?: parseVersionFromPom('kiegroup/drools', getDroolsBranch())}" - - // Needed by Kogito Apps - setupCypressEnv('9.7.0') - } - } - } - stage('Install build-chain tool') { - steps { - script { - println "[INFO] Installing build-chain tool" - sh "npm install -g @kie/build-chain-action${env.NPM_REGISTRY_URL ? " -registry=${NPM_REGISTRY_URL}" : ''}" - - sh "npm list -g | grep build-chain" - } - } - } - stage('Clone build configuration repo') { - steps { - script { - def currentBranch = getBuildConfigBranch() - println "Checking out ${env.BUILD_CONFIGURATION_REPO_URL}:${currentBranch} into build_config folder" - sh "git clone -b ${currentBranch} --single-branch ${env.BUILD_CONFIGURATION_REPO_URL} build_config" - - // export PME parameters and build scripts from nightly build config - def buildConfigAdditionalVariables = [ - 'productVersion': env.PRODUCT_VERSION, - 'droolsProductVersion': env.DROOLS_PRODUCT_VERSION - ] - pmebuild.parseBuildConfig("$WORKSPACE/build_config/openshift-serverless-logic/nightly", buildConfigAdditionalVariables) - - def PME_BUILD_VARIABLES = env.PME_BUILD_VARIABLES.split(';').collect{ it.split('=')}.inject([:]) {map, item -> map << [(item.length == 2 ? item[0] : null): (item.length == 2 ? item[1] : null)]} - - // export Quarkus community versions - env.QUARKUS_VERSION_COMMUNITY = PME_BUILD_VARIABLES['quarkusVersionCommunity'] - env.QUARKUS_VERSION = PME_BUILD_VARIABLES['quarkusVersion'].replaceAll("\\{\\{.*\\}\\}", env.QUARKUS_VERSION_COMMUNITY) - env.QUARKUS_PLATFORM_VERSION = PME_BUILD_VARIABLES['quarkusPlatformVersion'].replaceAll("\\{\\{.*\\}\\}", env.QUARKUS_VERSION_COMMUNITY) - - // export images registries - env.IMAGE_TAG_VERSION = PME_BUILD_VARIABLES['imageTagVersion'] - env.DATA_INDEX_IMAGE_REGISTRY = PME_BUILD_VARIABLES['dataIndexImageRegistry'].replaceAll("\\{\\{.*\\}\\}", env.IMAGE_TAG_VERSION) - env.SWF_BUILDER_IMAGE_REGISTRY = PME_BUILD_VARIABLES['swfBuilderImageRegistry'].replaceAll("\\{\\{.*\\}\\}", env.IMAGE_TAG_VERSION) - env.SWF_DEVMODE_IMAGE_REGISTRY = PME_BUILD_VARIABLES['swfDevModeImageRegistry'].replaceAll("\\{\\{.*\\}\\}", env.IMAGE_TAG_VERSION) - env.OPERATOR_IMAGE_REGISTRY = PME_BUILD_VARIABLES['operatorImageRegistry'].replaceAll("\\{\\{.*\\}\\}", env.IMAGE_TAG_VERSION) - - sh 'printenv' - } - } - } - stage('Build Nightly projects') { - steps { - script { - def pipelineHelper = new PipelineHelper(this) - pipelineHelper.retry( - { - withCredentials([string(credentialsId: "kie-ci1-token", variable: 'GITHUB_TOKEN')]) { - def SETTINGS_XML_ID = '5d9884a1-178a-4d67-a3ac-9735d2df2cef' - def buildBranch = getBuildBranch() - def definitionFile = "https://raw.githubusercontent.com/\\\${GROUP:kiegroup}/kogito-pipelines/${getDefinitionFileBranch()}/.ci/nightly-build-config.yaml" - - configFileProvider([configFile(fileId: "${SETTINGS_XML_ID}", variable: 'PME_MAVEN_SETTINGS_XML')]) { - sh "build-chain build branch --token=${GITHUB_TOKEN} -f ${definitionFile} -b ${buildBranch} -o bc -p kiegroup/kogito-runtimes --fullProjectDependencyTree --skipParallelCheckout" - } - } - }, 2, 480*60) - } - } - } - stage('Upload maven repository') { - steps { - script { - echo "[INFO] Start uploading ${env.WORKSPACE}/deployDirectory" - if(fileExists("${env.WORKSPACE}/deployDirectory")){ - def destDir = "${env.WORKSPACE}/deployDirectoryFinal/openshift-serverless-logic-${PRODUCT_VERSION}-maven-repository/maven-repository/" - sh "mkdir -p ${destDir}" - sh "mv ${env.WORKSPACE}/deployDirectory/* ${destDir}" - dir("${env.WORKSPACE}/deployDirectoryFinal") { - sh "zip -r maven-repository-${env.DATE_TIME_SUFFIX} ." - def relativeFile = "rhoss/rhoss-logic-${PRODUCT_VERSION}.nightly" - def folder="${env.RCM_GUEST_FOLDER}/${relativeFile}" - - util.withKerberos('rhba-prod-keytab') { - sh "ssh 'rhba-prod@${env.RCM_HOST}' 'mkdir -p ${folder}'" - sh "rsync -rlp --info=progress2 maven-repository-${env.DATE_TIME_SUFFIX}.zip rhba-prod@${env.RCM_HOST}:staging/${relativeFile}" - } - } - } - } - } - } - stage ('Extract Git Information') { - steps { - script { - def projectFolders = sh(returnStdout: true, script: "ls ${env.WORKSPACE}/bc").trim().split("\n") - for (f in projectFolders) { - dir("${env.WORKSPACE}/bc/${f}") { - def projectName = f.replace("_", "/") - // extract git infos - util.storeGitInformation(projectName) - env.ALREADY_BUILT_PROJECTS = "${env.ALREADY_BUILT_PROJECTS ?: ''}${projectName};" - - def sanitizedProjectName = f.replaceAll('/', '_').replaceAll('-', '_') - def buildScript = env["PME_BUILD_SCRIPT_${sanitizedProjectName}"] - def pomPath = maven.getProjectPomFromBuildCmd(buildScript) - env["VERSION_${f}"] = maven.mvnGetVersionProperty(new MavenCommand(this).withProperty('productized'), 'project.version', pomPath) - } - } - - sh 'printenv' - } - } - } - stage ('Send UMB Message to QE.') { - steps { - script { - echo '[INFO] Sending OPENSHIFT SERVERLESS LOGIC UMB message to QE.' - - def mavenRepositoryFileUrl = "${env.STAGING_SERVER_URL}/rhoss/rhoss-logic-${PRODUCT_VERSION}.nightly/maven-repository-${env.DATE_TIME_SUFFIX}.zip" - def topic = "VirtualTopic.qe.ci.ba.openshift-serverless-logic.${env.UMB_VERSION}.nightly.trigger" - def eventType = "openshift-serverless-logic-${env.UMB_VERSION}-nightly-qe-trigger" - def messageBody = getMessageBody( - mavenRepositoryFileUrl, - env.ALREADY_BUILT_PROJECTS, - ['serverlesslogic': env['VERSION_kiegroup_kogito-runtimes'], 'serverlesslogic-rhba': env['VERSION_kiegroup_kogito-runtimes'], 'drools': env.VERSION_kiegroup_drools, 'platform.quarkus.bom': env.QUARKUS_PLATFORM_VERSION, 'quarkus.bom': env.QUARKUS_VERSION], - ['data-index-ephemeral': env.DATA_INDEX_IMAGE_REGISTRY, 'swf-builder': env.SWF_BUILDER_IMAGE_REGISTRY, 'swf-devmode': env.SWF_DEVMODE_IMAGE_REGISTRY, 'operator': env.OPERATOR_IMAGE_REGISTRY], - gitHashesToCollection(env.GIT_INFORMATION_HASHES) - ) - echo "[INFO] Message Body: ${messageBody}" - echo "[INFO] Topic: ${topic}" - echo "[INFO] Event Type: ${eventType}" - build job: env.SEND_UMB_MESSAGE_JOB_PATH, parameters: [ - [$class: 'StringParameterValue', name: 'MESSAGE_BODY', value: messageBody], - [$class: 'StringParameterValue', name: 'TOPIC', value: topic], - [$class: 'StringParameterValue', name: 'EVENT_TYPE', value: eventType] - ] - echo '[SUCCESS] Message was successfully sent.' - } - } - } - } - post { - failure { - emailext body: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER} was: ' + "${currentBuild.currentResult}" + '\n' + - 'Please look here: ${BUILD_URL} \n' + - ' \n' + - '${BUILD_LOG, maxLines=750}', subject: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER}: ' + "${currentBuild.currentResult}", to: 'kie-jenkins-builds@redhat.com' - } - unstable { - emailext body: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER} was: ' + "${currentBuild.currentResult}" + '\n' + - 'Please look here: ${BUILD_URL} \n' + - ' \n' + - 'Failed tests: ${BUILD_URL}/testReport \n' + - ' \n' + - '${BUILD_LOG, maxLines=750}', subject: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER}: ' + "${currentBuild.currentResult}", to: 'kie-jenkins-builds@redhat.com' - } - fixed { - emailext body: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER} was: ' + "${currentBuild.currentResult}" + '\n' + - 'Please look here: ${BUILD_URL}', subject: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER}: ' + "${currentBuild.currentResult}", to: 'kie-jenkins-builds@redhat.com' - } - always { - archiveArtifacts artifacts: '**/*.maven.log', fingerprint: false, defaultExcludes: true, caseSensitive: true, allowEmptyArchive: true - - echo 'Generating JUnit report...' - junit allowEmptyResults: true, healthScaleFactor: 1.0, testResults: '**/target/*-reports/TEST-*.xml' - - echo 'Archiving logs...' - archiveArtifacts excludes: '**/target/checkstyle.log', artifacts: '**/*.maven.log,**/target/*.log', fingerprint: false, defaultExcludes: true, caseSensitive: true, allowEmptyArchive: true - } - cleanup { - cleanWs() - } - } -} - -def getMessageBody(String mavenRepositoryFileUrl, String alreadyBuiltProjects, Map versions, Map imagesRegistries, Map scmHashes) { - def alreadyBuiltProjectsArray = (alreadyBuiltProjects ?: '').split(";") - return """ -{ - "maven_repository_file_url": "${mavenRepositoryFileUrl}", - "version": ${new groovy.json.JsonBuilder(versions).toString()}, - "image": ${new groovy.json.JsonBuilder(imagesRegistries).toString()}, - "scm_hash": ${new groovy.json.JsonBuilder(scmHashes).toString()}, - "built_projects": ${new groovy.json.JsonBuilder(alreadyBuiltProjectsArray).toString()} -}""" -} - -def setupCypressEnv(String cypressVersion) { - if (env.CYPRESS_BINARY_URL) { - env.CYPRESS_INSTALL_BINARY = "${CYPRESS_BINARY_URL}/cypress-${cypressVersion}.zip" - } -} - -def gitHashesToCollection(String gitInformationHashes) { - return gitInformationHashes.replaceAll(/([\w\d\-\_\.]*\/)([\w\d\-\_\.]*)/,'$2').split(';').findAll { it.split('=').size() }.collectEntries{ [it.split('=')[0], it.split('=')[1]] } -} - -String getBuildConfigBranch() { - return env.CONFIG_BRANCH ?: env.DEFAULT_CONFIG_BRANCH ?: env.BRANCH_NAME ?: env.GIT_BRANCH -} - -String getBuildBranch() { - // Fallback to main if none exist - return env.CHANGE_BRANCH ?: env.BRANCH_NAME ?: env.GIT_BRANCH ?: 'main' -} - -String getDroolsBranch() { - return calculateKieRepoBranch(getBuildBranch()) -} - -String getDefinitionFileBranch() { - // Fallback to getBuildBranch if DEFINITION_FILE_BRANCH not set - return env.DEFINITION_FILE_BRANCH ?: getBuildBranch() -} - -String calculateKieRepoBranch(String branch) { - /* The Drools/OptaPlanner major version is shifted by 7 from the Kogito major version: - Kogito 1.x.y -> Drools 8.x.y. - Kogito 1.x.y -> OptaPlanner 8.x.y. */ - int majorVersionShift = 7 - String [] branchSplit = branch.split("\\.") - if (branchSplit.length == 3) { - Integer optaplannerMajorVersion = Integer.parseInt(branchSplit[0]) + majorVersionShift - return "${optaplannerMajorVersion}.${branchSplit[1]}.${branchSplit[2]}" - } else { - return branch - } -} - -// Parse version from main branch of the given project -// * project: in the form of owner/repository -def parseVersionFromPom(String project, String branch) { - def pomFilename = "${project.replaceAll("/", "_")}_pom.xml" - def pomPath = "${env.WORKSPACE}/${pomFilename}" - - sh "curl https://raw.githubusercontent.com/${project}/${branch}/pom.xml -o ${pomPath}" - def pom = readMavenPom file: pomPath - return pom.getVersion().replaceAll('-SNAPSHOT', '') -} diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index c7591c866..3c00a6f59 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -23,18 +23,6 @@ environments: # ADDITIONAL_TIMEOUT: 720 # ids: # - native - # native-lts: - # env_vars: - # NATIVE: true - # BUILD_MVN_OPTS: -Dproductized -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral - # BUILD_MVN_OPTS_CURRENT: -Dnative -Dquarkus.native.container-build=true - # QUARKUS_BRANCH: '2.13' - # ADDITIONAL_TIMEOUT: 720 - # BUILD_ENVIRONMENT: quarkus-lts - # ids: - # - native - # - prod - # - lts # quarkus-main: # enabled: false # env_vars: @@ -55,15 +43,6 @@ environments: # ids: # - quarkus # - quarkus3 - # quarkus-lts: - # pull_request_default_check: false - # env_vars: - # BUILD_MVN_OPTS: -Dproductized -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral - # QUARKUS_BRANCH: '2.13' - # ids: - # - quarkus - # - lts - # - prod sonarcloud: auto_generation: false env_vars: @@ -75,7 +54,6 @@ environments: auto_generation: false ids: - ecosystem -productized_branch: true disable: triggers: false repositories: @@ -89,16 +67,6 @@ repositories: # - name: kie-tools # - name: kogito-docs # - name: kie-docs -# branch: main-kogito -productized_repositories: -- name: kogito-pipelines -- name: kogito-runtimes -- name: kogito-apps -- name: kogito-examples -- name: kogito-images -- name: kogito-serverless-operator -- name: kogito-operator -- name: kogito-docs git: author: name: kiegroup diff --git a/.ci/jenkins/config/main.yaml b/.ci/jenkins/config/main.yaml index ca3320e4e..531d6eecc 100644 --- a/.ci/jenkins/config/main.yaml +++ b/.ci/jenkins/config/main.yaml @@ -9,12 +9,10 @@ ecosystem: - kogito.* - name: kie-tools ignore_release: true - ignore_productization: true regexs: - kie-tools.* - name: kie-docs ignore_release: true - ignore_productization: true regexs: - kie-docs.* git: diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index e62ceb8f6..d115ab7a1 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -263,7 +263,7 @@ void setupReleaseCloudJob() { GIT_BRANCH_NAME: "${GIT_BRANCH}", GIT_AUTHOR: "${GIT_AUTHOR_NAME}", - IMAGE_REGISTRY_CREDENTIALS: "${CLOUD_IMAGE_REGISTRY_CREDENTIALS_RELEASE}", + IMAGE_REGISTRY_CREDENTIALS: "${CLOUD_IMAGE_REGISTRY_CREDENTIALS}", IMAGE_REGISTRY: "${CLOUD_IMAGE_REGISTRY}", IMAGE_NAMESPACE: "${CLOUD_IMAGE_NAMESPACE}", BRANCH_FOR_LATEST: "${CLOUD_IMAGE_LATEST_GIT_BRANCH}", diff --git a/.ci/nightly-build-config.yaml b/.ci/nightly-build-config.yaml deleted file mode 100644 index 9613fbb85..000000000 --- a/.ci/nightly-build-config.yaml +++ /dev/null @@ -1,61 +0,0 @@ -version: "2.1" - -dependencies: ./nightly-project-dependencies.yaml - -pre: | - export PME_CMD="java -jar ${{ env.PME_CLI_PATH }} -s ${{ env.PME_MAVEN_SETTINGS_XML }} -DallowConfigFilePrecedence=true -DprojectSrcSkip=false" - echo "PME_CMD=${{ env.PME_CMD }}" - export BUILD_MVN_OPTS="${{ env.BUILD_MVN_OPTS }} -s ${{ env.PME_MAVEN_SETTINGS_XML }}" - echo "BUILD_MVN_OPTS=${{ env.BUILD_MVN_OPTS }}" - export ALIGN_QUARKUS="mvn versions:compare-dependencies -DremotePom=io.quarkus:quarkus-bom:${{ env.QUARKUS_VERSION_COMMUNITY }} -DupdatePropertyVersions=true -DupdateDependencies=true -DgenerateBackupPoms=false -Dproductized -s ${{ env.PME_MAVEN_SETTINGS_XML }}" - echo "ALIGN_QUARKUS=${{ env.ALIGN_QUARKUS }}" - -default: - build-command: - current: mvn clean install -Dproductized=true -DskipTests=true ${{ env.BUILD_MVN_OPTS }} - upstream: mvn clean install -Dquickly -Dproductized=true -DskipTests=true ${{ env.BUILD_MVN_OPTS }} - downstream: mvn clean install -Dquickly -Dproductized=true -DskipTests=true ${{ env.BUILD_MVN_OPTS }} - after: - current: | - docker system prune -f - -build: - - project: kiegroup/drools - build-command: - upstream: | - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_drools }} - bash -c "set -o pipefail ; ${{ env.ALIGN_QUARKUS }} -pl :drools-build-parent | tee ${{ env.WORKSPACE }}/drools.quarkus-alignment.maven.log" - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_drools }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/drools.maven.log" - - - project: kiegroup/kogito-runtimes - build-command: - current: | - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_kogito_runtimes }} - bash -c "set -o pipefail ; ${{ env.ALIGN_QUARKUS }} -pl :kogito-dependencies-bom | tee ${{ env.WORKSPACE }}/kogito_runtimes.quarkus-alignment.maven.log" - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_kogito_runtimes }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/kogito_runtimes.maven.log" - - - project: kiegroup/kogito-apps - build-command: - downstream: | - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_kogito_apps }} - bash -c "set -o pipefail ; ${{ env.ALIGN_QUARKUS }} -pl :kogito-apps-build-parent | tee ${{ env.WORKSPACE }}/kogito_apps.quarkus-alignment.maven.log" - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_kogito_apps }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/kogito_apps.maven.log" - - - project: kiegroup/kogito-examples - build-command: - downstream: | - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_kogito_examples }} - bash -c "set -o pipefail ; ${{ env.ALIGN_QUARKUS }} | tee ${{ env.WORKSPACE }}/kogito_examples.quarkus-alignment.maven.log" - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_kogito_examples }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/kogito_examples.maven.log" - - - project: kiegroup/kogito-images - build-command: - downstream: | - export MAVEN_OPTIONS="-s ${{ env.PME_MAVEN_SETTINGS_XML }}" - echo "MAVEN_OPTIONS=${{ env.MAVEN_OPTIONS }}" - export NIGHTLY="true" - echo "NIGHTLY=${{ env.NIGHTLY }}" - export NIGHTLY_DEPLOY_FOLDER="${{ env.WORKSPACE }}/deployDirectory" - echo "NIGHTLY_DEPLOY_FOLDER=${{ env.NIGHTLY_DEPLOY_FOLDER }}" - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_kogito_images }} - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_kogito_images }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/kogito_images.maven.log" \ No newline at end of file diff --git a/.ci/nightly-project-dependencies.yaml b/.ci/nightly-project-dependencies.yaml deleted file mode 100644 index 93da3ce4d..000000000 --- a/.ci/nightly-project-dependencies.yaml +++ /dev/null @@ -1,82 +0,0 @@ -version: "2.1" - -dependencies: - - project: kiegroup/drools - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - - - project: kiegroup/kogito-runtimes - dependencies: - - project: kiegroup/drools - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - exclude: - - kiegroup/kogito-examples - - kiegroup/kogito-apps - - kiegroup/kogito-images - - - project: kiegroup/kogito-apps - dependencies: - - project: kiegroup/kogito-runtimes - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - exclude: - - kiegroup/kogito-examples - - kiegroup/kogito-runtimes - - kiegroup/kogito-images - - - project: kiegroup/kogito-examples - dependencies: - - project: kiegroup/kogito-runtimes - - project: kiegroup/kogito-apps - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - exclude: - - kiegroup/kogito-apps - - kiegroup/kogito-runtimes - - kiegroup/kogito-images - - - project: kiegroup/kogito-images - dependencies: - - project: kiegroup/kogito-examples - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - exclude: - - kiegroup/kogito-runtimes - - kiegroup/kogito-apps - - kiegroup/kogito-examples \ No newline at end of file diff --git a/docs/jenkins.md b/docs/jenkins.md index 33bab7278..4d3033257 100644 --- a/docs/jenkins.md +++ b/docs/jenkins.md @@ -142,8 +142,6 @@ environments: ids: - List of ids identifying the environment -productized_branch: true - # Used to force the disabling of triggers # Useful when a branch is no more maintained but you still want to keep job history disable: diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare b/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare deleted file mode 100644 index dad5b6f9f..000000000 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare +++ /dev/null @@ -1,664 +0,0 @@ -import org.jenkinsci.plugins.workflow.libs.Library -import java.util.regex.Pattern - -@Library('jenkins-pipeline-shared-libraries')_ - -// parameters -// 1. _RELEASE_BRANCH -// 2. _UPDATE_VERSION (this includes also dependencies) -// 3. QUARKUS_VERSION -// 4. DOWNGRADE_QUARKUS_PR_BRANCH -// 5. ADDITIONAL_BUILD_MAVEN_OPTS -// 6. PROJECTS_TO_REMOVE_FROM_PR_CHECKS - -seedConfig = [:] -mainBranchConfig = [:] -repositoriesToBranch = [] - -pipeline{ - agent{ - label 'rhel8 && !built-in' - } - stages{ - stage('Initialize'){ - steps{ - script { - checkout scm - - String projectListStr = getEcosystemProjects().collect { "${getRepoNameCamelCase(it.name)} ${getBranchFromRepository(it.name)}" }.join(' / ') - currentBuild.displayName = "Productize Branch for ${projectListStr}" - } - } - } - - stage('Create productized branches'){ - steps{ - script { - List repositories = getProdRepositoriesToBranch() - createBranches(repositories) - } - } - } - - // update the main seed config by adding the new prod branch, e.g., - // - name: 8.36.x-prod - // seed: - // branch: seed-drools-8.36.x // this branch MUST already exist - stage('Update main seed config') { - steps { - script { - String seedConfigFile = env.SEED_CONFIG_FILE_PATH - - dir(checkoutMainSeedConfigFileRepo()) { - def seedConfig = readYaml file: seedConfigFile - - if (!seedConfig.git.branches.find { it.name == getEcosystemProductizedBranch() }) { - def branchConfig = [ - name: getEcosystemProductizedBranch(), - seed: [ - branch: getSeedRepoReleaseBranch(), - ] - ] - - seedConfig.git.branches.add(branchConfig) - } - - echo "Write seed config => ${seedConfig}" - sh "rm ${seedConfigFile}" - writeYaml file: seedConfigFile, data: seedConfig, overwrite: true - - pushIfChanged(env.SEED_CONFIG_FILE_GIT_BRANCH, env.SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID, "Add productized branch ${getEcosystemProductizedBranch()} after upstream release", "Main config has been updated with release branch ${getEcosystemProductizedBranch()}") - } - } - } - } - - // 1. update seed config repo on productized branch (e.g., 1.37.x-prod) - // - enable productized profile (-Dproductized) on jenkins environments branch.yaml [this MUST be executed only on config file repo, e.g., kogito-pipelines] - // - disable some environments when branching as they make no sense by default [this MUST be executed only on config file repo, e.g., kogito-pipelines] - // - set productized_repositories[].branch to the productized one, e.g., 1.37.x-prod - // - set productized_repositories[].is_branch_config_repo to true for the config file repo - // - enable productized profile (-Dproductized) on build chain config - // - do these changes directly on the prod branch - - - stage("Update branch config on productized branch"){ - steps{ - script { - def prodBranch = getBranchConfigProductizedBranch() - def prodRepositories = getProdRepositoriesToBranch() - - dir(checkoutMainBranchSeedConfigFileRepo()) { - getOrCreateGitBranch(prodBranch, getMainBranchConfigFileGitAuthorCredentialsId()) - - def branchConfigFilePath = getProdBranchConfigFilePath() - def branchConfig = readYaml file: branchConfigFilePath - - branchConfig.productized_repositories = prodRepositories - branchConfig.productized_branch = true - - // keep only those environments that are really needed on productized branch: - // native, quarkus-branch and default if exists - def envsToKeep = ['native', 'quarkus-branch', 'default'] - enabledEnvs = branchConfig.environments.findAll { envName, envData -> envsToKeep.contains(envName) } - - // additional maven opt that enables productized profile - // FIXME do we need -Dproductized here as we are already adding it on buildchain config - def buildMvnOpts = "-Dproductized ${getAdditionalDefaultBuildMavenOpts()} " - - // create default env if not already existing - enabledEnvs.default = enabledEnvs.default ?: [:] - - // apply changes on remaining envs - enabledEnvs.each { envName, envData -> - // ensure env is enabled - envData.enabled = true - - envData.env_vars = envData.env_vars ?: [:] - envData.env_vars.BUILD_MVN_OPTS = buildMvnOpts + (envData.env_vars.BUILD_MVN_OPTS ?: '') - } - - // update environments - branchConfig.environments = enabledEnvs - - // ignore the job creation for jobs in environment which are not defined (apart from the default env) - branchConfig.generation_config = branchConfig.generation_config ?: [:] - branchConfig.generation_config.missing_environment = 'ignore' - - // disable community triggers - branchConfig.disable = branchConfig.disable ?: [:] - branchConfig.disable.triggers = true - - // update repositories branches to the productized one - branchConfig.repositories.each { repo -> - repo.branch = getEcosystemProductizedBranch() - } - - // add cloud.images if operating on kogito project - - branchConfig.cloud = branchConfig.cloud ?: [:] - if (getEcosystemMainProject() == 'kogito') { - branchConfig.cloud.images = 'kogito-data-index-ephemeral,kogito-swf-builder,kogito-swf-devmode' - } - - // override nexus artifacts repository - - def nexusRepo = "${env.BXMS_QE_NEXUS}/content/repositories/osl-prod-nightly/" - branchConfig.maven = branchConfig.maven ?: [:] - branchConfig.maven.artifacts_repository = nexusRepo - - branchConfig.maven.artifacts_upload_repository = branchConfig.maven.artifacts_upload_repository ?: [:] - branchConfig.maven.artifacts_upload_repository.url = nexusRepo - branchConfig.maven.artifacts_upload_repository.creds_id = 'unpacks-zip-on-qa-nexus' - - echo "Write dsl branch_config => ${branchConfig}" - sh "rm -f ${branchConfigFilePath}" - writeYaml file: "${branchConfigFilePath}", data: branchConfig, overwrite: true - - // update build-chain configuration - - def buildChainConfigFilePath = readMainBranchConfig().buildchain_config.git.file_path - def buildChainConfig = readYaml file: buildChainConfigFilePath - - // add -Dproductized in BUILD_MVN_OPTS, then we don't have to add -Dproductized inside DSL configs - - sh "sed -i 's/export BUILD_MVN_OPTS=\"\\\${{ env.BUILD_MVN_OPTS }}/export BUILD_MVN_OPTS=\"\\\${{ env.BUILD_MVN_OPTS }} -Dproductized/' ${buildChainConfigFilePath}" - - // comment out projects we are not interested in from the buildchain-config.yaml - def projectToRemoveFromBCPRChecks = getProjectsToRemoveFromBuildChainConfig() - - if (projectToRemoveFromBCPRChecks.isEmpty()) { - echo "Removing projects ${projectToRemoveFromBCPRChecks} from build chain configuration" - - def builds = buildChainConfig.build.findAll { build -> - !projectToRemoveFromBCPRChecks.contains(build.project) - } - buildChainConfig.build = builds - - echo "Write build-chain configuration => ${buildChainConfig}" - sh "rm -f ${buildChainConfigFilePath}" - writeYaml file: "${buildChainConfigFilePath}", data: buildChainConfig, overwrite: true - - // update build-chain project dependencies - def buildChainConfigDependenciesFilePath = buildChainConfigFilePath.substring(0, buildChainConfigFilePath.lastIndexOf('/') + 1) + "${buildChainConfig.dependencies}" - def buildChainConfigDependencies = readYaml file: buildChainConfigDependenciesFilePath - - def deps = buildChainConfigDependencies.dependencies.findAll { dep -> - !projectToRemoveFromBCPRChecks.contains(dep.project) - } - buildChainConfigDependencies.dependencies = deps - - echo "Write build-chain dependencies configuration => ${buildChainConfigDependencies}" - sh "rm -f ${buildChainConfigDependenciesFilePath}" - writeYaml file: "${buildChainConfigDependenciesFilePath}", data: buildChainConfigDependencies, overwrite: true - } - - pushIfChanged(prodBranch, getMainBranchConfigFileGitAuthorCredentialsId(), "Setup DSL branch config after productized branch creation", "Branch config on ${prodBranch} has been updated with correct configuration") - } - } - } - } - - - // 2. update productized branch on other projects - // - disable PR checks from jobs.groovy - // - comment out projects we are not interested in from the buildchain-config.yaml - // - disable GHA jobs no more needed by commenting out workflows - // - ensure that fullProfile is not enabled on .ci/jenkins/dsl/jobs.groovy, if so remove it - stage("Update prod branch for all projects to be productized"){ - steps{ - script { - def prodRepositories = getProdRepositoriesToBranch() - prodRepositories.each { repoConfig -> - String repository = repoConfig.name - String repoAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name - String repoCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id - - String jobsGroovyPath = getJobDefinitionFilePath() - - dir(repository) { - deleteDir() - // here productized branches should have been already created on repos to be productized - checkout(githubscm.resolveRepository(repository, repoAuthor, getProductizedBranchFromRepository(repository), false, repoCredsId)) - - getOrCreateGitBranch(repoConfig.branch, repoCredsId) - - if (fileExists("${jobsGroovyPath}")) { - // def jobsGroovy = readFile file: jobsGroovyPath - // remove fullProfile - sh "sed -i 's/, addFullProfileJobParamsGetter)/)/g' ${jobsGroovyPath}" - // TODO: disable pr checks in getMultijobPRConfig, get them from params - } else { - echo "[WARN] Unable to find jobs.groovy for '${repository}' at '${jobsGroovyPath}'" - } - - - // TODO iterate over PR workflows - // - comment out/remove if not needed (get these from param?) - - // push changes - pushIfChanged(repoConfig.branch, repoCredsId, "Pull request checks updated for productized branch", "Pull request checks updated for productized branch") - } - } - } - } - } - - // 3. update the projects version in order to avoid any kind of conflicts with already existing and deployed artifacts - - stage("Update projects version"){ - when { - expression { return getProjectUpdateVersionKeys().findAll{ it.value != '' } } - } - steps{ - script { - println "Updating projects versions" - - def prodBranch = getEcosystemProductizedBranch() - String jobName = "${prodBranch}/setup-branch/0-setup-branch" - List jobParams = [] - jobParams.add(booleanParam(name: 'DEPLOY', value: false)) - // here we could have a variable number of params - getProjectUpdateVersionKeys().each{ p -> - jobParams.add(stringParam(name: p.key.replace('_UPDATE', ''), value: p.value)) - } - - if (!isDryRun()) { - def job = [:] - try { - // Retry 20 times every 5 min in order to assure seed job ran - retry(20) { - sleep 300 - echo "Build ./${jobName} with parameters ${jobParams}" - job = build(job: "./${jobName}", parameters: jobParams, wait: true, propagate: false) - } - - if (job.result != 'SUCCESS') { - unstable("Update projects version on repositories was not successful") - } - } catch (err) { - unstable("Error executing the ${prodBranch} setup job ...") - String msg = "ERROR: Setup branch job on newly created release branch '${prodBranch}' was unsuccessful." - if(job) { - msg += "\nPlease review job ${job.absoluteUrl}..." - } - sendNotification(msg) - } - } else { - // dry run - just print the job name and params - echo "Build ./${jobName} with parameters ${jobParams}" - } - } - } - } - - // 4. call update quarkus job in order to downgrade quarkus to LTS version (version provided as parameter QUARKUS_VERSION) - // this will: - // - downgrade quarkus, there already exists a job doing this - // - apply patches from .ci/environments/quarkus-lts on the same PR - - stage('Downgrade quarkus to LTS version'){ - when { - expression { return params.QUARKUS_VERSION } - } - steps{ - script { - println "Launch update quarkus all" - - String jobName = "${getEcosystemProductizedBranch()}/tools/update-quarkus-all" - String quarkusDowngradePrBranch = getQuarkusDowngradePrBranch() - List jobParams = [] - jobParams.add(stringParam(name: 'NEW_VERSION', value: getQuarkusVersion())) - jobParams.add(stringParam(name: 'PR_BRANCH', value: quarkusDowngradePrBranch)) - - echo "Build ./${jobName} with parameters ${jobParams}" - if (!isDryRun()) { - def job = [:] - try { - // Retry 20 times every 5 min in order to assure seed job ran - retry(20) { - sleep 300 - echo "Build ./${jobName} with parameters ${jobParams}" - job = build(job: "./${jobName}", parameters: jobParams, wait: true, propagate: false) - } - - if (job.result != 'SUCCESS') { - unstable("Downgrade quarkus version on repositories was not successful") - } else { - def prodRepositories = getProdRepositoriesToBranch() - prodRepositories.each { repoConfig -> - String repoAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name - String repoCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id - String repository = repoConfig.name - String baseBranch = repoConfig.branch ?: 'main' - - dir(repository) { - deleteDir() - checkout(githubscm.resolveRepository(repository, repoAuthor, baseBranch, false, repoCredsId)) - - // if quarkusDowngradePrBranch has been created by update-quarkus-all job, checkout it - if (checkoutGitBranchIfExists(quarkusDowngradePrBranch, repoCredsId)) { - // thefollowing script takes care of applying all patches needed when downgrading to Quarkus LTS - // i.e., it will apply all patches found under .ci/environments/quarkus-lts/patches - def updateQuarkusLtsScriptPath = '.ci/environments/update.sh' - if (fileExists(updateQuarkusLtsScriptPath)) { - sh "${updateQuarkusLtsScriptPath} quarkus-lts" - } else { - echo "[WARN] Unable to find '${updateQuarkusLtsScriptPath}' for '${repository}'" - } - - pushIfChanged(quarkusDowngradePrBranch, repoCredsId, "Applied patches after Quarkus downgrade to ${getQuarkusVersion()}", "Quarkus LTS patches on ${quarkusDowngradePrBranch} have been applied") - } - } - } - } - } catch (err) { - unstable("Error executing update quarkus all...") - String msg = "ERROR: Update quarkus all job on newly created prod branch was unsuccessful." - if(job) { - msg += "\nPlease review job ${job.absoluteUrl}..." - } - sendNotification(msg) - } - } - } - } - } - } - post{ - unsuccessful { - script { - sendNotification("${getEcosystemMainProject()}: Branch Productization Pipeline job #${BUILD_NUMBER} was: ${currentBuild.currentResult}\nPlease look here: ${BUILD_URL}") - } - } - success { - script { - sendNotification("${getEcosystemMainProject()}: Branch Productization Pipeline job #${BUILD_NUMBER} was: ${currentBuild.currentResult}") - } - } - } -} - - -void sendNotification(String body) { - echo 'Send Notification' - echo body - if (!isDryRun()) { - emailext body: body, subject: "[${getEcosystemProductizedBranch()}] ${getEcosystemMainProject()}", - to: env.CI_EMAIL_TO - } -} - -// Retrieve job parameters keys or values - -String getBranchParameterKey(String projectName) { - return "${projectName}_RELEASE_BRANCH".toUpperCase() -} - -String getProdBranchSuffixParameterKey(String projectName) { - def key = "${projectName}_PROD_BRANCH_SUFFIX".toUpperCase() - return params."${key}" -} - -def getProjectUpdateVersionKeys() { - return params.findAll{ it.key.contains('_UPDATE_VERSION') } -} - -String getQuarkusVersion() { - return params.QUARKUS_VERSION -} - -String getQuarkusDowngradePrBranch() { - return params.DOWNGRADE_QUARKUS_PR_BRANCH ?: "${getEcosystemProductizedBranch()}-${getQuarkusVersion()}-downgrade" -} - -Boolean isDryRun() { - return params.DRY_RUN -} - - -def getAdditionalDefaultBuildMavenOpts() { - return params.ADDITIONAL_BUILD_MAVEN_OPTS ?: '' -} - -def getProjectsToRemoveFromBuildChainConfig() { - return params.PROJECTS_TO_REMOVE_FROM_PR_CHECKS.split(',') as List ?: [] -} - -// Retrieve seed configuration - -String checkoutMainSeedConfigFileRepo() { - assert env.SEED_CONFIG_FILE_GIT_REPOSITORY : 'Missing main seed config file Git Repository into the environment. Please check the job configuration.' - assert env.SEED_CONFIG_FILE_GIT_AUTHOR_NAME : 'Missing main seed config file Git Author Name into the environment. Please check the job configuration.' - assert env.SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID : 'Missing main seed config file Git Author Credentials ID into the environment. Please check the job configuration.' - assert env.SEED_CONFIG_FILE_GIT_BRANCH : 'Missing main seed config file Git Branch into the environment. Please check the job configuration.' - - String dirName = "seed-main-${env.SEED_CONFIG_FILE_GIT_REPOSITORY}" - dir(dirName) { - deleteDir() - checkout(githubscm.resolveRepository(env.SEED_CONFIG_FILE_GIT_REPOSITORY, env.SEED_CONFIG_FILE_GIT_AUTHOR_NAME, env.SEED_CONFIG_FILE_GIT_BRANCH, false, env.SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID)) - sh "git checkout ${env.SEED_CONFIG_FILE_GIT_BRANCH}" - } - return dirName -} - -def readSeedConfig() { - assert env.SEED_CONFIG_FILE_PATH : 'Missing main seed config file Path into the environment. Please check the job configuration.' - if (!seedConfig) { - dir(checkoutMainSeedConfigFileRepo()) { - seedConfig = readYaml(file: "${env.SEED_CONFIG_FILE_PATH}") - } - } - return seedConfig -} - -String getProdBranchConfigFilePath() { - return readSeedConfig().seed.config_file.path -} - -String getMainBranchConfigFileGitAuthorCredentialsId() { - return readSeedConfig().seed.config_file.git.author.credentials_id -} - -String checkoutMainBranchSeedConfigFileRepo() { - String dirName = "seed-branch-${getMainBranchConfigFileGitRepository()}" - dir(dirName) { - deleteDir() - checkout(githubscm.resolveRepository(getMainBranchConfigFileGitRepository(), getMainBranchConfigFileGitAuthorName(), getMainBranchConfigFileGitBranch(), false, getMainBranchConfigFileGitAuthorCredentialsId())) - sh "git checkout ${getMainBranchConfigFileGitBranch()}" - } - return dirName -} - -def readMainBranchConfig() { - if (!mainBranchConfig) { - dir(checkoutMainBranchSeedConfigFileRepo()) { - mainBranchConfig = readYaml(file: "${getProdBranchConfigFilePath()}") - } - } - return mainBranchConfig -} - -String getRepoNameCamelCase(String repo) { - List words = repo.split('-') as List - return words.collect { it.isEmpty() ? it : it.substring(0, 1).toUpperCase() + it.substring(1).toLowerCase() }.join(' ') -} - -String getJobDefinitionFilePath() { - return readMainBranchConfig().jenkins?.jobs_definition_file ?: '.ci/jenkins/dsl/jobs.groovy' -} - -// Retrieve impacted projects - -List getEcosystemProjects() { - return readSeedConfig().ecosystem.projects.findAll { ! (it.ignore_productization ? it.ignore_productization.toBoolean() : false) } -} - - -// Retrieve branch to productize starting from current repository/project - -String getProjectFromRepository(String repository) { - return getEcosystemProjects().find { project -> project.regexs?.any { Pattern.compile(it).matcher(repository).matches() } }?.name -} - -String getBranchFromProject(String projectName) { - String key = getBranchParameterKey(projectName) - assert params."${key}" : "No parameter ${key} available for project ${projectName}. Please check job configuration." - return params."${key}" -} - -String getBranchFromRepository(String repository) { - def projectName = getProjectFromRepository(repository) - if (projectName) { - return getBranchFromProject(projectName) - } else { - error "Cannot retrieve repository from repository name: ${repository}. Make sure it matches pattern from ecosystem projects in the main config" - } -} - -// Retrieve repositories to branch - -String getMainBranchConfigFileGitRepository() { - return readSeedConfig().seed.config_file.git.repository -} - -String getMainBranchConfigFileGitAuthorName() { - return readSeedConfig().seed.config_file.git.author.name -} - -String getMainBranchConfigFileGitBranch() { - return readSeedConfig().seed.config_file.git.branch -} - -List getProdRepositoriesToBranch() { - if (!repositoriesToBranch) { - repositoriesToBranch = readMainBranchConfig().productized_repositories.findAll { getProjectFromRepository(it.name) } - def branches = readSeedConfig().git.branches - if (!branches) { - error "No branches are defined in main configuration..." - } - - def releaseBranch = branches.find { it.name == getEcosystemReleaseBranch() }?.name ?: '' - if (!releaseBranch) { - error "No ${getEcosystemReleaseBranch()} branch defined into the main configuration..." - } - repositoriesToBranch.each { repoConfig -> - // set repoConfig.branch to the prod release one (e.g., 1.37.x) - input parameter - def baseBranch = getBranchFromRepository(repoConfig.name) - repoConfig.branch = repoConfig.branch ?: baseBranch ?: releaseBranch - - if (repoConfig.name == getMainBranchConfigFileGitRepository()) { - String gitAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name - if (gitAuthor == getMainBranchConfigFileGitAuthorName() && - repoConfig.branch == getEcosystemReleaseBranch()) { - // we are already going to branch it with the setup of the branch config - echo "Repository ${repoConfig.name} is branch config repository" - repoConfig.is_branch_config_repo = true - } - } - } - } - return repositoriesToBranch -} - -// Create productized branches - -String getProductizedBranchFromProject(String projectName) { - String suffix = getProdBranchSuffixParameterKey(projectName) ?: 'prod' - return "${getBranchFromProject(projectName)}-${suffix}" -} - -String getProductizedBranchFromRepository(String repository) { - def projectName = getProjectFromRepository(repository) - return getProductizedBranchFromProject(projectName) -} - -void createBranches(List repositories) { - repositories.each { repoConfig -> - String repoName = repoConfig.name - String baseBranch = repoConfig.branch - String prodBranch = getProductizedBranchFromRepository(repoConfig.name) - String gitAuthorCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id - assert gitAuthorCredsId : "Cannot find any credentials Id for git author ${repoConfig.author.name}. Please check your branch seed config file." - String gitAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name - - echo "Checkout repo ${gitAuthor}/${repoName} on branch ${baseBranch}" - dir(repoName) { - deleteDir() - checkout(githubscm.resolveRepository(repoName, gitAuthor, baseBranch, false, gitAuthorCredsId)) - getOrCreateGitBranch(prodBranch, gitAuthorCredsId) - } - // Update the branch with the new productized branch - repoConfig.branch = prodBranch - } -} - -// Update main seed config - -String getEcosystemMainProject() { - return readSeedConfig().ecosystem.main_project -} - -String getEcosystemProductizedBranch() { - return getProductizedBranchFromProject(getEcosystemMainProject()) -} - -String getEcosystemReleaseBranch() { - return getBranchFromProject(getEcosystemMainProject()) -} - -String getSeedRepoReleaseBranch() { - return readSeedConfig().git.branches.find{ it -> it.name == getEcosystemReleaseBranch() }?.seed.branch ?: "seed-${getEcosystemMainProject()}-${getEcosystemReleaseBranch()}" -} - -String getBranchConfigProductizedBranch() { - // Check if repository already branched - return (getProdRepositoriesToBranch().find { it -> it.is_branch_config_repo }?.branch) ?: "${getEcosystemMainProject()}-${getEcosystemProductizedBranch()}" -} - -// Git utilities - -void getOrCreateGitBranch(String branch, String credentialsId) { - sh 'git fetch origin' - String branchRemoteResult = sh(script: "git ls-remote origin ${branch} | wc -l", returnStdout: true).trim() - if (Integer.parseInt(branchRemoteResult) > 0) { - echo "Branch ${branch} already exist ... will not create it. Checking out !" - sh "git checkout ${branch}" - } else { - echo "Branch ${branch} does not exist ... gonna create it" - githubscm.createBranch(branch) - if (!isDryRun()) { - githubscm.pushObject('origin', branch, credentialsId) - } - } -} - -// If the provided branch exists, checkout it and return true, otherwise return false and skip checkout. -boolean checkoutGitBranchIfExists(String branch, String credentialsId) { - sh 'git fetch origin' - String branchRemoteResult = sh(script: "git ls-remote origin ${branch} | wc -l", returnStdout: true).trim() - if (Integer.parseInt(branchRemoteResult) > 0) { - echo "Branch ${branch} exist. Checking out !" - sh "git checkout ${branch}" - return true - } - - echo "Branch ${branch} does not exist... skipping checkout" - return false -} - -def pushIfChanged(String branch, String credentialsId, String commitMsg, String notificationMsg) { - if (githubscm.isThereAnyChanges()) { - sh 'git diff' - - if (!isDryRun()) { - githubscm.commitChanges(commitMsg) - githubscm.pushObject('origin', branch, credentialsId) - } - - sendNotification(notificationMsg) - } else { - println '[WARN] No changes to commit' - } -} - diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare b/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare index 0a5514e79..12e4c056d 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare @@ -54,7 +54,6 @@ pipeline { // Update branch config def branchConfig = readYaml file: branchConfigFile branchConfig.repositories = repositories - branchConfig.productized_branch = params.PRODUCTIZED_BRANCH // Disable some environments when branching as they make no sense by default branchConfig.environments.each { envName, envData -> @@ -63,9 +62,6 @@ pipeline { envData.enabled = false // Keep only quarkus-branch } } - if (envData.ids.find { it == 'lts' }) { - envData.enabled = params.PRODUCTIZED_BRANCH // LTS envs depend on param - } } echo "Write dsl branch_config => ${branchConfig}" diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main index 7745c7ea1..462044579 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main @@ -43,11 +43,9 @@ pipeline { steps { script { def releaseProjects = readSeedConfig().ecosystem.projects.findAll { ! (it.ignore_release ? it.ignore_release.toBoolean() : false) } - def productizeProjects = readSeedConfig().ecosystem.projects.findAll { ! (it.ignore_productization ? it.ignore_productization.toBoolean() : false) } def envProperties = [ JENKINS_EMAIL_CREDS_ID: readSeedConfig().jenkins.email_creds_id, RELEASE_PROJECTS: releaseProjects.collect { it.name }.join(','), - PRODUCTIZED_PROJECTS: productizeProjects.collect { it.name }.join(','), DEPENDENCY_PROJECTS: readSeedConfig().ecosystem.dependencies.collect { it.name } .join(','), SEED_REPO: "${SEED_REPO}", SEED_AUTHOR: "${SEED_AUTHOR}", diff --git a/dsl/seed/jobs/root_jobs.groovy b/dsl/seed/jobs/root_jobs.groovy index 95a21e80d..f3e3dd006 100644 --- a/dsl/seed/jobs/root_jobs.groovy +++ b/dsl/seed/jobs/root_jobs.groovy @@ -30,8 +30,6 @@ KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { stringParam("${projectName}_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on") } } - - booleanParam('PRODUCTIZED_BRANCH', false, 'Is the created branch a productized one ?') } environmentVariables { @@ -85,61 +83,3 @@ if (nonMainBranches) { } else { println 'No branches to remove ...' } - -def jobParamsProd = [ - job: [ - name: '0-prepare-productized-branch', - description: 'Prepare productized branch', - ], - git: [ - repository: Utils.getSeedRepo(this), - author: Utils.getSeedAuthor(this), - credentials: Utils.getSeedAuthorCredsId(this), - branch: Utils.getSeedBranch(this), - ], - env: [:], - jenkinsfile: 'dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare', -] - -List communityReleaseBranches = ALL_BRANCHES.split(',').findAll { it != MAIN_BRANCH_NAME && !it.endsWith('-prod') } -if (communityReleaseBranches) { - KogitoJobTemplate.createPipelineJob(this, jobParamsProd)?.with { - parameters { - PRODUCTIZED_PROJECTS.split(',').each { projectName -> - choiceParam("${projectName}_RELEASE_BRANCH".toUpperCase(), communityReleaseBranches, "${Utils.getRepoNameCamelCase(projectName)} community branch to which to create the productized branch from") - stringParam("${projectName}_PROD_BRANCH_SUFFIX".toUpperCase(), 'prod', "${Utils.getRepoNameCamelCase(projectName)} productized branch suffix") - stringParam("${projectName}_UPDATE_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on") - } - - if (DEPENDENCY_PROJECTS) { - DEPENDENCY_PROJECTS.split(',').each { projectName -> - stringParam("${projectName}_UPDATE_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on") - } - } - - stringParam('QUARKUS_VERSION', '', 'Quarkus version to which to update all productized branches, usually latest LTS version') - stringParam('DOWNGRADE_QUARKUS_PR_BRANCH', '', 'Which PR branch name to use for Quarkus downgrade? If none given, a name will be generated automatically.') - stringParam('ADDITIONAL_BUILD_MAVEN_OPTS', '', 'Additional default maven opts for jenkins jobs, e.g., -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral') - stringParam('PROJECTS_TO_REMOVE_FROM_PR_CHECKS', '', 'Comma-separated list of projects (/) to be removed/disabled from build chain pull request config') - - booleanParam('DRY_RUN', false, 'If enabled no changes will be applied to remote branches') - } - - environmentVariables { - env('JENKINS_EMAIL_CREDS_ID', Utils.getJenkinsEmailCredsId(this)) - - env('SEED_CONFIG_FILE_GIT_REPOSITORY', "${SEED_CONFIG_FILE_GIT_REPOSITORY}") - env('SEED_CONFIG_FILE_GIT_AUTHOR_NAME', "${SEED_CONFIG_FILE_GIT_AUTHOR_NAME}") - env('SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID', "${SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID}") - env('SEED_CONFIG_FILE_GIT_BRANCH', "${SEED_CONFIG_FILE_GIT_BRANCH}") - env('SEED_CONFIG_FILE_PATH', "${SEED_CONFIG_FILE_PATH}") - - env('SEED_REPO', Utils.getSeedRepo(this)) - env('SEED_AUTHOR', Utils.getSeedAuthor(this)) - env('SEED_BRANCH', Utils.getSeedBranch(this)) - env('SEED_CREDENTIALS_ID', Utils.getSeedAuthorCredsId(this)) - } - } -} else { - println 'No branches to productize ...' -} \ No newline at end of file diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy index 9514b2bd9..a7598445d 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy @@ -48,10 +48,6 @@ class Utils { static boolean hasBindingValuesStartingWith(def script, String keyPrefix) { return getBindingValuesStartingWith(script, keyPrefix).size() > 0 } - - static boolean isProductizedBranch(def script) { - return getBindingValue(script, 'PRODUCTIZED_BRANCH').toBoolean() - } static String getGenerationBranch(def script) { return getBindingValue(script, 'GENERATION_BRANCH') diff --git a/dsl/seed/src/test/groovy/org/kie/jenkins/jobdsl/JobScriptsSpec.groovy b/dsl/seed/src/test/groovy/org/kie/jenkins/jobdsl/JobScriptsSpec.groovy index 555ab0e31..0e7d74138 100644 --- a/dsl/seed/src/test/groovy/org/kie/jenkins/jobdsl/JobScriptsSpec.groovy +++ b/dsl/seed/src/test/groovy/org/kie/jenkins/jobdsl/JobScriptsSpec.groovy @@ -44,7 +44,6 @@ class JobScriptsSpec extends Specification { envVars.put('GENERATION_BRANCH', 'GENERATION_BRANCH') envVars.put('RELEASE_PROJECTS', 'RELEASE_PROJECTS') - envVars.put('PRODUCTIZED_PROJECTS', 'PRODUCTIZED_PROJECTS') envVars.put('DEPENDENCY_PROJECTS', 'DEPENDENCY_PROJECTS') envVars.put('REPO_NAME', 'REPO_NAME') envVars.put('GIT_MAIN_BRANCH', 'GIT_MAIN_BRANCH') From 9100adbf24779f78de46410d48cf928e9ab4edfa Mon Sep 17 00:00:00 2001 From: radtriste Date: Tue, 29 Aug 2023 16:28:24 +0200 Subject: [PATCH 09/22] added Kogito artifacts repositories --- .ci/jenkins/config/branch.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index 3c00a6f59..6e0e438e5 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -58,9 +58,9 @@ disable: triggers: false repositories: - name: kogito-pipelines -# - name: kogito-runtimes -# - name: kogito-apps -# - name: kogito-examples +- name: kogito-runtimes +- name: kogito-apps +- name: kogito-examples # - name: kogito-images # - name: kogito-operator # - name: kogito-serverless-operator From f353bcd9cb455ca9094c4dd3ed4b297c8fa9c370 Mon Sep 17 00:00:00 2001 From: Tristan Radisson Date: Thu, 31 Aug 2023 16:12:03 +0200 Subject: [PATCH 10/22] Setup PR check script (#1061) --- dsl/scripts/pr_check.groovy | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 dsl/scripts/pr_check.groovy diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy new file mode 100644 index 000000000..55d41a035 --- /dev/null +++ b/dsl/scripts/pr_check.groovy @@ -0,0 +1,91 @@ +import org.kie.jenkins.MavenCommand +import org.kie.jenkins.MavenSettingsUtils + +void launch() { + stage('Initialize') { + sh 'printenv > env_props' + archiveArtifacts artifacts: 'env_props' + } + stage('check space before build') { + try { + util.spaceLeft() + } catch (err) { + echo "Error when checking the space on node ... ${err}" + } + } + stage('Install build-chain tool') { + println '[INFO] Getting build-chain version from composite action file' + def buildChainVersion = buildChain.getBuildChainVersionFromCompositeActionFile() + if ([null, 'null'].contains(buildChainVersion)) { + def errorMessage = "[ERROR] The build-chain version can't be recovered. Please contact administrator" + println errorMessage + error(errorMessage) + } + println "[INFO] build-chain version recovered '${buildChainVersion}'" + sh "npm install -g @kie/build-chain-action@${buildChainVersion}${env.NPM_REGISTRY_URL ? " -registry=${NPM_REGISTRY_URL}" : ''}" + + sh 'npm list -g | grep build-chain' + } + stage('Build projects') { + env.BUILD_MVN_OPTS_CURRENT = "${env.BUILD_MVN_OPTS_CURRENT ?: ''} ${getBuildMavenOptsCurrent()}" + echo "BUILD_MVN_OPTS_CURRENT = ${BUILD_MVN_OPTS_CURRENT}" + + configFileProvider([configFile(fileId: 'kogito_pr_settings', variable: 'MAVEN_SETTINGS_FILE')]) { // TODO as env ? + withCredentials([string(credentialsId: "kie-ci3-token", variable: 'GITHUB_TOKEN')]) { // TODO as env ? + env.BUILD_MVN_OPTS = "${env.BUILD_MVN_OPTS ?: ''} -s ${MAVEN_SETTINGS_FILE} -Dmaven.wagon.http.ssl.insecure=true -Dmaven.test.failure.ignore=true" + echo "BUILD_MVN_OPTS = ${BUILD_MVN_OPTS}" + + try { + util.runWithPythonVirtualEnv("${getBuildChainCommandline()}", 'swf') + } catch (err) { + echo 'Error running the build-chain ...' + util.archiveConsoleLog('', 300) + } finally { + // Remove `node_modules` to avoid heap space issues with junit command thereafter + // Related to https://github.com/jenkinsci/junit-plugin/issues/478 and https://github.com/jenkinsci/junit-plugin/issues/467 + sh 'find . -type d -name node_modules -exec rm -rf {} \\; || true' + + junit(testResults: '**/junit.xml, **/target/surefire-reports/**/*.xml, **/target/failsafe-reports/**/*.xml, **/target/invoker-reports/**/*.xml', allowEmptyResults: true) + archiveArtifacts(artifacts: '**/cypress/screenshots/**,**/cypress/videos/**', fingerprint: false, allowEmptyArchive: true) + } + } + } + } + stage('Sonar analysis') { + if (isEnableSonarCloudAnalysis()) { + dir(getProjectFolder()) { + maven.runMavenWithSettingsSonar(settingsXmlId, "-e -nsu validate -Psonarcloud-analysis -Denforcer.skip=true ${env.SONARCLOUD_ANALYSIS_MVN_OPTS ?: ''}", 'SONARCLOUD_TOKEN', 'sonar_analysis.maven.log') + } + } + } + stage('check space after build') { + util.spaceLeft() + } +} + +String getBuildChainCommandline() { + // TODO those should be parametrized + String buildChainProject = 'kogito-pipelines' + String buildChainConfigRepo = env.BUILDCHAIN_CONFIG_REPO ?: 'kogito-pipelines' + String buildChainConfigBranch = env.BUILDCHAIN_CONFIG_BRANCH ?: '\${BRANCH:main}' + String buildChainConfigGitAuthor = env.BUILDCHAIN_CONFIG_AUTHOR ?: '\${AUTHOR:kiegroup}' + String buildChainConfigDefinitionFilePath = env.BUILDCHAIN_CONFIG_FILE_PATH ?: '.ci/pull-request-config.yaml' + + List buildChainAdditionalArguments = [ + "-p ${buildChainProject}", + "-u ${CHANGE_URL}", // Provided by source branch plugin + ] + return "build-chain build cross_pr --token ${GITHUB_TOKEN} -f 'https://raw.githubusercontent.com/${buildChainConfigGitAuthor}/${buildChainConfigRepo}/${buildChainConfigBranch}/${buildChainConfigDefinitionFilePath}' -o 'bc' ${buildChainAdditionalArguments.join(' ')} --skipParallelCheckout" +} + +String getBuildMavenOptsCurrent() { + List opts_current = [] + isEnableSonarCloudAnalysis() ? opts_current.add('-Prun-code-coverage') : null + return opts_current.join(' ') +} + +boolean isEnableSonarCloudAnalysis() { + return env.ENABLE_SONARCLOUD ? env.ENABLE_SONARCLOUD.toBoolean() : false +} + +return this From 84ac86911dca20e3af90715c5a27bf166422ac04 Mon Sep 17 00:00:00 2001 From: radtriste Date: Thu, 31 Aug 2023 16:13:09 +0200 Subject: [PATCH 11/22] updated pr check script --- dsl/scripts/pr_check.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index 55d41a035..3594ef482 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -40,6 +40,7 @@ void launch() { } catch (err) { echo 'Error running the build-chain ...' util.archiveConsoleLog('', 300) + throw err } finally { // Remove `node_modules` to avoid heap space issues with junit command thereafter // Related to https://github.com/jenkinsci/junit-plugin/issues/478 and https://github.com/jenkinsci/junit-plugin/issues/467 From 7dd56c7bbf330b51deeef111964707b961749992 Mon Sep 17 00:00:00 2001 From: radtriste Date: Thu, 31 Aug 2023 16:40:31 +0200 Subject: [PATCH 12/22] updated pr check script --- dsl/scripts/pr_check.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index 3594ef482..7084178ce 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -66,7 +66,7 @@ void launch() { String getBuildChainCommandline() { // TODO those should be parametrized - String buildChainProject = 'kogito-pipelines' + String buildChainProject = env.BUILDCHAIN_PROJECT ?: CHANGE_REPO String buildChainConfigRepo = env.BUILDCHAIN_CONFIG_REPO ?: 'kogito-pipelines' String buildChainConfigBranch = env.BUILDCHAIN_CONFIG_BRANCH ?: '\${BRANCH:main}' String buildChainConfigGitAuthor = env.BUILDCHAIN_CONFIG_AUTHOR ?: '\${AUTHOR:kiegroup}' From 3e9699ba6fe120ed903aa4b93eab4f6f9b669ed0 Mon Sep 17 00:00:00 2001 From: radtriste Date: Thu, 31 Aug 2023 20:56:24 +0200 Subject: [PATCH 13/22] PR check script uses --- dsl/scripts/pr_check.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index 7084178ce..f152d5782 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -2,6 +2,12 @@ import org.kie.jenkins.MavenCommand import org.kie.jenkins.MavenSettingsUtils void launch() { + docker.image('quay.io/jan_stastny/kogito-ci-build:0.0.0-test2').inside { + launchStages() + } +} + +void launchStages() { stage('Initialize') { sh 'printenv > env_props' archiveArtifacts artifacts: 'env_props' @@ -31,7 +37,7 @@ void launch() { echo "BUILD_MVN_OPTS_CURRENT = ${BUILD_MVN_OPTS_CURRENT}" configFileProvider([configFile(fileId: 'kogito_pr_settings', variable: 'MAVEN_SETTINGS_FILE')]) { // TODO as env ? - withCredentials([string(credentialsId: "kie-ci3-token", variable: 'GITHUB_TOKEN')]) { // TODO as env ? + withCredentials([string(credentialsId: 'kie-ci3-token', variable: 'GITHUB_TOKEN')]) { // TODO as env ? env.BUILD_MVN_OPTS = "${env.BUILD_MVN_OPTS ?: ''} -s ${MAVEN_SETTINGS_FILE} -Dmaven.wagon.http.ssl.insecure=true -Dmaven.test.failure.ignore=true" echo "BUILD_MVN_OPTS = ${BUILD_MVN_OPTS}" From 9e38ae144ba42386ca4e2e9502020a7931a85ed9 Mon Sep 17 00:00:00 2001 From: radtriste Date: Thu, 31 Aug 2023 21:41:05 +0200 Subject: [PATCH 14/22] update pr check script --- dsl/scripts/pr_check.groovy | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index f152d5782..d3fbfd4e0 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -3,7 +3,14 @@ import org.kie.jenkins.MavenSettingsUtils void launch() { docker.image('quay.io/jan_stastny/kogito-ci-build:0.0.0-test2').inside { - launchStages() + try { + launchStages() + } finally { + if (currentBuild.currentResult != 'SUCCESS') { + // TODO ci token as env ? + pullrequest.postComment(util.getMarkdownTestSummary(notificationJobName, getReproducer(true), "${BUILD_URL}", 'GITHUB'), "kie-ci3-token") + } + } } } @@ -11,6 +18,9 @@ void launchStages() { stage('Initialize') { sh 'printenv > env_props' archiveArtifacts artifacts: 'env_props' + + // TODO temporary solution to install npm + } stage('check space before build') { try { @@ -95,4 +105,29 @@ boolean isEnableSonarCloudAnalysis() { return env.ENABLE_SONARCLOUD ? env.ENABLE_SONARCLOUD.toBoolean() : false } +String getReproducer(boolean isGH = false) { + String reproducer = """ +${env.QUARKUS_BRANCH ? "export QUARKUS_BRANCH=${env.QUARKUS_BRANCH}" : ''} +${env.BUILD_MVN_OPTS_CURRENT ? "export BUILD_MVN_OPTS_CURRENT=${env.BUILD_MVN_OPTS_CURRENT}" : ''} +${getBuildChainCommandline()} + +NOTE: To install the build-chain tool, please refer to https://github.com/kiegroup/github-action-build-chain#local-execution +""" + + if(isGH) { + return """ +
+Reproducer +${reproducer} +
+""" + } else { + return """ +```spoiler Reproducer +${reproducer} +``` +""" + } +} + return this From 7ae3bdb05943596363a4f7fb572e507d94dbceb5 Mon Sep 17 00:00:00 2001 From: radtriste Date: Thu, 31 Aug 2023 21:41:55 +0200 Subject: [PATCH 15/22] temp solution --- dsl/scripts/pr_check.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index d3fbfd4e0..09d2dff19 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -3,6 +3,10 @@ import org.kie.jenkins.MavenSettingsUtils void launch() { docker.image('quay.io/jan_stastny/kogito-ci-build:0.0.0-test2').inside { + // TODO temporary solution until image is rebuilt + sh 'sudo alternatives --install /usr/local/bin/npm npm /home/nonrootuser/.nvm/versions/node/v16.20.0/bin/npm 1' + sh 'npm --version' + try { launchStages() } finally { From 1b7ae05495a57619f5cc15b4fa0d51af4b08de0d Mon Sep 17 00:00:00 2001 From: radtriste Date: Thu, 31 Aug 2023 21:48:19 +0200 Subject: [PATCH 16/22] small update --- dsl/scripts/pr_check.groovy | 3 --- 1 file changed, 3 deletions(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index 09d2dff19..237c2d55d 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -1,6 +1,3 @@ -import org.kie.jenkins.MavenCommand -import org.kie.jenkins.MavenSettingsUtils - void launch() { docker.image('quay.io/jan_stastny/kogito-ci-build:0.0.0-test2').inside { // TODO temporary solution until image is rebuilt From cfff3ec9cd4fb82a5194087832e01db5aef9a9df Mon Sep 17 00:00:00 2001 From: radtriste Date: Thu, 31 Aug 2023 21:59:34 +0200 Subject: [PATCH 17/22] added build-ci image --- apache-nodes/Dockerfile.kogito-ci-build | 64 +++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 apache-nodes/Dockerfile.kogito-ci-build diff --git a/apache-nodes/Dockerfile.kogito-ci-build b/apache-nodes/Dockerfile.kogito-ci-build new file mode 100644 index 000000000..43dabb744 --- /dev/null +++ b/apache-nodes/Dockerfile.kogito-ci-build @@ -0,0 +1,64 @@ +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.2 + +SHELL ["/bin/bash", "-c"] + +ARG SDKMAN_JAVA="17.0.8-tem" +ARG SDKMAN_MAVEN="3.8.7" + +RUN microdnf -y update && microdnf install -y \ +# skdman deps (BEGIN) +findutils \ +tar \ +gzip \ +unzip \ +zip \ +# skdman deps (END) +# python3 (BEGIN) +python3.11 \ +python3.11-devel \ +python3.11-pip \ +python3-gssapi \ +krb5-devel \ +gcc \ +# python3 (END) +# system (BEGIN) +shadow-utils \ +sudo \ +wget \ +which \ +# system (END) +&& microdnf clean all + +RUN groupadd -g 910 nonrootuser && useradd -u 910 -g 910 nonrootuser && \ + echo "nonrootuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +USER nonrootuser + +RUN curl -s "https://get.sdkman.io" | bash && \ + source "$HOME/.sdkman/bin/sdkman-init.sh" && \ + sdk install java ${SDKMAN_JAVA} && \ + sudo alternatives --install /usr/local/bin/java java $(which java) 1 && \ + sdk install maven ${SDKMAN_MAVEN} && \ + sudo alternatives --install /usr/local/bin/mvn mvn $(which mvn) 1 && \ + sdk flush + +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash && \ + source ~/.bashrc && nvm install "v16.20.0" && \ + sudo alternatives --install /usr/local/bin/node node $(which node) 1 \ + sudo alternatives --install /usr/local/bin/npm npm $(which npm) 1 + +RUN wget https://go.dev/dl/go1.19.10.linux-amd64.tar.gz -P /tmp && \ + sudo mkdir /opt/golang && \ + sudo tar -C /opt/golang -xzf /tmp/go*.tar.gz && \ + sudo chown -R nonrootuser:nonrootuser /opt/golang/go && \ + sudo chmod -R 755 /opt/golang/go && \ + echo 'export GOPATH=${HOME}/go' | sudo tee /etc/profile.d/go.sh && \ + echo "source /etc/profile.d/go.sh" >> $HOME/.bashrc && \ + rm -rf /tmp/go* && \ + sudo alternatives --install /usr/local/bin/go go /opt/golang/go/bin/go 1 + +RUN pip3.11 install cekit==4.8.0 behave lxml docker docker-squash elementPath pyyaml ruamel.yaml python-dateutil Jinja2 pykwalify colorlog click + +ENV NODE_OPTIONS=--max_old_space_size=4096 +ENV HOME=/home/nonrootuser/ + +WORKDIR /project/directory \ No newline at end of file From 2d2972a342ab05cee0e82c40215020e93433938e Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 1 Sep 2023 09:26:04 +0200 Subject: [PATCH 18/22] update --- dsl/scripts/pr_check.groovy | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index 237c2d55d..8549f6d6e 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -19,16 +19,6 @@ void launchStages() { stage('Initialize') { sh 'printenv > env_props' archiveArtifacts artifacts: 'env_props' - - // TODO temporary solution to install npm - - } - stage('check space before build') { - try { - util.spaceLeft() - } catch (err) { - echo "Error when checking the space on node ... ${err}" - } } stage('Install build-chain tool') { println '[INFO] Getting build-chain version from composite action file' @@ -76,13 +66,10 @@ void launchStages() { } } } - stage('check space after build') { - util.spaceLeft() - } } String getBuildChainCommandline() { - // TODO those should be parametrized + // Those can be overriden in Jenkinsfiles String buildChainProject = env.BUILDCHAIN_PROJECT ?: CHANGE_REPO String buildChainConfigRepo = env.BUILDCHAIN_CONFIG_REPO ?: 'kogito-pipelines' String buildChainConfigBranch = env.BUILDCHAIN_CONFIG_BRANCH ?: '\${BRANCH:main}' From f5a1c3366429ee8f5bcae85dc19c009bb21b0495 Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 1 Sep 2023 09:28:25 +0200 Subject: [PATCH 19/22] updated config --- .ci/jenkins/config/branch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index 6e0e438e5..f04fe799c 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -55,7 +55,7 @@ environments: ids: - ecosystem disable: - triggers: false + triggers: true # TODO to set back repositories: - name: kogito-pipelines - name: kogito-runtimes From 935a559b1e26d7220ad9003568f1e380fdb19ed1 Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 1 Sep 2023 09:57:17 +0200 Subject: [PATCH 20/22] update pr check script --- dsl/scripts/pr_check.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index 8549f6d6e..7400d5c85 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -7,6 +7,7 @@ void launch() { try { launchStages() } finally { + echo "Got build result ${currentBuild.currentResult}" if (currentBuild.currentResult != 'SUCCESS') { // TODO ci token as env ? pullrequest.postComment(util.getMarkdownTestSummary(notificationJobName, getReproducer(true), "${BUILD_URL}", 'GITHUB'), "kie-ci3-token") @@ -37,7 +38,7 @@ void launchStages() { env.BUILD_MVN_OPTS_CURRENT = "${env.BUILD_MVN_OPTS_CURRENT ?: ''} ${getBuildMavenOptsCurrent()}" echo "BUILD_MVN_OPTS_CURRENT = ${BUILD_MVN_OPTS_CURRENT}" - configFileProvider([configFile(fileId: 'kogito_pr_settings', variable: 'MAVEN_SETTINGS_FILE')]) { // TODO as env ? + configFileProvider([configFile(fileId: 'kogito-pr-settings', variable: 'MAVEN_SETTINGS_FILE')]) { // TODO as env ? withCredentials([string(credentialsId: 'kie-ci3-token', variable: 'GITHUB_TOKEN')]) { // TODO as env ? env.BUILD_MVN_OPTS = "${env.BUILD_MVN_OPTS ?: ''} -s ${MAVEN_SETTINGS_FILE} -Dmaven.wagon.http.ssl.insecure=true -Dmaven.test.failure.ignore=true" echo "BUILD_MVN_OPTS = ${BUILD_MVN_OPTS}" From 8b76fef826d1513f55d9e79f7b49bd75f8b0f6ba Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 1 Sep 2023 10:06:23 +0200 Subject: [PATCH 21/22] update pr check script --- dsl/scripts/pr_check.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index 7400d5c85..ceb0a5cc9 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -38,7 +38,7 @@ void launchStages() { env.BUILD_MVN_OPTS_CURRENT = "${env.BUILD_MVN_OPTS_CURRENT ?: ''} ${getBuildMavenOptsCurrent()}" echo "BUILD_MVN_OPTS_CURRENT = ${BUILD_MVN_OPTS_CURRENT}" - configFileProvider([configFile(fileId: 'kogito-pr-settings', variable: 'MAVEN_SETTINGS_FILE')]) { // TODO as env ? + configFileProvider([configFile(fileId: 'kie-pr-settings', variable: 'MAVEN_SETTINGS_FILE')]) { // TODO as env ? withCredentials([string(credentialsId: 'kie-ci3-token', variable: 'GITHUB_TOKEN')]) { // TODO as env ? env.BUILD_MVN_OPTS = "${env.BUILD_MVN_OPTS ?: ''} -s ${MAVEN_SETTINGS_FILE} -Dmaven.wagon.http.ssl.insecure=true -Dmaven.test.failure.ignore=true" echo "BUILD_MVN_OPTS = ${BUILD_MVN_OPTS}" From 9f1d34b591b0aab4e29ac000db650aae37f61d74 Mon Sep 17 00:00:00 2001 From: radtriste Date: Fri, 1 Sep 2023 10:31:12 +0200 Subject: [PATCH 22/22] update PR check script --- dsl/scripts/pr_check.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index ceb0a5cc9..4a726d863 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -44,7 +44,7 @@ void launchStages() { echo "BUILD_MVN_OPTS = ${BUILD_MVN_OPTS}" try { - util.runWithPythonVirtualEnv("${getBuildChainCommandline()}", 'swf') + sh getBuildChainCommandline() } catch (err) { echo 'Error running the build-chain ...' util.archiveConsoleLog('', 300)