From d4e1b0d111d5fac74db97adee5adea06fee9ac22 Mon Sep 17 00:00:00 2001 From: mingshewhe Date: Tue, 24 Sep 2024 21:31:06 +0800 Subject: [PATCH] =?UTF-8?q?feat=EF=BC=9A=E6=B5=81=E6=B0=B4=E7=BA=BF?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E8=AF=AD=E6=B3=95=E6=94=AF=E6=8C=81=E4=B8=A4?= =?UTF-8?q?=E7=A7=8D=E9=A3=8E=E6=A0=BC=20#10576?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/util/ObjectReplaceEnvVarUtilTest.kt | 5 - .../pipeline/ExprReplaceEnvVarUtil.java | 159 +++++++++ .../common/pipeline/EnvReplacementParser.kt | 323 ++--------------- ...ntContext.kt => ExprReplacementOptions.kt} | 28 +- .../pipeline/utils/ExprReplacementUtil.kt | 325 ++++++++++++++++++ .../pipeline/EnvReplacementParserTest.kt | 122 ++++--- .../pipeline/ExprReplaceEnvVarUtilTest.kt | 319 +++++++++++++++++ .../tencent/devops/worker/common/Runner.kt | 7 +- .../common/task/market/MarketAtomTask.kt | 2 +- 9 files changed, 909 insertions(+), 381 deletions(-) create mode 100644 src/backend/ci/core/common/common-pipeline/src/main/java/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtil.java rename src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/{EnvReplacementContext.kt => ExprReplacementOptions.kt} (71%) create mode 100644 src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/ExprReplacementUtil.kt create mode 100644 src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtilTest.kt diff --git a/src/backend/ci/core/common/common-api/src/test/kotlin/com/tencent/devops/common/api/util/ObjectReplaceEnvVarUtilTest.kt b/src/backend/ci/core/common/common-api/src/test/kotlin/com/tencent/devops/common/api/util/ObjectReplaceEnvVarUtilTest.kt index e6e5b43a719..28ab9510ee9 100644 --- a/src/backend/ci/core/common/common-api/src/test/kotlin/com/tencent/devops/common/api/util/ObjectReplaceEnvVarUtilTest.kt +++ b/src/backend/ci/core/common/common-api/src/test/kotlin/com/tencent/devops/common/api/util/ObjectReplaceEnvVarUtilTest.kt @@ -194,11 +194,6 @@ class ObjectReplaceEnvVarUtilTest { originSubDataMapObj = cpb["originSubDataMapObj"] as MutableMap? // 判断嵌套的map中jsonStrEnvVarKey2对应的值进行变量替换后能否正常转换为json串 assertEquals(envMap["jsonStrEnvVar"], originSubDataMapObj!!["jsonStrEnvVarKey2"]!!) - - assertEquals( - toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(toJson(originDataMapObj), envMap)), - toJson(ObjectReplaceEnvVarUtil.replaceEnvVar(originDataMapObj, envMap)) - ) } @Test diff --git a/src/backend/ci/core/common/common-pipeline/src/main/java/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtil.java b/src/backend/ci/core/common/common-pipeline/src/main/java/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtil.java new file mode 100644 index 00000000000..8f68673917b --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/java/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtil.java @@ -0,0 +1,159 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline; + +import com.tencent.devops.common.api.util.JsonSchemaUtil; +import com.tencent.devops.common.api.util.JsonUtil; +import com.tencent.devops.common.api.util.ReflectUtil; +import com.tencent.devops.common.pipeline.utils.ExprReplacementUtil; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExprReplaceEnvVarUtil { + + public static Object replaceEnvVar(Object obj, Map envMap) { + return replaceEnvVar(obj, + new ExprReplacementOptions(envMap, false, null, null, null) + ); + } + + /** + * 把对象字段值中的表达式替换成环境变量 + * @param obj 需要把占位符替换环境变量的对象(对象如果是集合对象，注意要选择支持增加、删除等操作的集合类型，不要选择类似SingletonMap这种) + * @param envMap 环境变量Map + * @return 变量替换后的对象 + */ + @SuppressWarnings("all") + public static Object replaceEnvVar(Object obj, ExprReplacementOptions options) { + Map envMap = options.getContextMap(); + if (obj instanceof Map) { + // 递归替换map对象中的变量 + Set> entrySet = ((Map) obj).entrySet(); + for (Map.Entry entry : entrySet) { + Object value = entry.getValue(); + if (!isNormalReplaceEnvVar(value)) { + entry.setValue(replaceEnvVar(value, options)); + } else { + entry.setValue(handleNormalEnvVar(value, options)); + } + } + } else if (obj instanceof List) { + // 递归替换list对象中的变量 + List dataList = (List) obj; + for (int i = 0; i < dataList.size(); i++) { + Object value = dataList.get(i); + if (!isNormalReplaceEnvVar(value)) { + dataList.set(i, replaceEnvVar(value, options)); + } else { + dataList.set(i, handleNormalEnvVar(value, options)); + } + } + } else if (obj instanceof Set) { + // 递归替换set对象中的变量 + Set objSet = (Set) obj; + Set replaceObjSet = new HashSet(objSet); + Iterator it = replaceObjSet.iterator(); + while (it.hasNext()) { + Object value = it.next(); + objSet.remove(value); + if (!isNormalReplaceEnvVar(value)) { + objSet.add(replaceEnvVar(value, options)); + } else { + objSet.add(handleNormalEnvVar(value, options)); + } + } + } else if (isNormalReplaceEnvVar(obj)) { + // 替换基本类型对象或字符串对象中的变量 + obj = handleNormalEnvVar(obj, options); + } else { + try { + // 把对象转换成map后进行递归替换变量 + Map dataMap = JsonUtil.INSTANCE.toMap(obj); + replaceEnvVar(dataMap, options); + obj = JsonUtil.INSTANCE.to(JsonUtil.INSTANCE.toJson(dataMap, true), obj.getClass()); + } catch (Throwable e) { + // 转换不了map的对象则进行直接替换 + obj = ExprReplacementUtil.INSTANCE.parseExpression( + JsonUtil.INSTANCE.toJson(obj, true), options + ); + } + } + return obj; + } + + private static Object handleNormalEnvVar(Object obj, ExprReplacementOptions options) { + // 只有字符串参数才需要进行变量替换，其它基本类型参数无需进行变量替换 + if (obj instanceof String) { + String objStr = ((String) obj).trim(); + if (objStr.startsWith("{") && objStr.endsWith("}") && JsonSchemaUtil.INSTANCE.validateJson(objStr)) { + try { + Object dataObj = JsonUtil.INSTANCE.to((String) obj, Map.class); + // string能正常转换成map，则说明是json串，那么把dataObj进行递归替换变量后再转成json串 + dataObj = replaceEnvVar(dataObj, options); + obj = JsonUtil.INSTANCE.toJson(dataObj, true); + } catch (Throwable e) { + // 转换不了map的字符串对象则直接替换 + obj = ExprReplacementUtil.INSTANCE.parseExpression( + JsonUtil.INSTANCE.toJson(obj, true), options + ); + } + } else if (objStr.startsWith("[") && objStr.endsWith("]") && JsonSchemaUtil.INSTANCE.validateJson(objStr)) { + try { + Object dataObj = JsonUtil.INSTANCE.to((String) obj, List.class); + // string能正常转成list，说明是json串，把dataObj进行递归替换变量后再转成json串 + dataObj = replaceEnvVar(dataObj, options); + obj = JsonUtil.INSTANCE.toJson(dataObj, true); + } catch (Throwable e1) { + // 转换不了list的字符串对象则直接替换 + obj = ExprReplacementUtil.INSTANCE.parseExpression( + JsonUtil.INSTANCE.toJson(obj, true), options + ); + } + } else { + // 转换不了map或者list的字符串对象则直接替换 + obj = ExprReplacementUtil.INSTANCE.parseExpression( + JsonUtil.INSTANCE.toJson(obj, true), options + ); + } + } + return obj; + } + + /** + * 判断对象是否是普通替换对象 + * @param obj 需要把占位符替换环境变量的对象(对象如果是集合对象，注意要选择支持增加、删除等操作的集合类型，不要选择类似SingletonMap这种) + * @return 是否是普通替换对象 + */ + private static Boolean isNormalReplaceEnvVar(Object obj) { + return obj == null || ReflectUtil.INSTANCE.isNativeType(obj) || obj instanceof String; + } +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt index 731df6dbefc..4bdb0337c9d 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt @@ -27,28 +27,16 @@ package com.tencent.devops.common.pipeline -import com.tencent.devops.common.api.exception.VariableNotFoundException import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.ObjectReplaceEnvVarUtil -import com.tencent.devops.common.expression.ContextNotFoundException import com.tencent.devops.common.expression.ExecutionContext -import com.tencent.devops.common.expression.ExpressionParseException -import com.tencent.devops.common.expression.ExpressionParser -import com.tencent.devops.common.expression.context.ContextValueNode -import com.tencent.devops.common.expression.context.DictionaryContextData -import com.tencent.devops.common.expression.context.PipelineContextData -import com.tencent.devops.common.expression.context.RuntimeDictionaryContextData import com.tencent.devops.common.expression.context.RuntimeNamedValue -import com.tencent.devops.common.expression.expression.EvaluationOptions import com.tencent.devops.common.expression.expression.ExpressionOutput import com.tencent.devops.common.expression.expression.IFunctionInfo -import com.tencent.devops.common.expression.expression.ParseExceptionKind import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo import com.tencent.devops.common.pipeline.dialect.IPipelineDialect -import org.apache.tools.ant.filters.StringInputStream +import com.tencent.devops.common.pipeline.utils.ExprReplacementUtil import org.slf4j.LoggerFactory -import java.io.BufferedReader -import java.io.InputStreamReader import java.util.regex.Pattern @Suppress( @@ -74,22 +62,21 @@ object EnvReplacementParser { * @param output 表达式计算时输出 */ fun parse( - value: String?, + value: Any?, contextMap: Map, onlyExpression: Boolean? = false, contextPair: Pair>? = null, functions: Iterable? = null, output: ExpressionOutput? = null ): String { - val context = EnvReplacementContext( - value = value, + val options = ExprReplacementOptions( contextMap = contextMap, - useSingleCurlyBraces = !(onlyExpression ?: false), contextPair = contextPair, functions = functions, output = output ) - return parse(context) + val useSingleCurlyBraces = !(onlyExpression ?: false) + return parse(value = value, useSingleCurlyBraces = useSingleCurlyBraces, options = options) } /** @@ -97,36 +84,42 @@ object EnvReplacementParser { * 根据方言的配置判断是否能够使用${}或者变量值是否存在 */ fun parse( - value: String?, + value: Any?, contextMap: Map, dialect: IPipelineDialect, contextPair: Pair>? = null, functions: Iterable? = null, output: ExpressionOutput? = null ): String { - val context = EnvReplacementContext( - value = value, + val options = ExprReplacementOptions( contextMap = contextMap, - dialect = dialect, + contextNotNull = !dialect.supportMissingVar(), contextPair = contextPair, functions = functions, output = output ) - return parse(context) + return parse( + value = value, + useSingleCurlyBraces = dialect.supportUseSingleCurlyBracesVar(), + options = options + ) } - fun parse(context: EnvReplacementContext): String { - with(context) { - if (value.isNullOrBlank()) return "" - val newValue = parseExpression() - - return if (useSingleCurlyBraces) { - ObjectReplaceEnvVarUtil.replaceEnvVar(newValue, contextMap).let { - JsonUtil.toJson(it, false) - } - } else { - newValue - } + fun parse( + value: Any?, + useSingleCurlyBraces: Boolean, + options: ExprReplacementOptions + ): String { + if (value == null) return "" + // 先表达式替换 + val newValue = ExprReplaceEnvVarUtil.replaceEnvVar(value, options) + // 如果支持${}替换,则需要使用旧版变量替换 + return if (useSingleCurlyBraces) { + ObjectReplaceEnvVarUtil.replaceEnvVar(newValue, options.contextMap) + } else { + newValue + }.let { + JsonUtil.toJson(it, false) } } @@ -134,266 +127,14 @@ object EnvReplacementParser { variables: Map, extendNamedValueMap: List? = null ): Pair>? { - try { - val context = ExecutionContext(DictionaryContextData()) - val nameValue = mutableListOf() - extendNamedValueMap?.forEach { namedValue -> - nameValue.add(NamedValueInfo(namedValue.key, ContextValueNode())) - context.expressionValues.add( - namedValue.key, - RuntimeDictionaryContextData(namedValue) - ) - } - ExpressionParser.fillContextByMap(variables, context, nameValue) - return Pair(context, nameValue) - } catch (ignore: Throwable) { - println("EnvReplacementParser context invalid: $variables") - logger.warn("EnvReplacementParser context invalid: $variables", ignore) - return null - } - } - - private fun EnvReplacementContext.parseExpression(): String { - if (value.isNullOrBlank()) return "" - return try { - val (executeContext, nameValues) = contextPair - ?: getCustomExecutionContextByMap(contextMap) - ?: return value - parseExpression( - value = value, - context = executeContext, - nameValues = nameValues, - functions = functions, - output = output, - contextNotNull = contextNotNull - ) - } catch (ex: VariableNotFoundException) { - throw ex - } catch (ignore: Throwable) { - logger.warn("[$value]|EnvReplacementParser expression invalid: ", ignore) - value - } - } - - private fun parseExpression( - value: String, - nameValues: List, - context: ExecutionContext, - functions: Iterable? = null, - output: ExpressionOutput? = null, - contextNotNull: Boolean - ): String { - val strReader = InputStreamReader(StringInputStream(value)) - val bufferReader = BufferedReader(strReader) - val newValue = StringBuilder() - try { - var line = bufferReader.readLine() - while (line != null) { - // 跳过空行和注释行 - val blocks = findExpressions(line) - if (line.isBlank() || blocks.isEmpty()) { - newValue.append(line).append("\n") - line = bufferReader.readLine() - continue - } - val onceResult = parseExpressionLine( - value = line, - blocks = blocks, - context = context, - nameValues = nameValues, - functions = functions, - output = output, - contextNotNull = contextNotNull - ) - - val newLine = findExpressions(onceResult).let { - if (it.isEmpty()) { - onceResult - } else { - parseExpressionLine( - value = onceResult, - blocks = it, - context = context, - nameValues = nameValues, - functions = functions, - output = output, - contextNotNull = contextNotNull - ) - } - } - newValue.append(newLine).append("\n") - line = bufferReader.readLine() - } - } finally { - strReader.close() - bufferReader.close() - } - return newValue.toString().removeSuffix("\n") - } - - /** - * 解析表达式，根据 findExpressions 寻找的括号优先级进行解析 - */ - private fun parseExpressionLine( - value: String, - blocks: List>, - nameValues: List, - context: ExecutionContext, - functions: Iterable? = null, - output: ExpressionOutput? = null, - contextNotNull: Boolean - ): String { - var chars = value.toList() - blocks.forEachIndexed nextBlockLevel@{ blockLevel, blocksInLevel -> - blocksInLevel.forEachIndexed nextBlock@{ blockI, block -> - // 表达式因为含有 ${{ }} 所以起始向后推3位，末尾往前推两位 - val expression = chars.joinToString("").substring(block.startIndex + 3, block.endIndex - 1) - if (expression.isBlank()) return@nextBlock - val options = EvaluationOptions(contextNotNull) - var result = try { - ExpressionParser.createTree(expression, null, nameValues, functions)!! - .evaluate(null, context, options, output).value.let { - if (it is PipelineContextData) it.fetchValue() else it - }?.let { - JsonUtil.toJson(it, false) - } ?: "" - } catch (ignore: ContextNotFoundException) { - throw VariableNotFoundException( - variableKey = options.contextNotNull.errKey() - ) - } catch (ignore: ExpressionParseException) { - if (contextNotNull && ignore.kind == ParseExceptionKind.UnrecognizedNamedValue) { - throw VariableNotFoundException( - variableKey = ignore.expression - ) - } - return@nextBlock - } - - if ((blockLevel + 1 < blocks.size) && - !( - (block.startIndex - 1 >= 0 && chars[block.startIndex - 1] == '.') || - (block.endIndex + 1 < chars.size && chars[block.endIndex + 1] == '.') - ) - ) { - result = "'$result'" - } - - val charList = result.toList() - - // 将替换后的表达式嵌入原本的line - val startSub = if (block.startIndex - 1 < 0) { - listOf() - } else { - chars.slice(0 until block.startIndex) - } - val endSub = if (block.endIndex + 1 >= chars.size) { - listOf() - } else { - chars.slice(block.endIndex + 1 until chars.size) - } - chars = startSub + charList + endSub - - // 将替换后的字符查传递给后边的括号位数 - val diffNum = charList.size - (block.endIndex - block.startIndex + 1) - blocks.forEachIndexed { i, bl -> - bl.forEachIndexed level@{ j, b -> - if (i <= blockLevel && j <= blockI) { - return@level - } - if (blocks[i][j].startIndex > block.endIndex) { - blocks[i][j].startIndex += diffNum - } - if (blocks[i][j].endIndex > block.endIndex) { - blocks[i][j].endIndex += diffNum - } - } - } - } - } - - return chars.joinToString("") - } - - /** - * 寻找语句中包含 ${{}}的表达式的位置，返回成对的位置坐标，并根据优先级排序 - * 优先级算法目前暂定为 从里到外，从左到右 - * @param levelMax 返回的最大层数，从深到浅。默认为2层 - * 例如: 替换顺序如数字所示 ${{ 4 ${{ 2 ${{ 1 }} }} ${{ 3 }} }} - * @return [ 层数次序 [ 括号 ] ] [[1], [2, 3], [4]]]] - */ - private fun findExpressions(condition: String, levelMax: Int = 2): List> { - val stack = ArrayDeque() - var index = 0 - val chars = condition.toCharArray() - val levelMap = mutableMapOf>() - while (index < chars.size) { - if (index + 2 < chars.size && chars[index] == '$' && chars[index + 1] == '{' && chars[index + 2] == '{' - ) { - stack.addLast(index) - index += 3 - continue - } - - if (index + 1 < chars.size && chars[index] == '}' && chars[index + 1] == '}' - ) { - val start = stack.removeLastOrNull() - if (start != null) { - // 栈里剩下几个前括号，这个完整括号的优先级就是多少 - val level = stack.size + 1 - if (levelMap.containsKey(level)) { - levelMap[level]!!.add(ExpressionBlock(start, index + 1)) - } else { - levelMap[level] = mutableListOf(ExpressionBlock(start, index + 1)) - } - } - index += 2 - continue - } - - index++ - } - - if (levelMap.isEmpty()) { - return listOf() - } - - val result = mutableListOf>() - var max = 0 - var listIndex = 0 - run end@{ - levelMap.keys.sortedDescending().forEach result@{ level -> - val blocks = levelMap[level] ?: return@result - blocks.sortBy { it.startIndex } - blocks.forEach { block -> - if (result.size < listIndex + 1) { - result.add(mutableListOf(block)) - } else { - result[listIndex].add(block) - } - } - listIndex++ - max++ - if (max == levelMax) { - return@end - } - } - } - return result + return ExprReplacementUtil.getCustomExecutionContextByMap( + variables = variables, + extendNamedValueMap = extendNamedValueMap + ) } fun containsExpressions(value: String?): Boolean { if (value == null) return false return expressionPattern.matcher(value).find() } - - /** - * 表达式括号项 ${{ }} - * @param startIndex 括号开始位置即 $ 位置 - * @param endIndex 括号结束位置即最后一个 } 位置 - */ - data class ExpressionBlock( - var startIndex: Int, - var endIndex: Int - ) } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementContext.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/ExprReplacementOptions.kt similarity index 71% rename from src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementContext.kt rename to src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/ExprReplacementOptions.kt index ed073fadc98..b3f0abd8989 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementContext.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/ExprReplacementOptions.kt @@ -31,41 +31,19 @@ import com.tencent.devops.common.expression.ExecutionContext import com.tencent.devops.common.expression.expression.ExpressionOutput import com.tencent.devops.common.expression.expression.IFunctionInfo import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo -import com.tencent.devops.common.pipeline.dialect.IPipelineDialect import io.swagger.v3.oas.annotations.media.Schema /** * 表达式替换上下文 */ -@Schema(title = "表达式替换上下文") -data class EnvReplacementContext( - @get:Schema(title = "表达式", required = true) - val value: String?, +@Schema(title = "表达式替换参数") +data class ExprReplacementOptions( @get:Schema(title = "环境变量", required = true) val contextMap: Map, - @get:Schema(title = "是否能使用\${}占位符", required = true) - val useSingleCurlyBraces: Boolean = true, @get:Schema(title = "值是否能不存在", required = true) val contextNotNull: Boolean = false, @get:Schema(title = "表达式上下文", required = true) val contextPair: Pair>? = null, val functions: Iterable? = null, val output: ExpressionOutput? = null -) { - constructor( - value: String?, - contextMap: Map, - dialect: IPipelineDialect, - contextPair: Pair>? = null, - functions: Iterable? = null, - output: ExpressionOutput? = null - ) : this( - value = value, - contextMap = contextMap, - useSingleCurlyBraces = dialect.supportUseSingleCurlyBracesVar(), - contextNotNull = !dialect.supportMissingVar(), - contextPair = contextPair, - functions = functions, - output = output - ) -} +) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/ExprReplacementUtil.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/ExprReplacementUtil.kt new file mode 100644 index 00000000000..c60df2bcf81 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/ExprReplacementUtil.kt @@ -0,0 +1,325 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline.utils + +import com.tencent.devops.common.api.exception.VariableNotFoundException +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.expression.ContextNotFoundException +import com.tencent.devops.common.expression.ExecutionContext +import com.tencent.devops.common.expression.ExpressionParseException +import com.tencent.devops.common.expression.ExpressionParser +import com.tencent.devops.common.expression.context.ContextValueNode +import com.tencent.devops.common.expression.context.DictionaryContextData +import com.tencent.devops.common.expression.context.PipelineContextData +import com.tencent.devops.common.expression.context.RuntimeDictionaryContextData +import com.tencent.devops.common.expression.context.RuntimeNamedValue +import com.tencent.devops.common.expression.expression.EvaluationOptions +import com.tencent.devops.common.expression.expression.ExpressionOutput +import com.tencent.devops.common.expression.expression.IFunctionInfo +import com.tencent.devops.common.expression.expression.ParseExceptionKind +import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo +import com.tencent.devops.common.pipeline.ExprReplacementOptions +import org.apache.tools.ant.filters.StringInputStream +import org.slf4j.LoggerFactory +import java.io.BufferedReader +import java.io.InputStreamReader + +@Suppress( + "LoopWithTooManyJumpStatements", + "ComplexCondition", + "ComplexMethod", + "NestedBlockDepth", + "ReturnCount", + "LongParameterList" +) +object ExprReplacementUtil { + private val logger = LoggerFactory.getLogger(ExprReplacementUtil::class.java) + + fun parseExpression(value: String, options: ExprReplacementOptions): String { + with(options) { + return try { + val (executeContext, nameValues) = contextPair + ?: getCustomExecutionContextByMap(contextMap) + ?: return value + parseExpression( + value = value, + context = executeContext, + nameValues = nameValues, + functions = functions, + output = output, + contextNotNull = contextNotNull + ) + } catch (ex: VariableNotFoundException) { + throw ex + } catch (ignore: Throwable) { + logger.warn("[$value]|EnvReplacementParser expression invalid: ", ignore) + value + } + } + } + + fun getCustomExecutionContextByMap( + variables: Map, + extendNamedValueMap: List? = null + ): Pair>? { + try { + val context = ExecutionContext(DictionaryContextData()) + val nameValue = mutableListOf() + extendNamedValueMap?.forEach { namedValue -> + nameValue.add(NamedValueInfo(namedValue.key, ContextValueNode())) + context.expressionValues.add( + namedValue.key, + RuntimeDictionaryContextData(namedValue) + ) + } + ExpressionParser.fillContextByMap(variables, context, nameValue) + return Pair(context, nameValue) + } catch (ignore: Throwable) { + println("EnvReplacementParser context invalid: $variables") + logger.warn("EnvReplacementParser context invalid: $variables", ignore) + return null + } + } + + private fun parseExpression( + value: String, + nameValues: List, + context: ExecutionContext, + functions: Iterable? = null, + output: ExpressionOutput? = null, + contextNotNull: Boolean + ): String { + val strReader = InputStreamReader(StringInputStream(value)) + val bufferReader = BufferedReader(strReader) + val newValue = StringBuilder() + try { + var line = bufferReader.readLine() + while (line != null) { + // 跳过空行和注释行 + val blocks = findExpressions(line) + if (line.isBlank() || blocks.isEmpty()) { + newValue.append(line).append("\n") + line = bufferReader.readLine() + continue + } + val onceResult = parseExpressionLine( + value = line, + blocks = blocks, + context = context, + nameValues = nameValues, + functions = functions, + output = output, + contextNotNull = contextNotNull + ) + + val newLine = findExpressions(onceResult).let { + if (it.isEmpty()) { + onceResult + } else { + parseExpressionLine( + value = onceResult, + blocks = it, + context = context, + nameValues = nameValues, + functions = functions, + output = output, + contextNotNull = contextNotNull + ) + } + } + newValue.append(newLine).append("\n") + line = bufferReader.readLine() + } + } finally { + strReader.close() + bufferReader.close() + } + return newValue.toString().removeSuffix("\n") + } + + /** + * 解析表达式，根据 findExpressions 寻找的括号优先级进行解析 + */ + private fun parseExpressionLine( + value: String, + blocks: List>, + nameValues: List, + context: ExecutionContext, + functions: Iterable? = null, + output: ExpressionOutput? = null, + contextNotNull: Boolean + ): String { + var chars = value.toList() + blocks.forEachIndexed nextBlockLevel@{ blockLevel, blocksInLevel -> + blocksInLevel.forEachIndexed nextBlock@{ blockI, block -> + // 表达式因为含有 ${{ }} 所以起始向后推3位，末尾往前推两位 + val expression = chars.joinToString("").substring(block.startIndex + 3, block.endIndex - 1) + if (expression.isBlank()) return@nextBlock + val options = EvaluationOptions(contextNotNull) + var result = try { + ExpressionParser.createTree(expression, null, nameValues, functions)!! + .evaluate(null, context, options, output).value.let { + if (it is PipelineContextData) it.fetchValue() else it + }?.let { + JsonUtil.toJson(it, false) + } ?: "" + } catch (ignore: ContextNotFoundException) { + throw VariableNotFoundException( + variableKey = options.contextNotNull.errKey() + ) + } catch (ignore: ExpressionParseException) { + if (contextNotNull && ignore.kind == ParseExceptionKind.UnrecognizedNamedValue) { + throw VariableNotFoundException( + variableKey = ignore.expression + ) + } + return@nextBlock + } + + if ((blockLevel + 1 < blocks.size) && + !( + (block.startIndex - 1 >= 0 && chars[block.startIndex - 1] == '.') || + (block.endIndex + 1 < chars.size && chars[block.endIndex + 1] == '.') + ) + ) { + result = "'$result'" + } + + val charList = result.toList() + + // 将替换后的表达式嵌入原本的line + val startSub = if (block.startIndex - 1 < 0) { + listOf() + } else { + chars.slice(0 until block.startIndex) + } + val endSub = if (block.endIndex + 1 >= chars.size) { + listOf() + } else { + chars.slice(block.endIndex + 1 until chars.size) + } + chars = startSub + charList + endSub + + // 将替换后的字符查传递给后边的括号位数 + val diffNum = charList.size - (block.endIndex - block.startIndex + 1) + blocks.forEachIndexed { i, bl -> + bl.forEachIndexed level@{ j, b -> + if (i <= blockLevel && j <= blockI) { + return@level + } + if (blocks[i][j].startIndex > block.endIndex) { + blocks[i][j].startIndex += diffNum + } + if (blocks[i][j].endIndex > block.endIndex) { + blocks[i][j].endIndex += diffNum + } + } + } + } + } + + return chars.joinToString("") + } + + /** + * 寻找语句中包含 ${{}}的表达式的位置，返回成对的位置坐标，并根据优先级排序 + * 优先级算法目前暂定为 从里到外，从左到右 + * @param levelMax 返回的最大层数，从深到浅。默认为2层 + * 例如: 替换顺序如数字所示 ${{ 4 ${{ 2 ${{ 1 }} }} ${{ 3 }} }} + * @return [ 层数次序 [ 括号 ] ] [[1], [2, 3], [4]]]] + */ + private fun findExpressions(condition: String, levelMax: Int = 2): List> { + val stack = ArrayDeque() + var index = 0 + val chars = condition.toCharArray() + val levelMap = mutableMapOf>() + while (index < chars.size) { + if (index + 2 < chars.size && chars[index] == '$' && chars[index + 1] == '{' && chars[index + 2] == '{' + ) { + stack.addLast(index) + index += 3 + continue + } + + if (index + 1 < chars.size && chars[index] == '}' && chars[index + 1] == '}' + ) { + val start = stack.removeLastOrNull() + if (start != null) { + // 栈里剩下几个前括号，这个完整括号的优先级就是多少 + val level = stack.size + 1 + if (levelMap.containsKey(level)) { + levelMap[level]!!.add(ExpressionBlock(start, index + 1)) + } else { + levelMap[level] = mutableListOf(ExpressionBlock(start, index + 1)) + } + } + index += 2 + continue + } + + index++ + } + + if (levelMap.isEmpty()) { + return listOf() + } + + val result = mutableListOf>() + var max = 0 + var listIndex = 0 + run end@{ + levelMap.keys.sortedDescending().forEach result@{ level -> + val blocks = levelMap[level] ?: return@result + blocks.sortBy { it.startIndex } + blocks.forEach { block -> + if (result.size < listIndex + 1) { + result.add(mutableListOf(block)) + } else { + result[listIndex].add(block) + } + } + listIndex++ + max++ + if (max == levelMax) { + return@end + } + } + } + return result + } + + /** + * 表达式括号项 ${{ }} + * @param startIndex 括号开始位置即 $ 位置 + * @param endIndex 括号结束位置即最后一个 } 位置 + */ + data class ExpressionBlock( + var startIndex: Int, + var endIndex: Int + ) +} diff --git a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt index b5a051827fa..d41c80b7b5b 100644 --- a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt +++ b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt @@ -1,8 +1,5 @@ package com.tencent.devops.common.pipeline -import com.tencent.devops.common.api.exception.VariableNotFoundException -import com.tencent.devops.common.pipeline.dialect.ClassicPipelineDialect -import com.tencent.devops.common.pipeline.dialect.ConstrainedPipelineDialect import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test @@ -186,19 +183,22 @@ internal class EnvReplacementParserTest { parseAndEquals( data = map, template = "\${{ (variables.TXT == 'txt') }}", - expect = true.toString() + expect = true.toString(), + onlyExpression = true ) parseAndEquals( data = map, template = "\${{ (variables.TXT2 == 'txt2') }}", - expect = true.toString() + expect = true.toString(), + onlyExpression = true ) parseAndEquals( data = map, template = "\${{ ((variables.TXT == 'txt') && (variables.TXT2 == 'txt2')) }}", - expect = true.toString() + expect = true.toString(), + onlyExpression = true ) parseAndEquals( @@ -210,7 +210,8 @@ internal class EnvReplacementParserTest { expect = "echo \" 这可是来自master的改动 \"\n" + " echo \" true \"\n" + " echo \" true \"\n" + - " echo \" true \"" + " echo \" true \"", + onlyExpression = true ) parseAndEquals( @@ -222,13 +223,15 @@ internal class EnvReplacementParserTest { expect = "echo \" 这可是来自master的改动 \"\n" + "echo \" false \"\n" + "echo \" false \"\n" + - "echo \" false \"" + "echo \" false \"", + onlyExpression = true ) parseAndEquals( data = map, template = "{\"GDP\": \"\${{GDP}}亿\", \"People\": \${{People}} \"Country\": \"\${{twice}}\"}", - expect = "{\"GDP\": \"${map["GDP"]}亿\", \"People\": ${map["People"]} \"Country\": \"${map["Country"]}\"}" + expect = "{\"GDP\": \"${map["GDP"]}亿\", \"People\": ${map["People"]} \"Country\": \"${map["Country"]}\"}", + onlyExpression = true ) val data = HashMap() @@ -237,27 +240,30 @@ internal class EnvReplacementParserTest { data["t.cd"] = "\${{ab.cd}}" val template2 = "abcd_\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end" - val buff = EnvReplacementParser.parse(template2, data) + val buff = EnvReplacementParser.parse(template2, data, true) Assertions.assertEquals(template2, buff) parseAndEquals( data = data, template = "中国\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end", - expect = "中国\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end" + expect = "中国\$abc}_ffs_\${{\${{ce}}_\${{ab.c}_ end", + onlyExpression = true ) parseAndEquals( data = data, template = "abcd_\${abc}_ffs_\${{ce}}_\${{t.cd}}_ end结束%\n # 这是注释行a1\$ab_^%!#@", expect = "abcd_\${abc}_ffs_twice_${data["ab.cd"]}_ end结束%\n # 这是注释行a1\$ab_^%!#@", - contextMap = mapOf("ce" to "twice") + contextMap = mapOf("ce" to "twice"), + onlyExpression = true ) data["c_e"] = "\${none}" parseAndEquals( data = data, template = "abcd_\${abc}_ffs_\${{c_e}}_\${{t.cd}}_ end", - expect = "abcd_\${abc}_ffs_\${none}_${data["ab.cd"]}_ end" + expect = "abcd_\${abc}_ffs_\${none}_${data["ab.cd"]}_ end", + onlyExpression = true ) data["center中"] = "中国" @@ -273,14 +279,16 @@ internal class EnvReplacementParserTest { parseAndEquals( data = data, template = "\${{blank}}", - expect = "" + expect = "", + onlyExpression = true ) data["all"] = "hello" parseAndEquals( data = data, template = "\${{all}}", - expect = "hello" + expect = "hello", + onlyExpression = true ) } @@ -317,7 +325,8 @@ internal class EnvReplacementParserTest { "echo envs.env_e=e, env_e=\$env_e\n" + "echo envs.a=, a=\$a\n" + "echo settings.sensitive.password=\${{ settings.sensitive.password }}\n" + - "echo ::set-output name=a::i am a at step_1" + "echo ::set-output name=a::i am a at step_1", + onlyExpression = true ) } @@ -337,35 +346,44 @@ internal class EnvReplacementParserTest { val command8 = "echo \${{ variables.hello }}" val command9 = "echo \${{ ci.workspace }}" + val command10 = mutableMapOf( + "params" to mutableListOf( + mutableMapOf( + "key" to "instance", + "value" to "\${{variables.instance}}" + ) + ) + ) val data = mapOf( "variables.abc" to "variables.value", - "variables.hello" to "hahahahaha" + "variables.hello" to "hahahahaha", + "variables.instance" to "{\"instances\":[{\"cluster\":\"ci-prod\",\"pod\":\"ci-123\"}]}" ) // 与EnvUtils的差异点：不支持传可空对象 // Assertions.assertEquals("", EnvReplacementParser.parse(null, data)) - Assertions.assertEquals("", EnvReplacementParser.parse("", data)) - Assertions.assertEquals("", EnvReplacementParser.parse(null, data)) + Assertions.assertEquals("", EnvReplacementParser.parse("", data, true)) + Assertions.assertEquals("", EnvReplacementParser.parse(null, data, true)) Assertions.assertEquals( "hello variables.value world", - EnvReplacementParser.parse(command1, data) + EnvReplacementParser.parse(command1, data, true) ) Assertions.assertEquals( "variables.valueworld", - EnvReplacementParser.parse(command2, data) + EnvReplacementParser.parse(command2, data, true) ) Assertions.assertEquals( "hellovariables.value", - EnvReplacementParser.parse(command3, data) + EnvReplacementParser.parse(command3, data, true) ) Assertions.assertEquals( "hello\${{variables.abc", - EnvReplacementParser.parse(command4, data) + EnvReplacementParser.parse(command4, data, true) ) Assertions.assertEquals( "hello\${{variables.abc}", - EnvReplacementParser.parse(command5, data) + EnvReplacementParser.parse(command5, data, true) ) Assertions.assertEquals( command6, @@ -377,15 +395,23 @@ internal class EnvReplacementParserTest { ) Assertions.assertEquals( "echo hahahahaha", - EnvReplacementParser.parse(command8, data) + EnvReplacementParser.parse(command8, data, true) ) Assertions.assertEquals( "echo /data/landun/workspace", EnvReplacementParser.parse( value = command9, - contextMap = map.plus("ci.workspace" to "/data/landun/workspace") + contextMap = map.plus("ci.workspace" to "/data/landun/workspace"), + onlyExpression = true ) ) + val command10Expected = """ + {"params":[{"key":"instance","value":"{\"instances\":[{\"cluster\":\"ci-prod\",\"pod\":\"ci-123\"}]}"}]} + """.trimIndent() + Assertions.assertEquals( + command10Expected, + EnvReplacementParser.parse(command10, data, true) + ) } @Test @@ -397,7 +423,7 @@ internal class EnvReplacementParserTest { println("parseEnvTestData $command") Assertions.assertEquals( "{\"age\": ${map["age"]} , \"sex\": \"boy\", \"name\": ${map["name"]}}", - EnvReplacementParser.parse(command, map) + EnvReplacementParser.parse(command, map, true) ) val command1 = "hello \${{variables.abc}} world" @@ -423,26 +449,27 @@ internal class EnvReplacementParserTest { "{variables.abc" to "jacky" ) - Assertions.assertEquals("hello variables.value world", EnvReplacementParser.parse(command1, data)) - Assertions.assertEquals("variables.valueworld", EnvReplacementParser.parse(command2, data)) - Assertions.assertEquals("hellovariables.value", EnvReplacementParser.parse(command3, data)) - Assertions.assertEquals("hello\${{variables.abc", EnvReplacementParser.parse(command4, data)) - Assertions.assertEquals("hello\${{variables.abc}", EnvReplacementParser.parse(command5, data)) + Assertions.assertEquals("hello variables.value world", EnvReplacementParser.parse(command1, data, true)) + Assertions.assertEquals("variables.valueworld", EnvReplacementParser.parse(command2, data, true)) + Assertions.assertEquals("hellovariables.value", EnvReplacementParser.parse(command3, data, true)) + Assertions.assertEquals("hello\${{variables.abc", EnvReplacementParser.parse(command4, data, true)) + Assertions.assertEquals("hello\${{variables.abc}", EnvReplacementParser.parse(command5, data, true)) Assertions.assertEquals("hello\${variables.abc}}", EnvReplacementParser.parse(command6, data, true)) - Assertions.assertEquals("hello\$variables.abc}}", EnvReplacementParser.parse(command7, data)) - Assertions.assertEquals("echo hahahahaha", EnvReplacementParser.parse(command8, data)) + Assertions.assertEquals("hello\$variables.abc}}", EnvReplacementParser.parse(command7, data, true)) + Assertions.assertEquals("echo hahahahaha", EnvReplacementParser.parse(command8, data, true)) Assertions.assertEquals( "echo /data/landun/workspace || hahahahaha", EnvReplacementParser.parse( value = command9, - contextMap = data.plus("ci.workspace" to "/data/landun/workspace") + contextMap = data.plus("ci.workspace" to "/data/landun/workspace"), + onlyExpression = true ) ) - Assertions.assertEquals("echo \${{ ci.xyz == 'zzzz' }}", EnvReplacementParser.parse(command10, data)) - Assertions.assertEquals("echo true", EnvReplacementParser.parse(command11, data)) - Assertions.assertEquals("echo false", EnvReplacementParser.parse(command12, data)) - Assertions.assertEquals("echo true", EnvReplacementParser.parse(command13, data)) - Assertions.assertEquals("true", EnvReplacementParser.parse(command14, data)) + Assertions.assertEquals("echo \${{ ci.xyz == 'zzzz' }}", EnvReplacementParser.parse(command10, data, true)) + Assertions.assertEquals("echo true", EnvReplacementParser.parse(command11, data, true)) + Assertions.assertEquals("echo false", EnvReplacementParser.parse(command12, data, true)) + Assertions.assertEquals("echo true", EnvReplacementParser.parse(command13, data, true)) + Assertions.assertEquals("true", EnvReplacementParser.parse(command14, data, true)) } @Test @@ -484,14 +511,7 @@ console.log("全局配置", variables) let branch = master let branch1 = let branchs = branch.split("/")""" - Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data)) - Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, ClassicPipelineDialect())) - Assertions.assertThrows(VariableNotFoundException::class.java) { - EnvReplacementParser.parse(command1, data, ConstrainedPipelineDialect()) - } - Assertions.assertThrows(VariableNotFoundException::class.java) { - EnvReplacementParser.parse(command1, mapOf(), ConstrainedPipelineDialect()) - } + Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, true)) } @Test @@ -541,11 +561,7 @@ echo true""" "variables.is_build" to "false", "ci.branch" to "master" ) - Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data)) - Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, ClassicPipelineDialect())) - Assertions.assertThrows(VariableNotFoundException::class.java) { - EnvReplacementParser.parse(command1, mapOf(), ConstrainedPipelineDialect()) - } + Assertions.assertEquals(result, EnvReplacementParser.parse(command1, data, true)) } private fun parseAndEquals( diff --git a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtilTest.kt b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtilTest.kt new file mode 100644 index 00000000000..f6a11c54c0a --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtilTest.kt @@ -0,0 +1,319 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline + +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.api.util.JsonUtil.toJson +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +@Suppress("ALL", "UNCHECKED_CAST") +class ExprReplaceEnvVarUtilTest { + + private val envMap: MutableMap = HashMap() + + @BeforeEach + fun setup() { + envMap["normalStrEnvVar"] = "123" + envMap["specStrEnvVar"] = "D:\\tmp\\hha" + envMap["jsonStrEnvVar"] = "{\"abc\":\"123\"}" + } + + private val lineSeparator = System.getProperty("line.separator") + private val jsonExcept = "{$lineSeparator" + + " \"abc\" : \"变量替换测试_{\\\"abc\\\":\\\"123\\\"}\"$lineSeparator" + + "}" + + private val arrayJsonExcept = "[ \"变量替换测试_{\\\"abc\\\":\\\"123\\\"}\" ]" + + @Test + fun replaceList() { + val testBean = TestBean( + testBeanKey = "bean变量替换测试_\${{specStrEnvVar}}", + testBeanValue = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + ) + // 对list对象进行变量替换 + val originDataListObj = ArrayList() + originDataListObj.add("变量替换测试_\${{normalStrEnvVar}}") + originDataListObj.add("变量替换测试_\${{specStrEnvVar}}") + originDataListObj.add("变量替换测试_\${{jsonStrEnvVar}}") + originDataListObj.add("{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}") + originDataListObj.add("[\"变量替换测试_\${{jsonStrEnvVar}}\"]") + originDataListObj.add(testBean) + val dataMapObj: MutableMap = HashMap() + dataMapObj["dataMapKey"] = "变量替换测试_\${{specStrEnvVar}}" + dataMapObj["testBean"] = testBean + originDataListObj.add(dataMapObj) + val convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataListObj, envMap) as List<*> + + assertEquals("变量替换测试_${envMap["normalStrEnvVar"]}", convertDataObj[0]) + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertDataObj[1]) + assertEquals("变量替换测试_${envMap["jsonStrEnvVar"]}", convertDataObj[2]) + assertEquals(jsonExcept, convertDataObj[3]) + assertEquals(arrayJsonExcept, convertDataObj[4]) + + val convertTestBean = convertDataObj[5] as TestBean + assertEquals("bean变量替换测试_${envMap["specStrEnvVar"]}", convertTestBean.testBeanKey) + assertEquals(jsonExcept, convertTestBean.testBeanValue) + } + + @Test + fun replaceIllegalJson() { + val objectJson = "{\"abc:\"变量替换测试_\${{normalStrEnvVar}}\"" + val convertDataObj1 = ExprReplaceEnvVarUtil.replaceEnvVar(objectJson, envMap) + println(convertDataObj1) + assertEquals("{\"abc:\"变量替换测试_${envMap["normalStrEnvVar"]}\"", convertDataObj1) + + val arrayJson = "[1, \"变量替换测试_\${{normalStrEnvVar}}\"" + val convertDataObj2 = ExprReplaceEnvVarUtil.replaceEnvVar(arrayJson, envMap) + println(convertDataObj2) + assertEquals("[1, \"变量替换测试_${envMap["normalStrEnvVar"]}\"", convertDataObj2) + } + + @Test + fun replaceSet() { + + val testBean = TestBean( + testBeanKey = "bean变量替换测试_\${{specStrEnvVar}}", + testBeanValue = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + ) + // 对set对象进行变量替换 + val originDataSetObj = HashSet() + originDataSetObj.add("1变量替换测试_\${{normalStrEnvVar}}") + originDataSetObj.add("2变量替换测试_\${{specStrEnvVar}}") + originDataSetObj.add("3变量替换测试_\${{jsonStrEnvVar}}") + originDataSetObj.add("{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}") + originDataSetObj.add("[\"变量替换测试_\${{jsonStrEnvVar}}\"]") + originDataSetObj.add(testBean) + + val setDataMapObj: MutableMap = HashMap() + setDataMapObj["dataMapKey"] = "变量替换测试_\${{specStrEnvVar}}" + setDataMapObj["testBean"] = testBean + originDataSetObj.add(setDataMapObj) + val convertDataObj = (ExprReplaceEnvVarUtil.replaceEnvVar(originDataSetObj, envMap) as Set<*>) + + convertDataObj.forEach { member -> + when { + member is Map<*, *> -> { + member.forEach { sm -> + when { + sm.key.toString() == "testBean" -> { + assertEquals( + "bean变量替换测试_${envMap["specStrEnvVar"]}", + (sm.value as TestBean).testBeanKey + ) + assertEquals(jsonExcept, (sm.value as TestBean).testBeanValue) + } + sm.key.toString() == "dataMapKey" -> { + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", sm.value) + } + else -> { + assertEquals(member.toString(), "setDataMapObj") + } + } + } + } + member is TestBean -> { + assertEquals("bean变量替换测试_${envMap["specStrEnvVar"]}", member.testBeanKey) + assertEquals(jsonExcept, member.testBeanValue) + } + member.toString().startsWith("1") -> { + assertEquals("1变量替换测试_${envMap["normalStrEnvVar"]}", member) + } + member.toString().startsWith("2") -> { + assertEquals("2变量替换测试_${envMap["specStrEnvVar"]}", member) + } + member.toString().startsWith("3") -> { + assertEquals("3变量替换测试_${envMap["jsonStrEnvVar"]}", member) + } + member.toString().startsWith("{") -> { + assertEquals(jsonExcept, member) + } + member.toString().startsWith("[") -> { + assertEquals(arrayJsonExcept, member) + } + else -> { + assertEquals(member.toString(), "convertDataObj") + } + } + } + } + + @Test + fun replaceMapWithTestBean() { + // 对map对象进行变量替换 + val originDataMapObj: MutableMap = HashMap() + originDataMapObj["normalStrEnvVarKey"] = "变量替换测试_\${{normalStrEnvVar}}" + originDataMapObj["specStrEnvVarKey"] = "变量替换测试_\${{specStrEnvVar}}" + originDataMapObj["jsonStrEnvVarKey1"] = "变量替换测试_\${{jsonStrEnvVar}}" + originDataMapObj["jsonStrEnvVarKey2"] = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + originDataMapObj["jsonStrEnvVarKey3"] = "\${{jsonStrEnvVar}}" + var originSubDataMapObj: MutableMap? = HashMap() + originSubDataMapObj!!["normalStrEnvVarKey"] = "变量替换测试_\${{normalStrEnvVar}}" + originSubDataMapObj["specStrEnvVarKey"] = "变量替换测试_\${{specStrEnvVar}}" + originSubDataMapObj["jsonStrEnvVarKey1"] = "变量替换测试_\${{jsonStrEnvVar}}" + originSubDataMapObj["jsonStrEnvVarKey2"] = "\${{jsonStrEnvVar}}" + + val testBean = TestBean( + testBeanKey = "变量替换测试_\${{specStrEnvVar}}", + testBeanValue = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + ) + originSubDataMapObj["testBean"] = testBean + originDataMapObj["originSubDataMapObj"] = originSubDataMapObj + + val cpb = ExprReplaceEnvVarUtil.replaceEnvVar(originDataMapObj, envMap) + val testBeanMap = ((cpb as Map)["originSubDataMapObj"] as Map)["testBean"] as TestBean + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", testBeanMap.testBeanKey) + assertEquals(jsonExcept, testBeanMap.testBeanValue) + // 判断map中jsonStrEnvVarKey3对应的值进行变量替换后能否正常转换为json串 + assertEquals(envMap["jsonStrEnvVar"], (cpb as Map)["jsonStrEnvVarKey3"]!!) + originSubDataMapObj = cpb["originSubDataMapObj"] as MutableMap? + // 判断嵌套的map中jsonStrEnvVarKey2对应的值进行变量替换后能否正常转换为json串 + assertEquals(envMap["jsonStrEnvVar"], originSubDataMapObj!!["jsonStrEnvVarKey2"]!!) + } + + @Test + fun replaceTestComplexBean() { + // 对普通的javaBean对象进行转换 + val testComplexBean = TestComplexBean() + testComplexBean.testBeanKey = "变量替换测试_\${{specStrEnvVar}}" + testComplexBean.testBeanValue = "[\"变量替换测试_\${{jsonStrEnvVar}}\"]" + + val dataList = ArrayList() + dataList.add("变量替换测试_\${{normalStrEnvVar}}") + dataList.add("变量替换测试_\${{specStrEnvVar}}") + dataList.add("变量替换测试_\${{jsonStrEnvVar}}") + dataList.add("{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}") + dataList.add("[\"变量替换测试_\${{jsonStrEnvVar}}\"]") + testComplexBean.dataList = dataList + + var dataMap: MutableMap = HashMap() + dataMap["normalStrEnvVarKey"] = " 变量替换测试_\${{normalStrEnvVar}} " + dataMap["specStrEnvVarKey"] = "变量替换测试_\${{specStrEnvVar}}" + dataMap["jsonStrEnvVarKey1"] = "变量替换测试_\${{jsonStrEnvVar}}" + dataMap["jsonStrEnvVarKey2"] = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + dataMap["jsonStrEnvVarKey3"] = "[\"变量替换测试_\${{jsonStrEnvVar}}\"]" + val subDataMap: MutableMap = HashMap() + subDataMap["normalStrEnvVarKey"] = "变量替换测试_\${{normalStrEnvVar}}" + subDataMap["specStrEnvVarKey"] = "变量替换测试_\${{specStrEnvVar}}" + subDataMap["jsonStrEnvVarKey1"] = "变量替换测试_\${{jsonStrEnvVar}}" + subDataMap["jsonStrEnvVarKey2"] = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + + val testBean = TestBean( + testBeanKey = "bean变量替换测试_\${{specStrEnvVar}}", + testBeanValue = "{\"abc\":\"bean变量替换测试_\${{jsonStrEnvVar}}\"}" + ) + subDataMap["testBean"] = testBean + dataMap["subDataMap"] = subDataMap + testComplexBean.dataMap = dataMap + + val dataSet = HashSet() + dataSet.add("变量替换测试_\${{normalStrEnvVar}}") + dataSet.add("变量替换测试_\${{specStrEnvVar}}") + dataSet.add("变量替换测试_\${{jsonStrEnvVar}}") + dataSet.add("{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}") + dataSet.add("[\"变量替换测试_\${{jsonStrEnvVar}}\"]") + testComplexBean.dataSet = dataSet + + // start to test + var convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(testComplexBean, envMap) + val convertBean = convertDataObj as TestComplexBean + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertBean.testBeanKey) + + assertEquals("变量替换测试_${envMap["normalStrEnvVar"]}", convertBean.dataList!![0]) + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertBean.dataList!![1]) + assertEquals("变量替换测试_${envMap["jsonStrEnvVar"]}", convertBean.dataList!![2]) + assertEquals(jsonExcept, convertBean.dataList!![3]) + assertEquals("[ \"变量替换测试_{\\\"abc\\\":\\\"123\\\"}\" ]", convertBean.dataList!![4]) + + assertEquals(" 变量替换测试_${envMap["normalStrEnvVar"]} ", convertBean.dataMap!!["normalStrEnvVarKey"]) + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertBean.dataMap!!["specStrEnvVarKey"]) + assertEquals("变量替换测试_${envMap["jsonStrEnvVar"]}", convertBean.dataMap!!["jsonStrEnvVarKey1"]) + assertEquals(jsonExcept, convertBean.dataMap!!["jsonStrEnvVarKey2"]) + assertEquals(arrayJsonExcept, convertBean.dataMap!!["jsonStrEnvVarKey3"]) + + // 替换包含null的对象 + dataMap = HashMap() + dataMap["key1"] = "变量" + dataMap["key2"] = arrayOf(null, "哈哈") + + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(dataMap, envMap) as Map<*, *> + assertEquals(dataMap["key1"], convertDataObj["key1"]) + assertEquals(toJson(dataMap["key2"]!!), convertDataObj["key2"]) + println("convertDataObj=$convertDataObj") + } + + @Test + fun replaceEnvVar() { + + // 对普通字符串进行普通字符串变量替换 + var originDataObj: Any = "变量替换测试_\${{normalStrEnvVar}}" + var convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap) + assertEquals("变量替换测试_123", toJson(convertDataObj)) + + // 对普通字符串进行带特殊字符字符串变量替换 + originDataObj = "变量替换测试_\${{specStrEnvVar}}" + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap) + assertEquals("变量替换测试_D:\\tmp\\hha", toJson(convertDataObj)) + + // 对普通字符串进行json字符串变量替换 + originDataObj = "变量替换测试_\${{jsonStrEnvVar}}" + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap) + assertEquals("变量替换测试_{\"abc\":\"123\"}", toJson(convertDataObj)) + + // number类型变量替换 + originDataObj = "[1,2,3]" + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap) + println(toJson(convertDataObj)) + assertEquals(toJson(JsonUtil.to(originDataObj, List::class.java)), toJson(convertDataObj)) + + // 魔法数字符创测试 + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar("12E2", envMap) + assertEquals("12E2", toJson(convertDataObj)) + // 替换”[133]-[sid-${normalStrEnvVar}]-[sid-zhiliang-test1]“带多个[]的字符串 + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar( + "[133]-[sid-\${{normalStrEnvVar}}]-[sid-zhiliang-test1]", + envMap + ) + assertEquals("[133]-[sid-123]-[sid-zhiliang-test1]", toJson(convertDataObj)) + } + + internal data class TestBean( + var testBeanKey: String? = null, + var testBeanValue: String? = null + ) + + internal data class TestComplexBean( + var testBeanKey: String? = null, + var testBeanValue: String? = null, + var dataList: List<*>? = null, + var dataMap: Map<*, *>? = null, + var dataSet: Set<*>? = null + ) +} diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt index e7d3776aa84..7e88172775e 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt @@ -37,7 +37,6 @@ import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.MessageUtil import com.tencent.devops.common.pipeline.EnvReplacementParser import com.tencent.devops.common.pipeline.NameAndValue -import com.tencent.devops.common.pipeline.dialect.PipelineDialectType import com.tencent.devops.common.pipeline.enums.BuildFormPropertyType import com.tencent.devops.common.pipeline.enums.BuildTaskStatus import com.tencent.devops.common.pipeline.pojo.BuildParameters @@ -46,7 +45,6 @@ import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.pojo.BuildJobResult import com.tencent.devops.process.pojo.BuildTask import com.tencent.devops.process.pojo.BuildVariables -import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.PIPELINE_RETRY_COUNT import com.tencent.devops.process.utils.PipelineVarUtil import com.tencent.devops.worker.common.constants.WorkerMessageCode.BK_PREPARE_TO_BUILD @@ -67,10 +65,10 @@ import com.tencent.devops.worker.common.task.TaskFactory import com.tencent.devops.worker.common.utils.CredentialUtils import com.tencent.devops.worker.common.utils.KillBuildProcessTree import com.tencent.devops.worker.common.utils.ShellUtil +import org.slf4j.LoggerFactory import java.io.File import java.io.FileNotFoundException import java.io.IOException -import org.slf4j.LoggerFactory import kotlin.system.exitProcess object Runner { @@ -454,9 +452,6 @@ object Runner { } if (customEnv.isNullOrEmpty()) return val jobVariables = jobBuildVariables.variables.toMutableMap() - val dialect = jobBuildVariables.variables[PIPELINE_DIALECT]?.let { - PipelineDialectType.valueOf(it).dialect - } ?: PipelineDialectType.CLASSIC.dialect customEnv.forEach { if (!it.key.isNullOrBlank()) { // 解决BUG:93319235,将Task的env变量key加env.前缀塞入variables，塞入之前需要对value做替换 diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt index 8dfa0dacd96..c8ef516ea59 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt @@ -504,7 +504,7 @@ open class MarketAtomTask : ITask() { } inputMap.forEach { (name, value) -> atomParams[name] = EnvReplacementParser.parse( - value = JsonUtil.toJson(value), + value = value, contextMap = variables, dialect = dialect, contextPair = customReplacement,