feat：流水线变量语法支持两种风格 #10576

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt

@@ -185,6 +185,7 @@ object CommonMessageCode {
 
     const val SVN_TOKEN_FAIL = "2100135" // SVN Token 不正确
     const val SVN_TOKEN_EMPTY = "2100136" // SVN Token 为空, 请检查代码库的凭证类型
+    const val ERROR_VARIABLE_NOT_FOUND = "2100137" // SVN Token 为空, 请检查代码库的凭证类型
     const val BK_CONTAINER_TIMED_OUT = "bkContainerTimedOut" // 创建容器超时
     const val BK_CREATION_FAILED_EXCEPTION_INFORMATION = "bkCreationFailedExceptionInformation" // 创建失败，异常信息
 

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/exception/VariableNotFoundException.kt

@@ -0,0 +1,43 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.devops.common.api.exception
+
+import com.tencent.devops.common.api.constant.CommonMessageCode
+
+/**
+ * 变量不存在异常
+ */
+class VariableNotFoundException(
+    val variableKey: String?,
+    errorCode: String = CommonMessageCode.ERROR_VARIABLE_NOT_FOUND
+) :
+    ErrorCodeException(
+        errorCode = errorCode,
+        defaultMessage = "variable $variableKey not found",
+        params = variableKey?.let { arrayOf(it) }
+    )

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/PipelineAsCodeSettings.kt

@@ -32,5 +32,35 @@ import io.swagger.v3.oas.annotations.media.Schema
 @Schema(title = "设置-YAML流水线功能设置")
 data class PipelineAsCodeSettings(
     @get:Schema(title = "是否支持YAML流水线功能", required = true)
-    val enable: Boolean = false
-)
+    val enable: Boolean = false,
+    @get:Schema(title = "项目级流水线语法风格", required = false)
+    var projectDialect: String? = null,
+    @get:Schema(title = "是否继承项目流水线语言风格", required = false)
+    val inheritedDialect: Boolean? = true,
+    @get:Schema(title = "流水线语言风格", required = false)
+    var pipelineDialect: String? = null
+) {
+    companion object {
+        fun initDialect(inheritedDialect: Boolean?, pipelineDialect: String?): PipelineAsCodeSettings {
+            return PipelineAsCodeSettings(
+                inheritedDialect = inheritedDialect ?: true,
+                // 如果继承项目方言配置,置空pipelineDialect字段,防止数据库存储多余数据
+                pipelineDialect = if (inheritedDialect == false) {
+                    pipelineDialect
+                } else {
+                    null
+                }
+            )
+        }
+    }
+
+    /**
+     * 入库时,重置方言字段值
+     */
+    fun resetDialect() {
+        projectDialect = null
+        if (inheritedDialect != false) {
+            pipelineDialect = null
+        }
+    }
+}

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ContextTree.kt

@@ -1,10 +1,5 @@
 package com.tencent.devops.common.expression
 
-import com.fasterxml.jackson.databind.JsonNode
-import com.fasterxml.jackson.databind.node.ArrayNode
-import com.fasterxml.jackson.databind.node.JsonNodeFactory
-import com.fasterxml.jackson.databind.node.ObjectNode
-import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.tencent.devops.common.expression.context.ContextValueNode
 import com.tencent.devops.common.expression.context.DictionaryContextData
 import com.tencent.devops.common.expression.context.DictionaryContextDataWithVal
@@ -13,8 +8,6 @@ import com.tencent.devops.common.expression.context.StringContextData
 import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo
 import java.util.LinkedList
 import java.util.Queue
-import java.util.Stack
-import kotlin.jvm.Throws
 
 /**
  * 用来将流水线变量转为树的形式，来对其转换到表达式引擎做兼容处理
@@ -144,22 +137,6 @@ open class ContextTreeNode(
         children.add(child)
     }
 
-    fun depthFirstTraversal(run: (node: ContextTreeNode) -> Unit) {
-        val stack = Stack<ContextTreeNode>()
-        stack.push(this)
-
-        while (!stack.isEmpty()) {
-            val node = stack.pop()
-
-            run(node)
-
-            // 将子节点逆序入栈，保证先访问左边的子节点
-            for (i in node.children.lastIndex downTo 0) {
-                stack.push(node.children[i])
-            }
-        }
-    }
-
     fun breadthFirstTraversal(run: (node: ContextTreeNode) -> Boolean) {
         val queue: Queue<ContextTreeNode> = LinkedList()
         queue.offer(this)
@@ -188,66 +165,4 @@ open class ContextTreeNode(
         }
         return dict
     }
-
-    // 校验当前节点的值转换的JSON树是否与子节点结构和值相同
-//    @Throws(ContextJsonFormatException::class)
-//    private fun checkJson() {
-//        val jsonTree = try {
-//            ObjectMapper().readTree(this.value)
-//        } catch (e: Exception) {
-//            throw ContextJsonFormatException("${this.value} to json error ${e.localizedMessage}")
-//        }
-//        // TODO: 是否需要兼容 json 不同类型
-//        jsonTree.equals(ObjectNodeComparator(), this.toJson())
-//    }
-
-    private fun toJson(): JsonNode {
-        val jsonNodeFactory = JsonNodeFactory.instance
-        val rootObj = jacksonObjectMapper().createObjectNode()
-        if (this.children.isEmpty()) {
-            return jsonNodeFactory.textNode(this.value)
-        }
-        this.children.forEach { child ->
-            rootObj.putIfAbsent(child.key, child.toJson())
-        }
-        return rootObj
-    }
-}
-
-// 存在用户的 json 值为 array 但是翻译成了 map，这里我们认为是等价的
-class ObjectNodeComparator : Comparator<JsonNode> {
-    override fun compare(o1: JsonNode?, o2: JsonNode?): Int {
-        if (o1 == null && o2 == null) {
-            return 0
-        }
-        if (o1 == null || o2 == null) {
-            return 1
-        }
-        if (o1 == o2) {
-            return 0
-        }
-        if (o1 is ArrayNode && o2 is ObjectNode) {
-            if (o1.size() != o2.size()) {
-                return 1
-            }
-            o1.forEachIndexed { index, node ->
-                if (o2[index] == null || node != o2[index]) {
-                    return 1
-                }
-            }
-            return 0
-        }
-        if (o1 is ObjectNode && o2 is ArrayNode) {
-            if (o1.size() != o2.size()) {
-                return 1
-            }
-            o2.forEachIndexed { index, node ->
-                if (o1[index] == null || node != o1[index]) {
-                    return 1
-                }
-            }
-            return 0
-        }
-        return 1
-    }
 }

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt

@@ -82,12 +82,6 @@ class FunctionFormatException(override val message: String?) : ExpressionExcepti
     }
 }
 
-class ContextNotFoundException(override val message: String?) : ExpressionException() {
-    companion object {
-        fun contextNameNotFound(arg0: String) = ContextNotFoundException(
-            "Expression context $arg0 not found."
-        )
-    }
-}
+class ContextNotFoundException(override var message: String? = null) : ExpressionException()
 
 class ContextJsonFormatException(override val message: String?) : ExpressionException()

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt

@@ -27,9 +27,11 @@
 
 package com.tencent.devops.common.expression
 
+import com.tencent.devops.common.expression.ExpressionParser.legalizeExpression
 import com.tencent.devops.common.expression.context.ContextValueNode
 import com.tencent.devops.common.expression.context.DictionaryContextData
 import com.tencent.devops.common.expression.context.PipelineContextData
+import com.tencent.devops.common.expression.expression.EvaluationOptions
 import com.tencent.devops.common.expression.expression.ExpressionConstants
 import com.tencent.devops.common.expression.expression.IExpressionNode
 import com.tencent.devops.common.expression.expression.IFunctionInfo
@@ -74,8 +76,9 @@ object ExpressionParser {
         nameValue: List<NamedValueInfo>,
         fetchValue: Boolean
     ): Any? {
+        val options = EvaluationOptions(false)
         val result = createTree(expression.legalizeExpression(), null, nameValue, null)!!
-            .evaluate(null, context, null, null)
+            .evaluate(null, context, options, null)
         if (!fetchValue) {
             return result
         }
@@ -90,10 +93,32 @@ object ExpressionParser {
         val context = ExecutionContext(DictionaryContextData())
         val nameValue = mutableListOf<NamedValueInfo>()
         fillContextByMap(contextMap, context, nameValue)
+        val options = EvaluationOptions(false)
+        try {
+            return doEvaluateByMap(
+                expression = expression,
+                context = context,
+                nameValue = nameValue,
+                options = options,
+                fetchValue = fetchValue
+            )
+        } catch (e: Throwable) {
+            if (options.contextNotNull() && e is ContextNotFoundException) {
+                throw ContextNotFoundException("Expression context ${options.contextNotNull.errKey()} not found.")
+            }
+            throw e
+        }
+    }
 
+    private fun doEvaluateByMap(
+        expression: String,
+        context: ExecutionContext,
+        nameValue: MutableList<NamedValueInfo>,
+        options: EvaluationOptions,
+        fetchValue: Boolean
+    ): Any? {
         val result = createTree(expression.legalizeExpression(), null, nameValue, null)!!
-            .evaluate(null, context, null, null)
-
+            .evaluate(null, context, options, null)
         if (!fetchValue) {
             return result
         }
@@ -104,8 +129,9 @@ object ExpressionParser {
     /**
      * 将流水线变量转换为表达式上下文类型，存在如下情况
      * 1、a = str, 直接使用 string 类型的上下文保存即可
-     * 2、a.b.c = str, 将 a.b.c 升格为上下文中的嵌套 map 保存 既 a {b: {c: str}}
-     * 3、a.b.c = str 且 a.b = {"c": "str"}, 需要校验 a.b 所保存的 json 与 a.b.c 结构和数据是否相同后再升格
+     * 2、a.b.c = str, 将 a.b.c 转换为上下文中的嵌套 map 保存 既 a {b: {c: str}}
+     * 3、a.b.c = str 且 a.b = {"c": "str"}, 将 a.b 保存为兼容用户的数据类型，在不涉及引擎计算纯输出的情况下使用 a.b 的原数据，
+     * 在涉及引擎计算时，则使用 a.b.c 转换后的 map，同 2 中所述
      */
     fun fillContextByMap(
         contextMap: Map<String, String>,

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/AbsDictionaryContextData.kt

@@ -1,18 +1,15 @@
 package com.tencent.devops.common.expression.context
 
 import com.fasterxml.jackson.databind.JsonNode
+import com.tencent.devops.common.expression.expression.sdk.CollectionPipelineResult
 import com.tencent.devops.common.expression.expression.sdk.IReadOnlyObject
 import com.tencent.devops.common.expression.utils.ExpressionJsonUtil
 import java.util.TreeMap
 
 /**
  * dict 的抽象类，总结公共方法
  */
-abstract class AbsDictionaryContextData :
-    PipelineContextData(PipelineContextDataType.DICTIONARY),
-    Iterable<Pair<String, PipelineContextData?>>,
-    IReadOnlyObject {
-
+abstract class AbsDictionaryContextData : PipelineContextData(PipelineContextDataType.DICTIONARY), IReadOnlyObject {
     protected open var mIndexLookup: TreeMap<String, Int>? = null
     protected open var mList: MutableList<DictionaryContextDataPair> = mutableListOf()
 
@@ -24,14 +21,6 @@ abstract class AbsDictionaryContextData :
             return emptyList()
         }
 
-    override fun tryGetValue(key: String): Pair<Any?, Boolean> {
-        if (mList.isNotEmpty() && indexLookup.containsKey(key)) {
-            return Pair(mList[indexLookup[key]!!].value, true)
-        }
-
-        return Pair(null, false)
-    }
-
     protected val indexLookup: MutableMap<String, Int>
         get() {
             if (mIndexLookup == null) {
@@ -51,6 +40,29 @@ abstract class AbsDictionaryContextData :
             return mList
         }
 
+    override operator fun get(key: String): PipelineContextData? {
+        val index = indexLookup[key] ?: return null
+        return list[index].value
+    }
+
+    override fun getRes(key: String): CollectionPipelineResult {
+        return if (containsKey(key)) {
+            CollectionPipelineResult(list.getOrNull(indexLookup[key]!!)?.value)
+        } else {
+            CollectionPipelineResult.noKey()
+        }
+    }
+
+    override fun toJson(): JsonNode {
+        val json = ExpressionJsonUtil.createObjectNode()
+        if (mList.isNotEmpty()) {
+            mList.forEach {
+                json.set<JsonNode>(it.key, it.value?.toJson())
+            }
+        }
+        return json
+    }
+
     operator fun set(k: String, value: PipelineContextData?) {
         // Existing
         val index = indexLookup[k]
@@ -64,22 +76,6 @@ abstract class AbsDictionaryContextData :
         }
     }
 
-    open operator fun get(k: String): PipelineContextData? {
-        // Existing
-        val index = indexLookup[k] ?: return null
-        return list[index].value
-    }
-
-    open operator fun IReadOnlyObject.get(key: String): Any? {
-        val index = indexLookup[key] ?: return null
-        return list[index].value
-    }
-
-    operator fun Pair<String, PipelineContextData>.get(key: Int): Pair<String, PipelineContextData?> {
-        val pair = mList[key]
-        return Pair(pair.key, pair.value)
-    }
-
     fun add(pairs: Iterable<Pair<String, PipelineContextData>>) {
         pairs.forEach { pair ->
             add(pair.first, pair.second)
@@ -94,18 +90,8 @@ abstract class AbsDictionaryContextData :
         list.add(DictionaryContextDataPair(key, value))
     }
 
-    override fun iterator(): Iterator<Pair<String, PipelineContextData?>> {
-        return mList.map { pair -> Pair(pair.key, pair.value); }.iterator()
-    }
-
-    override fun toJson(): JsonNode {
-        val json = ExpressionJsonUtil.createObjectNode()
-        if (mList.isNotEmpty()) {
-            mList.forEach {
-                json.set<JsonNode>(it.key, it.value?.toJson())
-            }
-        }
-        return json
+    fun containsKey(key: String): Boolean {
+        return mList.isNotEmpty() && indexLookup.containsKey(key)
     }
 
     protected data class DictionaryContextDataPair(