Merge pull request #8524 from yaoxuwan/issue_8505

feat: 主集群和子集群间构建制品网关与流量网关的配置优化 #8505
TencentBlueKing · Mar 22, 2023 · 288073e · 288073e
2 parents f56c3c7 + b050f5e
commit 288073e
Show file tree

Hide file tree

Showing 9 changed files with 36 additions and 33 deletions.
diff --git a/...-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/FileGatewayService.kt b/...-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/FileGatewayService.kt
@@ -33,10 +33,10 @@ import org.springframework.stereotype.Service
 
 @Service
 class FileGatewayService {
-    @Value("\${artifactory.fileDevnetGateway:}")
+    @Value("\${devopsGateway.fileDevnetGateway:}")
     private lateinit var fileDevnetGateway: String
 
-    @Value("\${artifactory.fileIdcGateway:}")
+    @Value("\${devopsGateway.fileIdcGateway:}")
     private lateinit var fileIdcGateway: String
 
     fun getFileGateway(): FileGatewayInfo {

diff --git a/...pi-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveResourceApi.kt b/...pi-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveResourceApi.kt
@@ -37,7 +37,6 @@ import com.tencent.devops.common.api.pojo.Result
 import com.tencent.devops.process.pojo.BuildVariables
 import com.tencent.devops.worker.common.api.AbstractBuildResourceApi
 import com.tencent.devops.worker.common.logger.LoggerService
-import com.tencent.devops.worker.common.utils.TaskUtil
 import okhttp3.MultipartBody
 import okhttp3.RequestBody
 import java.io.File
@@ -95,11 +94,7 @@ class ArchiveResourceApi : AbstractBuildResourceApi(), ArchiveSDKApi {
             .addFormDataPart("file", URLEncoder.encode(file.name, "utf-8"), fileBody)
             .build()
 
-        val request = buildPost(
-            path = url,
-            requestBody = requestBody,
-            useFileDevnetGateway = TaskUtil.isVmBuildEnv(buildVariables.containerType)
-        )
+        val request = buildPost(path = url, requestBody = requestBody)
         val response = request(request, "上传自定义文件失败")
         try {
             val obj = JsonParser.parseString(response).asJsonObject
@@ -121,11 +116,7 @@ class ArchiveResourceApi : AbstractBuildResourceApi(), ArchiveSDKApi {
             .addFormDataPart("file", URLEncoder.encode(file.name, "utf-8"), fileBody)
             .build()
 
-        val request = buildPost(
-            path = url,
-            requestBody = requestBody,
-            useFileDevnetGateway = TaskUtil.isVmBuildEnv(buildVariables.containerType)
-        )
+        val request = buildPost(path = url, requestBody = requestBody)
         val response = request(request, "上传流水线文件失败")
         try {
             val obj = JsonParser.parseString(response).asJsonObject
@@ -153,7 +144,7 @@ class ArchiveResourceApi : AbstractBuildResourceApi(), ArchiveSDKApi {
             "/ms/artifactory/api/build/artifactories/file/archive/download" +
                     "?fileType=${FileTypeEnum.BK_CUSTOM}&customFilePath=$uri"
         }
-        val request = buildGet(url, useFileDevnetGateway = isVmBuildEnv)
+        val request = buildGet(url)
         download(request, destPath)
     }
 
@@ -173,7 +164,7 @@ class ArchiveResourceApi : AbstractBuildResourceApi(), ArchiveSDKApi {
             "/ms/artifactory/api/build/artifactories/file/archive/download" +
                     "?fileType=${FileTypeEnum.BK_ARCHIVE}&customFilePath=$uri"
         }
-        val request = buildGet(url, useFileDevnetGateway = isVmBuildEnv)
+        val request = buildGet(url)
         download(request, destPath)
     }
 

diff --git a/.../src/main/kotlin/com/tencent/devops/worker/common/api/archive/BkRepoArchiveResourceApi.kt b/.../src/main/kotlin/com/tencent/devops/worker/common/api/archive/BkRepoArchiveResourceApi.kt
@@ -40,7 +40,6 @@ import com.tencent.devops.process.pojo.BuildVariables
 import com.tencent.devops.worker.common.api.AbstractBuildResourceApi
 import com.tencent.devops.worker.common.api.ApiPriority
 import com.tencent.devops.worker.common.logger.LoggerService
-import com.tencent.devops.worker.common.utils.TaskUtil
 import okhttp3.MediaType.Companion.toMediaTypeOrNull
 import okhttp3.MultipartBody
 import okhttp3.RequestBody
@@ -103,8 +102,7 @@ class BkRepoArchiveResourceApi : AbstractBuildResourceApi(), ArchiveSDKApi {
         val request = buildPut(
             url,
             RequestBody.create("application/octet-stream".toMediaTypeOrNull(), file),
-            bkrepoResourceApi.getUploadHeader(file, buildVariables, true),
-            useFileDevnetGateway = TaskUtil.isVmBuildEnv(buildVariables.containerType)
+            bkrepoResourceApi.getUploadHeader(file, buildVariables, true)
         )
         val response = request(request, "上传自定义文件失败")
         try {
@@ -163,8 +161,7 @@ class BkRepoArchiveResourceApi : AbstractBuildResourceApi(), ArchiveSDKApi {
         val request = buildPut(
             url,
             RequestBody.create("application/octet-stream".toMediaTypeOrNull(), file),
-            bkrepoResourceApi.getUploadHeader(file, buildVariables, true),
-            useFileDevnetGateway = TaskUtil.isVmBuildEnv(buildVariables.containerType)
+            bkrepoResourceApi.getUploadHeader(file, buildVariables, true)
         )
         val response = request(request, "上传流水线文件失败")
         try {

diff --git a/...api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/BkRepoResourceApi.kt b/...api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/BkRepoResourceApi.kt
@@ -56,6 +56,7 @@ import com.tencent.devops.worker.common.api.archive.pojo.BkRepoResponse
 import com.tencent.devops.worker.common.api.archive.pojo.QueryData
 import com.tencent.devops.worker.common.api.archive.pojo.QueryNodeInfo
 import com.tencent.devops.worker.common.api.archive.pojo.TokenType
+import com.tencent.devops.worker.common.env.AgentEnv
 import com.tencent.devops.worker.common.utils.IosUtils
 import com.tencent.devops.worker.common.utils.TaskUtil
 import net.dongliu.apk.parser.ApkFile
@@ -84,6 +85,9 @@ class BkRepoResourceApi : AbstractBuildResourceApi() {
     }
 
     fun tokenAccess(): Boolean {
+        if (!AgentEnv.getFileGateway().isNullOrBlank()) {
+            return true
+        }
         var fileDevnetGateway = CommonEnv.fileDevnetGateway
         var fileIdcGateway = CommonEnv.fileIdcGateway
         if (fileDevnetGateway == null || fileIdcGateway == null) {
@@ -192,7 +196,7 @@ class BkRepoResourceApi : AbstractBuildResourceApi() {
         val url = "/bkrepo/api/build/generic/$projectId/$repoName/${urlEncode(fullPath)}"
         val header = HashMap<String, String>()
         header[BKREPO_UID] = user
-        val request = buildGet(url, header, true)
+        val request = buildGet(url, header)
         download(request, destPath)
     }
 
@@ -262,8 +266,7 @@ class BkRepoResourceApi : AbstractBuildResourceApi() {
         val request = buildPut(
             path = url,
             requestBody = RequestBody.create("application/octet-stream".toMediaTypeOrNull(), file),
-            headers = getUploadHeader(file, buildVariables, parseAppMetadata),
-            useFileDevnetGateway = true
+            headers = getUploadHeader(file, buildVariables, parseAppMetadata)
         )
         val response = request(request, "上传文件失败")
         try {

diff --git a/...dk/src/main/kotlin/com/tencent/devops/worker/common/api/report/BkRepoReportResourceApi.kt b/...dk/src/main/kotlin/com/tencent/devops/worker/common/api/report/BkRepoReportResourceApi.kt
@@ -38,7 +38,6 @@ import com.tencent.devops.worker.common.api.ApiPriority
 import com.tencent.devops.worker.common.api.archive.BkRepoResourceApi
 import com.tencent.devops.worker.common.logger.LoggerService
 import com.tencent.devops.worker.common.logger.LoggerService.elementId
-import com.tencent.devops.worker.common.utils.TaskUtil
 import okhttp3.MediaType.Companion.toMediaTypeOrNull
 import okhttp3.RequestBody
 import java.io.File
@@ -112,8 +111,7 @@ class BkRepoReportResourceApi : AbstractBuildResourceApi(), ReportSDKApi {
         val request = buildPut(
             path = url,
             requestBody = RequestBody.create("application/octet-stream".toMediaTypeOrNull(), file),
-            headers = bkrepoResourceApi.getUploadHeader(file, buildVariables, parseAppMetadata = false),
-            useFileDevnetGateway = TaskUtil.isVmBuildEnv(buildVariables.containerType)
+            headers = bkrepoResourceApi.getUploadHeader(file, buildVariables, parseAppMetadata = false)
         )
         val responseContent = request(request, "上传自定义报告失败")
         try {

diff --git a/...-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportResourceApi.kt b/...-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportResourceApi.kt
@@ -37,7 +37,6 @@ import com.tencent.devops.process.pojo.BuildVariables
 import com.tencent.devops.process.pojo.report.ReportEmail
 import com.tencent.devops.worker.common.api.AbstractBuildResourceApi
 import com.tencent.devops.worker.common.logger.LoggerService
-import com.tencent.devops.worker.common.utils.TaskUtil
 import okhttp3.MediaType.Companion.toMediaTypeOrNull
 import okhttp3.MultipartBody
 import okhttp3.RequestBody
@@ -69,11 +68,7 @@ class ReportResourceApi : AbstractBuildResourceApi(), ReportSDKApi {
             .addFormDataPart("file", file.name, fileBody)
             .build()
 
-        val request = buildPost(
-            path = url,
-            requestBody = requestBody,
-            useFileDevnetGateway = TaskUtil.isVmBuildEnv(buildVariables.containerType)
-        )
+        val request = buildPost(path = url, requestBody = requestBody)
 
         val response = request(request, "上传自定义报告失败")
 

diff --git a/...r-common/src/main/kotlin/com/tencent/devops/worker/common/api/AbstractBuildResourceApi.kt b/...r-common/src/main/kotlin/com/tencent/devops/worker/common/api/AbstractBuildResourceApi.kt
@@ -381,7 +381,9 @@ abstract class AbstractBuildResourceApi : WorkerRestApiSDK {
         return if (path.startsWith("http://") || path.startsWith("https://")) {
             path
         } else if (useFileDevnetGateway != null) {
-            if (useFileDevnetGateway) {
+            if (!AgentEnv.getFileGateway().isNullOrBlank()) {
+                fixUrl(AgentEnv.getFileGateway()!!, path)
+            } else if (useFileDevnetGateway) {
                 val fileDevnetGateway = CommonEnv.fileDevnetGateway
                 fixUrl(if (fileDevnetGateway.isNullOrBlank()) gateway else fileDevnetGateway, path)
             } else {

diff --git a/...ore/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/env/AgentEnv.kt b/...ore/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/env/AgentEnv.kt
@@ -53,6 +53,7 @@ object AgentEnv {
     private const val DOCKER_AGENT_SECRET_KEY = "devops_agent_secret_key"
     private const val AGENT_GATEWAY = "landun.gateway"
     private const val DOCKER_GATEWAY = "devops_gateway"
+    private const val AGENT_FILE_GATEWAY = "DEVOPS_FILE_GATEWAY"
     private const val AGENT_ENV = "landun.env"
     private const val AGENT_LOG_SAVE_MODE = "devops_log_save_mode"
     private const val AGENT_PROPERTIES_FILE_NAME = ".agent.properties"
@@ -62,6 +63,7 @@ object AgentEnv {
     private var agentId: String? = null
     private var secretKey: String? = null
     private var gateway: String? = null
+    private var fileGateway: String? = null
     private var os: OSType? = null
     private var env: Env? = null
     private var logStorageMode: LogStorageMode? = null
@@ -167,6 +169,18 @@ object AgentEnv {
         return gateway!!
     }
 
+    fun getFileGateway(): String? {
+        if (fileGateway.isNullOrBlank()) {
+            synchronized(this) {
+                if (fileGateway.isNullOrBlank()) {
+                    fileGateway = getEnvProp(AGENT_FILE_GATEWAY)
+                    logger.info("file gateway: $fileGateway")
+                }
+            }
+        }
+        return fileGateway
+    }
+
     fun getOS(): OSType {
         if (os == null) {
             synchronized(this) {

diff --git a/...ker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt b/...ker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt
@@ -581,6 +581,9 @@ open class MarketAtomTask : ITask() {
     }
 
     private fun getFileGateway(containerType: String?): String {
+        if (!AgentEnv.getFileGateway().isNullOrBlank()) {
+            return AgentEnv.getFileGateway()!!
+        }
         val vmBuildEnvFlag = TaskUtil.isVmBuildEnv(containerType)
         var fileDevnetGateway = CommonEnv.fileDevnetGateway
         var fileIdcGateway = CommonEnv.fileIdcGateway