Skip to content

Commit

Permalink
Merge remote-tracking branch 'github-bk-bcs/master'
Browse files Browse the repository at this point in the history 
* github-bk-bcs/master:
  feat: bcs-monitor 新增接口支持批量查询节点 metrics overview (#3299)
  fix: 配置示例http调整--bug=118652507 (#3394)
  fix: 配置文件路径和名称需要支持符号"." (#3390)
  fix: 优化配置模版产品交互弱化版本概念以及修复描述相关问题 (#3387)
  1. feed-server新增根据key获取值和原数据 (#3391)
  feat: 配置示例更新：1.键值型新增http示例；2.p2p集群id从下拉框变更为输入框 (#3389)
  feat: 前端配置返回feed-server服务相关配置 (#3392)
  fix: 从历史版本导入配置项-重新选择配置项后没有显示配置项信息--bug=127768737 (#3393)
  fix: 批量导入配置文件验证文件数 (#3374)
  fix:  新建配置文件上传二进制文件乱码问题修复--bug=127338589 (#3388)
  feat: bscp增加配置下发负载平衡能力 (#3308)
  feat: 1.配置示例新增p2p开关；2.节点管理临时目录不符合规则不显示复制按钮及内容--story=118517185 (#3385)
  • Loading branch information
@wenxinlee2015
wenxinlee2015 committed Jul 23, 2024
2 parents 73238ff + d6d6307 commit f4694d3
Show file tree
Hide file tree
Showing 90 changed files with 10,405 additions and 6,547 deletions.
149 changes: 99 additions & 50 deletions bcs-services/bcs-bscp/cmd/api-server/service/config_import.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/criteria/errf"
"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/repository"
"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/dal/table"
"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/i18n"
"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/iam/auth"
"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/kit"
pbcs "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/config-server"
Expand Down Expand Up @@ -73,26 +74,23 @@ func (c *configImport) TemplateConfigFileImport(w http.ResponseWriter, r *http.R
tmplSpaceIdStr := chi.URLParam(r, "template_space_id")
tmplSpaceID, _ := strconv.Atoi(tmplSpaceIdStr)
if tmplSpaceID == 0 {
_ = render.Render(w, r, rest.BadRequest(errors.New("validation parameter fail")))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "id is required"))))
return
}

fileName := chi.URLParam(r, "filename")
if fileName == "" {
_ = render.Render(w, r, rest.BadRequest(errors.New("file name cannot be empty")))
return
}

// Validation size
totleContentLength, singleContentLength := getUploadConfig(kt.BizID)

var maxSize int64
if unzip {
maxSize = totleContentLength
} else {
maxSize = singleContentLength
}

if err := checkUploadSize(r, maxSize); err != nil {
if err := checkUploadSize(kt, r, maxSize); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
return
}
Expand All @@ -103,7 +101,8 @@ func (c *configImport) TemplateConfigFileImport(w http.ResponseWriter, r *http.R
// 读取前512字节以检测文件类型
n, errR := r.Body.Read(buffer[:512])
if errR != nil && errR != io.EOF {
_ = render.Render(w, r, rest.BadRequest(errR))
_ = render.Render(w, r,
rest.BadRequest(errors.New(i18n.T(kt, "read file failed %s", errR))))
return
}

Expand All @@ -120,24 +119,33 @@ func (c *configImport) TemplateConfigFileImport(w http.ResponseWriter, r *http.R
// 创建目录
dirPath := path.Join(os.TempDir(), constant.UploadTemporaryDirectory)
if err := createTemporaryDirectory(dirPath); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "create directory failed %s", err))))
return
}
// 随机生成临时目录
tempDir, err := os.MkdirTemp(dirPath, "templateConfigItem-")
if err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "create temporary directory failed %s", err))))
return
}

if identifyFileType != archive.Unknown {
if err = archive.Unpack(combinedReader, identifyFileType, tempDir, singleContentLength*constant.MB); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
compare := new(errf.ErrorF)
if errors.As(err, &compare) {
if compare.Code == int32(archive.FileTooLarge) {
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt,
"decompress the file. The size of file %s exceeds the maximum limit of %s", compare.Message,
tools.BytesToHumanReadable(uint64(singleContentLength*constant.MB))))))
return
}
}
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "decompression failed %s", err))))
return
}
} else {
if err = saveFile(combinedReader, tempDir, fileName); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "upload file failed %s", err))))
return
}
}
Expand All @@ -147,7 +155,7 @@ func (c *configImport) TemplateConfigFileImport(w http.ResponseWriter, r *http.R
// 先扫描一遍文件夹，获取路径和名称，
fileItems, err := getFilePathsAndNames(tempDir)
if err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "upload file failed %s", err))))
return
}

Expand All @@ -157,13 +165,13 @@ func (c *configImport) TemplateConfigFileImport(w http.ResponseWriter, r *http.R
c.uploadFileMetrics(kt.BizID, tmplSpaceIdStr, dirPath, totalSize)

if err = c.checkFileConfictsWithTemplates(kt, uint32(tmplSpaceID), fileItems); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "detecting file conflicts failed %s", err))))
return
}

folder, err := c.processAndUploadDirectoryFiles(kt, tempDir, len(fileItems))
if err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "upload file failed %s", err))))
return
}

Expand Down Expand Up @@ -231,9 +239,10 @@ func (c *configImport) TemplateConfigFileImport(w http.ResponseWriter, r *http.R
exist = append(exist, newItem)
}
}
msg := "上传完成"

msg := i18n.T(kt, "upload completed")
if uploadErrCount > 0 {
msg = fmt.Sprintf("上传完成，失败 %d 个", uploadErrCount)
msg = i18n.T(kt, "upload completed, %d failed", uploadErrCount)
}
sortByPathName(exist)
sortByPathName(nonExist)
Expand All @@ -259,16 +268,12 @@ func (c *configImport) ConfigFileImport(w http.ResponseWriter, r *http.Request)
appIdStr := chi.URLParam(r, "app_id")
appId, _ := strconv.Atoi(appIdStr)
if appId == 0 {
_ = render.Render(w, r, rest.BadRequest(errors.New("validation parameter fail")))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "id is required"))))
return
}
kt.AppID = uint32(appId)

fileName := chi.URLParam(r, "filename")
if fileName == "" {
_ = render.Render(w, r, rest.BadRequest(errors.New("file name cannot be empty")))
return
}

// Validation size
totleContentLength, singleContentLength := getUploadConfig(kt.BizID)
Expand All @@ -278,15 +283,16 @@ func (c *configImport) ConfigFileImport(w http.ResponseWriter, r *http.Request)
} else {
maxSize = singleContentLength
}
if err := checkUploadSize(r, maxSize); err != nil {
if err := checkUploadSize(kt, r, maxSize); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
return
}
buffer := make([]byte, 512)
// 读取前512字节以检测文件类型
n, errR := r.Body.Read(buffer[:512])
if errR != nil && errR != io.EOF {
_ = render.Render(w, r, rest.BadRequest(errR))
_ = render.Render(w, r,
rest.BadRequest(errors.New(i18n.T(kt, "read file failed %s", errR))))
return
}

Expand All @@ -303,24 +309,35 @@ func (c *configImport) ConfigFileImport(w http.ResponseWriter, r *http.Request)
// 创建目录
dirPath := path.Join(os.TempDir(), constant.UploadTemporaryDirectory)
if err := createTemporaryDirectory(dirPath); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r,
rest.BadRequest(errors.New(i18n.T(kt, "create directory failed %s", err))))
return
}

// 随机生成临时目录
tempDir, err := os.MkdirTemp(dirPath, "configItem-")
if err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "create temporary directory failed %s", err))))
return
}

if identifyFileType != archive.Unknown {
if err = archive.Unpack(combinedReader, identifyFileType, tempDir, singleContentLength*constant.MB); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
compare := new(errf.ErrorF)
if errors.As(err, &compare) {
if compare.Code == int32(archive.FileTooLarge) {
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt,
"decompress the file. The size of file %s exceeds the maximum limit of %s", compare.Message,
tools.BytesToHumanReadable(uint64(singleContentLength*constant.MB))))))
return
}
}
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "decompression failed %s", err))))
return
}
} else {
if err = saveFile(combinedReader, tempDir, fileName); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "upload file failed %s", err))))
return
}
}
Expand All @@ -330,7 +347,7 @@ func (c *configImport) ConfigFileImport(w http.ResponseWriter, r *http.Request)
// 先扫描一遍文件夹，获取路径和名称，
fileItems, err := getFilePathsAndNames(tempDir)
if err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "upload file failed %s", err))))
return
}

Expand All @@ -340,25 +357,16 @@ func (c *configImport) ConfigFileImport(w http.ResponseWriter, r *http.Request)
c.uploadFileMetrics(kt.BizID, appIdStr, dirPath, totalSize)

if err = c.checkFileConfictsWithNonTemplates(kt, fileItems); err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
return
}

folder, err := c.processAndUploadDirectoryFiles(kt, tempDir, len(fileItems))
if err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "detecting file conflicts failed %s", err))))
return
}

// 批量验证biz_id、app_id、name、path是否存在
configItems := []*pbcs.ListConfigItemByTupleReq_Item{}
uploadErrCount := 0
for _, item := range folder {
if item.Err != nil {
uploadErrCount++
}
for _, item := range fileItems {
configItems = append(configItems, &pbcs.ListConfigItemByTupleReq_Item{
Name: item.File.Name,
Path: item.File.Path,
Name: item.Name,
Path: item.Path,
})
}

Expand All @@ -371,21 +379,52 @@ func (c *configImport) ConfigFileImport(w http.ResponseWriter, r *http.Request)
end = len(configItems)
}
batch := configItems[i:end]
// 批量验证biz_id、app_id、name、path是否存在
tuple, err := c.cfgClient.ListConfigItemByTuple(kt.RpcCtx(), &pbcs.ListConfigItemByTupleReq{
tuple, errC := c.cfgClient.ListConfigItemByTuple(kt.RpcCtx(), &pbcs.ListConfigItemByTupleReq{
BizId: kt.BizID,
AppId: kt.AppID,
Items: batch,
})
if err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
if errC != nil {
_ = render.Render(w, r, rest.BadRequest(errC))
return
}
for _, item := range tuple.GetDetails() {
configItem[path.Join(item.GetSpec().GetPath(), item.GetSpec().GetName())] = item
}
}

// 配置项总数 + 引入的套餐配置项数量 - 已存在的数量 + 新增的数量
result, err := c.cfgClient.GetTemplateAndNonTemplateCICount(kt.RpcCtx(), &pbcs.GetTemplateAndNonTemplateCICountReq{
BizId: kt.BizID,
AppId: kt.AppID,
})
if err != nil {
_ = render.Render(w, r, rest.BadRequest(err))
return
}

addCount := len(fileItems) - len(configItem)
total := int(result.GetConfigItemCount()+result.GetTemplateConfigItemCount()) + len(configItem) + addCount
limit := getAppConfigCnt(kt.BizID)
if total > limit {
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt,
"decompress file failed, exceeding the maximum file limit threshold of %d", limit))))
return
}

folder, err := c.processAndUploadDirectoryFiles(kt, tempDir, len(fileItems))
if err != nil {
_ = render.Render(w, r, rest.BadRequest(errors.New(i18n.T(kt, "upload file failed %s", err))))
return
}

uploadErrCount := 0
for _, item := range folder {
if item.Err != nil {
uploadErrCount++
}
}

exist := make([]*types.TemplateItem, 0)
nonExist := make([]*types.TemplateItem, 0)
for _, item := range folder {
Expand All @@ -412,9 +451,9 @@ func (c *configImport) ConfigFileImport(w http.ResponseWriter, r *http.Request)
}
}

msg := "上传完成"
msg := i18n.T(kt, "upload completed")
if uploadErrCount > 0 {
msg = fmt.Sprintf("上传完成，失败 %d 个", uploadErrCount)
msg = i18n.T(kt, "upload completed, %d failed", uploadErrCount)
}
sortByPathName(exist)
sortByPathName(nonExist)
Expand All @@ -426,10 +465,11 @@ func (c *configImport) ConfigFileImport(w http.ResponseWriter, r *http.Request)
}

// 检测上传文件的大小
func checkUploadSize(r *http.Request, maxSizeMB int64) error {
func checkUploadSize(kt *kit.Kit, r *http.Request, maxSizeMB int64) error {
maxSize := maxSizeMB * constant.MB
if r.ContentLength > maxSize {
return errors.New("request body size exceeds the allowed limit")
return errors.New(i18n.T(kt, "upload failed, please make sure the file size does not exceed %s",
tools.BytesToHumanReadable(uint64(maxSize))))
}
return nil
}
Expand Down Expand Up @@ -790,3 +830,12 @@ func getUploadConfig(bizID uint32) (int64, int64) {
resLimit := cc.ApiServer().FeatureFlags.ResourceLimit
return int64(resLimit.Default.MaxUploadContentLength), int64(resLimit.Default.MaxFileSize)
}

func getAppConfigCnt(bizID uint32) int {
if resLimit, ok := cc.ApiServer().FeatureFlags.ResourceLimit.Spec[fmt.Sprintf("%d", bizID)]; ok {
if resLimit.AppConfigCnt > 0 {
return int(resLimit.AppConfigCnt)
}
}
return int(cc.ApiServer().FeatureFlags.ResourceLimit.Default.AppConfigCnt)
}
30 changes: 29 additions & 1 deletion bcs-services/bcs-bscp/cmd/config-server/service/config_item.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -664,7 +664,6 @@ func (s *Service) CompareConfigItemConflicts(ctx context.Context, req *pbcs.Comp

res := []*meta.ResourceAttribute{
{Basic: meta.Basic{Type: meta.Biz, Action: meta.FindBusinessResource}, BizID: req.BizId},
{Basic: meta.Basic{Type: meta.App, Action: meta.Update, ResourceID: req.AppId}, BizID: req.BizId},
}

if err := s.authorizer.Authorize(grpcKit, res...); err != nil {
Expand Down Expand Up @@ -723,3 +722,32 @@ func (s *Service) CompareConfigItemConflicts(ctx context.Context, req *pbcs.Comp
TemplateConfigs: templateConfigs,
}, nil
}

// GetTemplateAndNonTemplateCICount 获取模板和非模板配置项数量
func (s *Service) GetTemplateAndNonTemplateCICount(ctx context.Context, req *pbcs.GetTemplateAndNonTemplateCICountReq) (
*pbcs.GetTemplateAndNonTemplateCICountResp, error) {

grpcKit := kit.FromGrpcContext(ctx)

res := []*meta.ResourceAttribute{
{Basic: meta.Basic{Type: meta.Biz, Action: meta.FindBusinessResource}, BizID: req.BizId},
}

if err := s.authorizer.Authorize(grpcKit, res...); err != nil {
return nil, err
}

result, err := s.client.DS.GetTemplateAndNonTemplateCICount(grpcKit.RpcCtx(),
&pbds.GetTemplateAndNonTemplateCICountReq{
BizId: req.GetBizId(),
AppId: req.GetAppId(),
})
if err != nil {
return nil, err
}

return &pbcs.GetTemplateAndNonTemplateCICountResp{
ConfigItemCount: result.GetConfigItemCount(),
TemplateConfigItemCount: result.GetTemplateConfigItemCount(),
}, err
}
2 changes: 1 addition & 1 deletion bcs-services/bcs-bscp/cmd/config-server/service/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,7 @@ func (s *Service) BatchUpsertTemplates(ctx context.Context, req *pbcs.BatchUpser
if err := g.Wait(); err != nil {
return nil, err
}
in := &pbds.BatchUpsertTemplatesReq{Items: items}
in := &pbds.BatchUpsertTemplatesReq{Items: items, TemplateSetIds: req.GetTemplateSetIds(), BizId: req.GetBizId()}
data, err := s.client.DS.BatchUpsertTemplates(grpcKit.RpcCtx(), in)
if err != nil {
return nil, err
Expand Down
1 change: 1 addition & 0 deletions bcs-services/bcs-bscp/cmd/config-server/service/template_revision.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,7 @@ func (s *Service) UpdateTemplateRevision(ctx context.Context, req *pbcs.UpdateTe
Md5: metadata.Md5,
},
},
TemplateRevisionId: req.GetTemplateRevisionId(),
}
rp, err := s.client.DS.UpdateTemplateRevision(grpcKit.RpcCtx(), r)
if err != nil {
Expand Down
Loading

0 comments on commit f4694d3

Please sign in to comment.