Feat bitbucket server support (apache#6689)

* feat: initial commit for bitbucket-server

* chore: remove bb-cloud code from server

Signed-off-by: Marais Van Zyl <marais@projexis.co.za>

* chore: remove more bb cloud references

Signed-off-by: Marais Van Zyl <marais@projexis.co.za>

* chore: set line ending to lf

Signed-off-by: Marais Van Zyl <marais@projexis.co.za>

* chore(bitbucket-server): spelling mistake

* chore(bitbucket-server): update swagger paths

* chore(bitbucket-server): fix migration name

* chore(bitbucket-server): renames from bitbucket

* chore: change test endpoint

* refactor: getting scopes to function

* chore: update repo/project remote search

Signed-off-by: Marais Van Zyl <marais@projexis.co.za>

* chore: update paging

* fix(bitbucket-server): remote scopes paging

* fix(bitbucket-server): remote scopes

* refactor: small changes to account for server apis

* test: update raw repository data

* test: repo test

* chore: refactor fields

* several fixes

* fix: made prs and pr comments extractable, renamed account to user (better suits bb terminology)

* fix: user convert

* fix: pr comment convertor

* fix: pr_commit_collector and pr_commit_extractor

* fix: commit collector/extractor

* feat: branch collector, fixes

* fix: made collection of commits happen across all branches

* fix: deleted build_collector - no use

* feat: bitbucket-server in ui + several fixes

* fix: renamed bitbucket-server to bitbucket_server

* fix: pr base_repo_id issue

* feat: grafana dashboard

* fix: pr collection issue

* fix: pr_collector end

* test: e2e for prs

* test: e2e for pr_commits

* fix: extraction commit author + e2e tests for commits

* fix: renamed account_convertor to user_convertor

* chore: removed dead comments

* fix: rebase issues

* fix: test connection

* refactor: using dsHelper

* fix: scope config crud

* feat: implemented transformations, removed previous transformations

* refactor: renamed migration

* chore: removed dead code

* fix: domain types, enabling tasks by default

* fix: merge commit sha for pull requests

* fix: pr commits pagination issue, remote scopes pagination

* fix: pagination fix for all collections in bitbucket_server

* fix: recollect issues

* fix: merge commit sha through activities api

* refactor: removed useless code and bumped up concurrency

* refactor: reduced repeating code

* fix: renamed bitbucket_server structs and fixed RootPkgPath for both bitbucket and bitbucket_server

* fix: remove commit sha parents

* fix: pagination issue, removed not used concurrency

* fix: conflict BitBucket

* fix: cicd detected issues

* fix: models

* fix: ui issues, linting issues, removed dead code

* fix: test issues

* fix: branches are not incremental

* fix: branches are not incremental

* fix: all collectors are now stateless

* fix: removed useless tasks

* fix: stateful collectors

---------

Signed-off-by: Marais Van Zyl <marais@projexis.co.za>
Co-authored-by: Marais Van Zyl <marais@projexis.co.za>
Co-authored-by: Lynwee <linwei.hou@merico.dev>

.editorconfig

@@ -1,5 +1,6 @@
 [*]
 trim_trailing_whitespace = true
+end_of_line = lf
 
 [*.go]
 indent_style = tab

backend/plugins/bitbucket/impl/impl.go

@@ -180,7 +180,7 @@ func (p Bitbucket) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin
 }
 
 func (p Bitbucket) RootPkgPath() string {
-	return "github.com/apache/incubator-devlake/plugins/bitbucket"
+	return "github.com/apache/incubator-devlake/plugins/bitbucket/" // the "/" fixes an issue where records from "bitbucket_server" are counted as "bitbucket" records and vice versa
 }
 
 func (p Bitbucket) MigrationScripts() []plugin.MigrationScript {

backend/plugins/bitbucket_server/api/blueprint_v200.go

@@ -0,0 +1,194 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
+	"github.com/apache/incubator-devlake/core/models/domainlayer"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/utils"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/srvhelper"
+	"github.com/apache/incubator-devlake/plugins/bitbucket_server/models"
+	"github.com/apache/incubator-devlake/plugins/bitbucket_server/tasks"
+)
+
+func MakeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	connectionId uint64,
+	bpScopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+	// get the connection info for url
+	connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// needed for the connection to populate its access tokens
+	// if AppKey authentication method is selected
+	_, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopes, err := makeScopesV200(scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return plan, scopes, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	scopeDetails []*srvhelper.ScopeDetail[models.BitbucketServerRepo, models.BitbucketServerScopeConfig],
+	connection *models.BitbucketServerConnection,
+) (coreModels.PipelinePlan, errors.Error) {
+	plan := make(coreModels.PipelinePlan, len(scopeDetails))
+	for i, scopeDetail := range scopeDetails {
+		repo, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig
+		stage := plan[i]
+		if stage == nil {
+			stage = coreModels.PipelineStage{}
+		}
+		// refdiff
+		if scopeConfig != nil && scopeConfig.Refdiff != nil {
+			// add a new task to next stage
+			j := i + 1
+			if j == len(plan) {
+				plan = append(plan, nil)
+			}
+			refdiffOp := scopeConfig.Refdiff
+			refdiffOp["repoId"] = didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId)
+			plan[j] = coreModels.PipelineStage{
+				{
+					Plugin:  "refdiff",
+					Options: refdiffOp,
+				},
+			}
+			scopeConfig.Refdiff = nil
+		}
+
+		// construct task options for bitbucket
+		op := &tasks.BitbucketServerOptions{
+			ConnectionId: repo.ConnectionId,
+			FullName:     repo.BitbucketId,
+		}
+		options, err := tasks.EncodeTaskOptions(op)
+		if err != nil {
+			return nil, err
+		}
+
+		subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeConfig.Entities)
+		if err != nil {
+			return nil, err
+		}
+		stage = append(stage, &coreModels.PipelineTask{
+			Plugin:   "bitbucket_server",
+			Subtasks: subtasks,
+			Options:  options,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// add gitex stage
+		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) {
+			cloneUrl, err := errors.Convert01(url.Parse(repo.CloneUrl))
+			if err != nil {
+				return nil, err
+			}
+			cloneUrl.User = url.UserPassword(connection.Username, connection.Password)
+			stage = append(stage, &coreModels.PipelineTask{
+				Plugin: "gitextractor",
+				Options: map[string]interface{}{
+					"url":    cloneUrl.String(),
+					"name":   repo.BitbucketId,
+					"repoId": didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId),
+					"proxy":  connection.Proxy,
+				},
+			})
+
+		}
+		plan[i] = stage
+	}
+	return plan, nil
+}
+
+func makeScopesV200(
+	scopeDetails []*srvhelper.ScopeDetail[models.BitbucketServerRepo, models.BitbucketServerScopeConfig],
+	connection *models.BitbucketServerConnection,
+) ([]plugin.Scope, errors.Error) {
+	scopes := make([]plugin.Scope, 0)
+	for _, scopeDetail := range scopeDetails {
+		repo, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig
+		// if no entities specified, use all entities enabled by default
+		if len(scopeConfig.Entities) == 0 {
+			scopeConfig.Entities = plugin.DOMAIN_TYPES
+		}
+		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) ||
+			utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) ||
+			utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CROSS) {
+			// if we don't need to collect gitex, we need to add repo to scopes here
+			scopeRepo := &code.Repo{
+				DomainEntity: domainlayer.DomainEntity{
+					Id: didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId),
+				},
+				Name: repo.BitbucketId,
+			}
+			scopes = append(scopes, scopeRepo)
+		}
+		// add cicd_scope to scopes
+		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) {
+			scopeCICD := &devops.CicdScope{
+				DomainEntity: domainlayer.DomainEntity{
+					Id: didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId),
+				},
+				Name: repo.BitbucketId,
+			}
+			scopes = append(scopes, scopeCICD)
+		}
+		// add board to scopes
+		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) {
+			scopeTicket := &ticket.Board{
+				DomainEntity: domainlayer.DomainEntity{
+					Id: didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId),
+				},
+				Name: repo.BitbucketId,
+			}
+			scopes = append(scopes, scopeTicket)
+		}
+	}
+	return scopes, nil
+}

backend/plugins/bitbucket_server/api/connection.go

@@ -0,0 +1,163 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/apache/incubator-devlake/server/api/shared"
+
+	"github.com/apache/incubator-devlake/core/errors"
+	plugin "github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/bitbucket_server/models"
+)
+
+type BitBucketServerTestConnResponse struct {
+	shared.ApiBody
+	Connection *models.BitbucketServerConn
+}
+
+func testConnection(ctx context.Context, connection models.BitbucketServerConn) (*BitBucketServerTestConnResponse, errors.Error) {
+	// test connection
+	apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+	if err != nil {
+		return nil, err
+	}
+	res, err := apiClient.Get("rest/api/1.0/projects", nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if res.StatusCode == http.StatusUnauthorized {
+		return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error when testing connection")
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code when testing connection")
+	}
+	body := BitBucketServerTestConnResponse{}
+	body.Success = true
+	body.Message = "success"
+	body.Connection = &connection
+	// output
+	return &body, nil
+}
+
+// @Summary test bitbucket connection
+// @Description Test bitbucket Connection
+// @Tags plugins/bitbucket_server
+// @Param body body models.BitbucketServerConn true "json body"
+// @Success 200  {object} BitBucketServerTestConnResponse "Success"
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/test [POST]
+func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	// decode
+	var err errors.Error
+	var connection models.BitbucketServerConn
+	if err := api.Decode(input.Body, &connection, vld); err != nil {
+		return nil, errors.BadInput.Wrap(err, "could not decode request parameters")
+	}
+	// test connection
+	result, err := testConnection(context.TODO(), connection)
+	if err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil
+}
+
+// TestExistingConnection test bitbucket connection
+// @Summary test bitbucket connection
+// @Description Test bitbucket Connection
+// @Tags plugins/bitbucket
+// @Success 200  {object} BitBucketTestConnResponse "Success"
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/{connectionId}/test [POST]
+func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection, err := dsHelper.ConnApi.FindByPk(input)
+	if err != nil {
+		return nil, err
+	}
+	// test connection
+	result, err := testConnection(context.TODO(), connection.BitbucketServerConn)
+	if err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil
+}
+
+// @Summary create bitbucket connection
+// @Description Create bitbucket connection
+// @Tags plugins/bitbucket_server
+// @Param body body models.BitbucketServerConnection true "json body"
+// @Success 200  {object} models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections [POST]
+func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.Post(input)
+}
+
+// @Summary patch bitbucket connection
+// @Description Patch bitbucket connection
+// @Tags plugins/bitbucket_server
+// @Param body body models.BitbucketServerConnection true "json body"
+// @Success 200  {object} models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections/{connectionId} [PATCH]
+func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.Patch(input)
+}
+
+// @Summary delete a bitbucket connection
+// @Description Delete a bitbucket connection
+// @Tags plugins/bitbucket_server
+// @Success 200  {object} models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 409  {object} services.BlueprintProjectPairs "References exist to this connection"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections/{connectionId} [DELETE]
+func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.Delete(input)
+}
+
+// @Summary get all bitbucket connections
+// @Description Get all bitbucket connections
+// @Tags plugins/bitbucket_server
+// @Success 200  {object} []models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections [GET]
+func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.GetAll(input)
+}
+
+// @Summary get bitbucket connection detail
+// @Description Get bitbucket connection detail
+// @Tags plugins/bitbucket_server
+// @Success 200  {object} models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections/{connectionId} [GET]
+func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.GetDetail(input)
+}