Merge branch 'v3' into PMM-13077-list-pmm-server-versions

.github/workflows/main.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Update API compatibility descriptors
         run: |
           # log if descriptors changed, useful for "update descriptors" PRs
-          make descriptors
+          make -C api descriptors
           git diff --text
 
       - name: Run check-license

Makefile.include

@@ -1,5 +1,4 @@
-# vim: ts=8:sw=8:ft=make:noai:noet
-SWAGGER_UI_VERSION?=latest
+
 
 .PHONY: default init release gen test clean all
 default: help
@@ -25,8 +24,8 @@ release:              ## Build release versions of all components
 	make -C qan-api2 release
 
 gen: clean            ## Generate files
-	make gen-api
-	make clean_swagger
+	make -C api gen
+	make -C api clean-swagger
 
 	make -C agent gen
 	make -C admin gen
@@ -38,129 +37,13 @@ gen: clean            ## Generate files
 	make format ## TODO: One formatting run is not enough, figure out why.
 	go install -v ./...
 
+clean: 			  ## Remove generated files
+	make -C api clean
+
 gen-mocks:
 	find . -name mock_*.go -delete
 	./bin/mockery --config .mockery.yaml
 
-gen-api:            ## Generate PMM API
-	# generated by descriptors target
-	bin/buf breaking --against descriptor.bin api
-
-	bin/buf generate -v api
-
-	SPECS="\
-		api/agentlocal/v1 \
-		api/server/v1 \
-		api/user/v1 \
-		api/inventory/v1 \
-		api/management/v1 \
-		api/management/v1/service \
-		api/actions/v1 \
-		api/advisors/v1 \
-		api/alerting/v1 \
-		api/backup/v1 \
-		api/dump/v1beta1 \
-		api/accesscontrol/v1beta1 \
-		api/qan/v1 \
-		api/platform/v1"; \
-	for API in $$SPECS; do \
-		set -x ; \
-		bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
-		bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
-		bin/swagger validate $$API/json/$$(basename $$API).json ; \
-		bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
-			--additional-initialism=aws \
-			--additional-initialism=db \
-			--additional-initialism=ok \
-			--additional-initialism=pmm \
-			--additional-initialism=psmdb \
-			--additional-initialism=pxc \
-			--additional-initialism=pt \
-			--additional-initialism=qan \
-			--additional-initialism=rds \
-			--additional-initialism=sql \
-			--additional-initialism=ha ; \
-	done
-
-	# generate public API spec, omit agentlocal and inventory (always private),
-	# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
-	bin/swagger mixin --output=api/swagger/swagger.json \
-		api/swagger/header.json \
-		api/server/v1/json/v1.json \
-		api/user/v1/json/v1.json \
-		api/inventory/v1/json/v1.json \
-		api/management/v1/json/v1.json \
-		api/actions/v1/json/v1.json \
-		api/alerting/v1/json/v1.json \
-		api/advisors/v1/json/v1.json \
-		api/backup/v1/json/v1.json \
-		api/qan/v1/json/v1.json \
-		api/platform/v1/json/v1.json
-	bin/swagger validate api/swagger/swagger.json
-
-	bin/swagger-order --output=api/swagger/swagger.json api/swagger/swagger.json
-
-	# generate API spec with all PMM Server APIs (omit agentlocal)
-	bin/swagger mixin --output=api/swagger/swagger-dev.json \
-		api/swagger/header-dev.json \
-		api/server/v1/json/v1.json \
-		api/user/v1/json/v1.json \
-		api/inventory/v1/json/v1.json \
-		api/management/v1/json/v1.json \
-		api/actions/v1/json/v1.json \
-		api/alerting/v1/json/v1.json \
-		api/advisors/v1/json/v1.json \
-		api/backup/v1/json/v1.json \
-		api/dump/v1beta1/json/v1beta1.json \
-		api/accesscontrol/v1beta1/json/v1beta1.json \
-		api/qan/v1/json/v1.json \
-		api/platform/v1/json/v1.json
-
-	bin/swagger validate api/swagger/swagger-dev.json
-
-	bin/swagger-order --output=api/swagger/swagger-dev.json api/swagger/swagger-dev.json
-
-clean_swagger:
-	find api -name '*.swagger.json' -print -delete
-
-
-ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
-get_swagger_version:
-override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
-else
-get_swagger_version:
-endif
-
-update-swagger: get_swagger_version               ## Update Swagger UI in api/swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
-	curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
-	tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;api/swagger/;g'
-	rm -f api/swagger/.npmrc
-	sed -i 's/url: ".*"/url: "\/swagger.json",\n    validatorUrl: "none"/' api/swagger/swagger-initializer.js
-
-clean: clean_swagger  ## Remove generated files
-	find api -name '*.pb.go' -print -delete
-	find api -name '*.pb.gw.go' -print -delete
-	find api -name '*.validate.go' -print -delete
-
-	SPECS="\
-		api/agentlocal/v1 \
-		api/server/v1 \
-		api/user/v1 \
-		api/inventory/v1 \
-		api/management/v1 \
-		api/actions/v1 \
-		api/alerting/v1 \
-		api/advisors/v1 \
-		api/backup/v1 \
-		api/dump/v1beta1 \
-		api/accesscontrol/v1beta1 \
-		api/qan/v1 \
-		api/platform/v1"; \
-	for API in $$SPECS; do \
-		rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
-	done
-	rm -f api/swagger/swagger.json api/swagger/swagger-dev.json
-
 test-common:          ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed)
 	go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update)
 
@@ -182,14 +65,10 @@ check-all: check-license check    ## Run golangci linter to check for changes ag
 FILES = $(shell find . -type f -name '*.go')
 
 format:               ## Format source code
+	make -C api format
 	bin/gofumpt -l -w $(FILES)
 	bin/goimports -local github.com/percona/pmm -l -w $(FILES)
 	bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
-	bin/buf format api -w
 
 serve:                ## Serve API documentation with nginx
 	nginx -p . -c api/nginx/nginx.conf
-
-descriptors:          ## Update API compatibility descriptors
-	#./prototool break descriptor-set . -o api/api.descriptor
-	bin/buf build -o descriptor.bin --as-file-descriptor-set api

agent/runner/actions/mongodb_explain_action.go

@@ -18,9 +18,9 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
-	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -39,6 +39,15 @@ type mongodbExplainAction struct {
 	dsn     string
 }
 
+type explain struct {
+	Ns                 string `json:"ns"`
+	Op                 string `json:"op"`
+	Query              bson.D `json:"query,omitempty"`
+	Command            bson.D `json:"command,omitempty"`
+	OriginatingCommand bson.D `json:"originatingCommand,omitempty"`
+	UpdateObj          bson.D `json:"updateobj,omitempty"`
+}
+
 var errCannotExplain = fmt.Errorf("cannot explain this type of query")
 
 // NewMongoDBExplainAction creates a MongoDB EXPLAIN query Action.
@@ -89,28 +98,165 @@ func (a *mongodbExplainAction) Run(ctx context.Context) ([]byte, error) {
 	}
 	defer client.Disconnect(ctx) //nolint:errcheck
 
-	var eq proto.ExampleQuery
+	return explainForQuery(ctx, client, a.params.Query)
+}
+
+func (a *mongodbExplainAction) sealed() {}
+
+func (e explain) prepareCommand() (bson.D, error) {
+	command := e.Command
+
+	switch e.Op {
+	case "query":
+		if len(command) == 0 {
+			command = e.Query
+		}
+
+		if len(command) == 0 || command[0].Key != "find" {
+			return bson.D{
+				{Key: "find", Value: e.getCollection()},
+				{Key: "filter", Value: command},
+			}, nil
+		}
+
+		if len(command) != 0 && command[0].Key == "query" {
+			return bson.D{
+				{Key: "find", Value: e.getCollection()},
+				{Key: "filter", Value: command[0].Value},
+			}, nil
+		}
+
+		return dropDBField(command), nil
+	case "update":
+		if len(command) == 0 {
+			command = bson.D{
+				{Key: "q", Value: e.Query},
+				{Key: "u", Value: e.UpdateObj},
+			}
+		}
+
+		return bson.D{
+			{Key: "update", Value: e.getCollection()},
+			{Key: "updates", Value: []any{command}},
+		}, nil
+	case "remove":
+		if len(command) == 0 {
+			command = bson.D{{Key: "q", Value: e.Query}}
+		}
+
+		return bson.D{
+			{Key: "delete", Value: e.getCollection()},
+			{Key: "deletes", Value: []any{command}},
+		}, nil
+	case "getmore":
+		if len(e.OriginatingCommand) == 0 {
+			return bson.D{{Key: "getmore", Value: ""}}, nil
+		}
+
+		command = e.OriginatingCommand
+
+		return dropDBField(command), nil
+	case "command":
+		command = dropDBField(command)
+
+		if len(command) == 0 {
+			return command, nil
+		}
+
+		switch command[0].Key {
+		// Not supported commands.
+		case "dbStats":
+			return nil, errors.Errorf("command %s is not supported for explain", command[0].Key)
+		case "group":
+		default:
+			return command, nil
+		}
+
+		return fixReduceField(command), nil
+	// Not supported operations.
+	case "insert", "drop":
+		return nil, errors.Errorf("operation %s is not supported for explain", e.Op)
+	}
+
+	return command, nil
+}
+
+func (e explain) getDB() string {
+	s := strings.SplitN(e.Ns, ".", 2)
+	if len(s) == 2 {
+		return s[0]
+	}
+
+	return ""
+}
+
+func (e explain) getCollection() string {
+	s := strings.SplitN(e.Ns, ".", 2)
+	if len(s) == 2 {
+		return s[1]
+	}
+
+	return ""
+}
+
+// dropDBField remove DB field to be able run explain on all supported types.
+// Otherwise it could end up with BSON field 'xxx.$db' is a duplicate field.
+func dropDBField(command bson.D) bson.D {
+	for i := range command {
+		if command[i].Key != "$db" {
+			continue
+		}
+
+		if len(command)-1 == i {
+			return command[:i]
+		}
+
+		return append(command[:i], command[i+1:]...)
+	}
+
+	return command
+}
+
+// fixReduceField fixing nil/empty values after unmarshalling funcs.
+func fixReduceField(command bson.D) bson.D {
+	var group bson.D
+	var ok bool
+	if group, ok = command[0].Value.(bson.D); !ok {
+		return command
+	}
 
-	err = bson.UnmarshalExtJSON([]byte(a.params.Query), true, &eq)
+	for i := range group {
+		if group[i].Key == "$reduce" {
+			group[i].Value = "{}"
+			command[0].Value = group
+			break
+		}
+	}
+
+	return command
+}
+
+func explainForQuery(ctx context.Context, client *mongo.Client, query string) ([]byte, error) {
+	var e explain
+	err := bson.UnmarshalExtJSON([]byte(query), false, &e)
 	if err != nil {
-		return nil, errors.Wrapf(err, "Query: %s", a.params.Query)
+		return nil, errors.Wrapf(err, "Query: %s", query)
 	}
 
-	database := "admin"
-	if eq.Db() != "" {
-		database = eq.Db()
+	preparedCommand, err := e.prepareCommand()
+	if err != nil {
+		return nil, errors.Wrap(errCannotExplain, err.Error())
 	}
-	res := client.Database(database).RunCommand(ctx, eq.ExplainCmd())
+	command := bson.D{{Key: "explain", Value: preparedCommand}}
+	res := client.Database(e.getDB()).RunCommand(ctx, command)
 	if res.Err() != nil {
 		return nil, errors.Wrap(errCannotExplain, res.Err().Error())
 	}
 
-	result, err := res.DecodeBytes()
+	result, err := res.Raw()
 	if err != nil {
 		return nil, err
 	}
-	// We need it because result
+
 	return []byte(result.String()), nil
 }
-
-func (a *mongodbExplainAction) sealed() {}