Merge branch 'v3' into v3

.github/workflows/main.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Update API compatibility descriptors
         run: |
           # log if descriptors changed, useful for "update descriptors" PRs
-          make descriptors
+          make -C api descriptors
           git diff --text
 
       - name: Run check-license

.gitignore

@@ -42,3 +42,5 @@ cli-tests/playwright-report/
 
 api-tests/pmm-api-tests-output.txt
 api-tests/pmm-api-tests-junit-report.xml
+
+packer.log

CONTRIBUTING.md

@@ -31,7 +31,7 @@ This project is built from several repositories:
 #### Frontends
 
 * [percona/grafana-dashboards](https://github.com/percona/grafana-dashboards) PMM dashboards for database monitoring
-* [percona-platform/grafana](https://github.com/percona-platform/grafana) user interface for PMM
+* [percona/grafana](https://github.com/percona/grafana) user interface for PMM
 
 ### PMM Client
 
@@ -44,7 +44,6 @@ This project is built from several repositories:
 * [percona/proxysql_exporter](https://github.com/percona/proxysql_exporter) exports ProxySQL server's metrics
 * [percona/rds_exporter](https://github.com/percona/rds_exporter) exports metrics from RDS
 * [percona/azure_exporter](https://github.com/percona/azure_metrics_exporter) exports metrics from Azure
-* [Percona-Lab/clickhouse_exporter](https://github.com/Percona-Lab/clickhouse_exporter) exports metrics from ClickHouse
 * [percona/percona-toolkit](https://github.com/percona/percona-toolkit) is a collection of advanced command-line tools to perform a variety of MySQL and system tasks that are too difficult or complex to perform manually
 
 
@@ -117,8 +116,8 @@ Since PMM has a lot of components, we will mention only three big parts of it.
 ### PMM Server
 
 * Clone [pmm repository](https://github.com/percona/pmm)
-* Run `make env-up` to start development container. This will be slow on first run, all consequent calls will be order of magnitude faster, because development container will be reused. From time to time it is recommended to perform container rebuild to pull the latest changes, for that run `make env-up-rebuild`.
-* To run pmm-managed with a new changes just run `make env TARGET="run-managed"`, it updates `pmm-managed` running in container.
+* Run `make env-up` to start development container. This will be slow on first run, all subsequent runs will be order of magnitude faster, because development container will be reused. From time to time it is recommended to rebuild the container to pull the latest changes by running `make env-up-rebuild`.
+* To run pmm-managed with your code changes, just run `make run-managed`. It updates `pmm-managed` running in the container.
 
 ### PMM Client
 
@@ -127,11 +126,11 @@ Since PMM has a lot of components, we will mention only three big parts of it.
 * Run `make setup-dev` to connect pmm-agent to PMM Server.
   * This command will register local pmm-agent to PMM Server and generate config file `pmm-agent-dev.yaml`
 * Once it's connected just use `make run` to run pmm-agent.
-* To work correctly pmm-agent needs vmagent and exporters installed on the system.
-  * First option is just install pmm-client using this instrucion https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#install. It will install all exporters as well.
+* To work correctly, pmm-agent needs vmagent and exporters installed on the system.
+  * The first option is to install pmm-client using this instrucion https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#install. It will install all exporters as well.
   * Another option is to do it manually
-    * vmagent and exporters can be installed by building each of them or by downloading the pmm-client tarball from [percona.com](https://www.percona.com/downloads/pmm/) and copying binaries to the exporters_base directory configured in a `pmm-agent-dev.yaml` file.
-    * All paths to exporters binaries are configured in `pmm-agent-dev.yaml`, so they can be changed manually
+    * vmagent and exporters can be installed by building each of them or by downloading the pmm-client tarball from [percona.com](https://www.percona.com/downloads/pmm2/) and copying binaries to the exporters_base directory configured in `pmm-agent-dev.yaml` file.
+    * All paths to exporter binaries are configured in `pmm-agent-dev.yaml`, so they can be changed manually if necessary.
 
 ### Exporters
 
@@ -143,20 +142,24 @@ See [Grafana Dashboards Contribution Guide](https://github.com/percona/grafana-d
 
 ## Tests
 
-In a PMM we have 3 kind of tests.
+In PMM we have 3 kinds of tests:
+
+  - unit tests
+  - API tests
+  - end-to-end, or e2e, tests
 
 ### Unit tests
 
-The first one is a Unit testing, so we have unit tests in each repository mentioned above. each of repositories has it's own instruction how to run unit tests.
+Each repository mentioned above has its own set of unit tests, as well as its own instruction on how to run unit tests.
 
 ### API tests
 
-API tests are included into pmm repository and located in [api-tests directory](https://github.com/percona/pmm/tree/main/api-tests). API tests runs against running PMM Server container.
+API tests are part of the PMM repository and can be found in [api-tests directory](https://github.com/percona/pmm/tree/main/api-tests). API tests run inside of an active PMM Server container.
 
-### End to End (E2E) tests
+### End-to-end (E2E) tests
 
-End to End tests are located in [pmm-qa repository](https://github.com/percona/pmm-qa). They includes UI tests and CLI tests.
-Please see [readme](https://github.com/percona/pmm-qa#readme) for details on how to run these.
+End-to-end tests are located in [pmm-qa repository](https://github.com/percona/pmm-qa). They include UI tests and CLI tests.
+Please refer to [readme](https://github.com/percona/pmm-qa#readme) for details on how to run these.
 
 ## Submitting a Pull Request
 
@@ -165,49 +168,53 @@ Before proceeding with your first pull request, we highly recommend you to read
 - [Tech stack](docs/process/tech_stack.md)
 - [Best practices](docs/process/best_practices.md)
 
-As a PR created you are responsible to:
-* make sure PR is ready (linted, tested and etc)
-* make sure it is reviewed (ask for review, ping reviewers and etc)
-* make sure it is merged
-  * merge when it is reviewed and tested
-  * ask code owners/admins to merge it if merging is blocked for some reason
+Once your PR is created, please do the following:
+* prepare your PR for review
+  * run code syntax checks, or linters
+  * run tests and make sure they all pass
+* pass the review (ask for review, ping reviewers)
+* then merge it
+  * ask code owners or admins to merge it if merging is blocked for some reason
 
 ## Feature Build
 
-PMM is quite complex project, it consists from many different repos descibed above. Feature Build (FB) is a way to get changes all together, build them all together, run tests and get client and server containers.
+PMM is quite a complex project, it consists of many different repos described above. A Feature Build (FB) is a way to put everything together, build all components, run tests and, finally, build client and server containers.
 
 Please see: [How to create a feature build](https://github.com/Percona-Lab/pmm-submodules/blob/v3/README.md#how-to-create-a-feature-build)
 
-### The Goals of the Feature Builds
+### The Goals of Feature Builds
 
-1. Provide an easy way to test/accept functionality for PO/PM and QA
-2. Inform the Developer about Automation Tests results before the code is merged
-3. (Future) Let the Developers add/modify e2e tests when they change functionality
+1. Provide a way to have the functionality tested by the developer and QA (or other PMM team members)
+2. Inform the Developer about Automation Test results before the code is merged
+3. Let the Developers add or modify e2e tests whenever there are functional changes
 
 ### The Rules
 
-1. Start Feature Build for every feature/improvement you are working on.
-2. Start PullRequest to percona-lab/pmm-submodules as DRAFT.
-3. Change the status of Pull Request from Draft to Open ONLY if your changes must be merged to pmm-submodules.
-4. Include a short explanation in the Long Description field of the Feature in PR for feature build and checkboxes to all related Pull Requests. Check other [PRs](https://github.com/Percona-Lab/pmm-submodules/pulls) as examples.
+1. Create a Feature Build for every feature/improvement/bugfix you are working on.
+2. Create a draft Pull Request in https://percona-lab/pmm-submodules.
+3. Change the status of the Pull Request from Draft to Open ONLY if you are contributing code changes to pmm-submodules (very rare).
+4. Provide a short explanation in the Description field of you feature build PR and checkboxes to all related Pull Requests. If you need examples, check out [PRs](https://github.com/Percona-Lab/pmm-submodules/pulls) made by others.
 5. After all related PRs in feature build are merged you should:
    a. either close the PR and delete the branch (this is the default option) or
-   b. merge the PR to pmm-submodules repository (please note, this rarely needs to be merged, for example infrastructure changes do)
+   b. merge the PR to pmm-submodules repository (please note, this rarely needs to be merged, for example infrastructure changes)
 
 ## Code Reviews
 
-There are number of approaches for the code review and ownership: Code Ownership (CODEOWNERS), [github auto review](https://docs.github.com/en/github/setting-up-and-managing-organizations-and-teams/managing-code-review-assignment-for-your-team), PR owner assign ppl that are better fit for the particular code/job.
+There is a number of approaches we use for the code review and ownership: 
+
+- code ownership, which is enforced via github's CODEOWNERS file
+- github [code review assignment](https://docs.github.com/en/github/setting-up-and-managing-organizations-and-teams/managing-code-review-assignment-for-your-team)
+- finally, a PR owner can manually assign reviewers (usually one or more PMM team members).
 
-For more efficient review process we use a mixed approach:
-* repos that have CODEOWNERS
+To make the review process effective, we use a mixed approach:
+* for repos that have CODEOWNERS
   * github will assign reviewers automatically
-* repos that don't have CODEOWNERS
+* for repos that don't have CODEOWNERS
   * add reviewers as follows:
       * add `pmm-review-fe` for UI/UX reviews
       * add `pmm-review-exporters` for exporter reviews [see PMM Client](#PMM-Client)
       * add `pmm-review-be` for backend reviews
-* if you know exactly who should review your code
-  * add them to the review
+* if you know exactly who should review your code, add them to the review
 
 
 | Team                 | Description                                                    | Members |

Dockerfile

@@ -1,6 +1,6 @@
 # This Dockerfile is used only for API tests.
 
-FROM golang:1.22
+FROM golang:1.23
 
 RUN export GOPATH=$(go env GOPATH) && \
     mkdir -p $GOPATH/src/github.com/percona/pmm

Makefile.include

@@ -1,5 +1,4 @@
-# vim: ts=8:sw=8:ft=make:noai:noet
-SWAGGER_UI_VERSION?=latest
+
 
 .PHONY: default init release gen test clean all
 default: help
@@ -25,8 +24,8 @@ release:              ## Build release versions of all components
 	make -C qan-api2 release
 
 gen: clean            ## Generate files
-	make gen-api
-	make clean_swagger
+	make -C api gen
+	make -C api clean-swagger
 
 	make -C agent gen
 	make -C admin gen
@@ -38,129 +37,13 @@ gen: clean            ## Generate files
 	make format ## TODO: One formatting run is not enough, figure out why.
 	go install -v ./...
 
+clean: 			  ## Remove generated files
+	make -C api clean
+
 gen-mocks:
 	find . -name mock_*.go -delete
 	./bin/mockery --config .mockery.yaml
 
-gen-api:            ## Generate PMM API
-	# generated by descriptors target
-	bin/buf breaking --against descriptor.bin api
-
-	bin/buf generate -v api
-
-	SPECS="\
-		api/agentlocal/v1 \
-		api/server/v1 \
-		api/user/v1 \
-		api/inventory/v1 \
-		api/management/v1 \
-		api/management/v1/service \
-		api/actions/v1 \
-		api/advisors/v1 \
-		api/alerting/v1 \
-		api/backup/v1 \
-		api/dump/v1beta1 \
-		api/accesscontrol/v1beta1 \
-		api/qan/v1 \
-		api/platform/v1"; \
-	for API in $$SPECS; do \
-		set -x ; \
-		bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
-		bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
-		bin/swagger validate $$API/json/$$(basename $$API).json ; \
-		bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
-			--additional-initialism=aws \
-			--additional-initialism=db \
-			--additional-initialism=ok \
-			--additional-initialism=pmm \
-			--additional-initialism=psmdb \
-			--additional-initialism=pxc \
-			--additional-initialism=pt \
-			--additional-initialism=qan \
-			--additional-initialism=rds \
-			--additional-initialism=sql \
-			--additional-initialism=ha ; \
-	done
-
-	# generate public API spec, omit agentlocal and inventory (always private),
-	# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
-	bin/swagger mixin --output=api/swagger/swagger.json \
-		api/swagger/header.json \
-		api/server/v1/json/v1.json \
-		api/user/v1/json/v1.json \
-		api/inventory/v1/json/v1.json \
-		api/management/v1/json/v1.json \
-		api/actions/v1/json/v1.json \
-		api/alerting/v1/json/v1.json \
-		api/advisors/v1/json/v1.json \
-		api/backup/v1/json/v1.json \
-		api/qan/v1/json/v1.json \
-		api/platform/v1/json/v1.json
-	bin/swagger validate api/swagger/swagger.json
-
-	bin/swagger-order --output=api/swagger/swagger.json api/swagger/swagger.json
-
-	# generate API spec with all PMM Server APIs (omit agentlocal)
-	bin/swagger mixin --output=api/swagger/swagger-dev.json \
-		api/swagger/header-dev.json \
-		api/server/v1/json/v1.json \
-		api/user/v1/json/v1.json \
-		api/inventory/v1/json/v1.json \
-		api/management/v1/json/v1.json \
-		api/actions/v1/json/v1.json \
-		api/alerting/v1/json/v1.json \
-		api/advisors/v1/json/v1.json \
-		api/backup/v1/json/v1.json \
-		api/dump/v1beta1/json/v1beta1.json \
-		api/accesscontrol/v1beta1/json/v1beta1.json \
-		api/qan/v1/json/v1.json \
-		api/platform/v1/json/v1.json
-
-	bin/swagger validate api/swagger/swagger-dev.json
-
-	bin/swagger-order --output=api/swagger/swagger-dev.json api/swagger/swagger-dev.json
-
-clean_swagger:
-	find api -name '*.swagger.json' -print -delete
-
-
-ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
-get_swagger_version:
-override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
-else
-get_swagger_version:
-endif
-
-update-swagger: get_swagger_version               ## Update Swagger UI in api/swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
-	curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
-	tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;api/swagger/;g'
-	rm -f api/swagger/.npmrc
-	sed -i 's/url: ".*"/url: "\/swagger.json",\n    validatorUrl: "none"/' api/swagger/swagger-initializer.js
-
-clean: clean_swagger  ## Remove generated files
-	find api -name '*.pb.go' -print -delete
-	find api -name '*.pb.gw.go' -print -delete
-	find api -name '*.validate.go' -print -delete
-
-	SPECS="\
-		api/agentlocal/v1 \
-		api/server/v1 \
-		api/user/v1 \
-		api/inventory/v1 \
-		api/management/v1 \
-		api/actions/v1 \
-		api/alerting/v1 \
-		api/advisors/v1 \
-		api/backup/v1 \
-		api/dump/v1beta1 \
-		api/accesscontrol/v1beta1 \
-		api/qan/v1 \
-		api/platform/v1"; \
-	for API in $$SPECS; do \
-		rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
-	done
-	rm -f api/swagger/swagger.json api/swagger/swagger-dev.json
-
 test-common:          ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed)
 	go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update)
 
@@ -182,14 +65,10 @@ check-all: check-license check    ## Run golangci linter to check for changes ag
 FILES = $(shell find . -type f -name '*.go')
 
 format:               ## Format source code
+	make -C api format
 	bin/gofumpt -l -w $(FILES)
 	bin/goimports -local github.com/percona/pmm -l -w $(FILES)
 	bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
-	bin/buf format api -w
 
 serve:                ## Serve API documentation with nginx
 	nginx -p . -c api/nginx/nginx.conf
-
-descriptors:          ## Update API compatibility descriptors
-	#./prototool break descriptor-set . -o api/api.descriptor
-	bin/buf build -o descriptor.bin --as-file-descriptor-set api

admin/commands/list.go

@@ -61,7 +61,7 @@ func (a listResultAgent) HumanReadableAgentType() string {
 }
 
 func (a listResultAgent) NiceAgentStatus() string {
-	res := a.Status
+	res, _ := strings.CutPrefix(a.Status, "AGENT_STATUS_")
 	if res == "" {
 		res = "unknown" //nolint:goconst
 	}

agent/agents/postgres/pgstatstatements/pgstatstatements.go

@@ -287,7 +287,7 @@ func (m *PGStatStatementsQAN) getNewBuckets(ctx context.Context, periodStart tim
 		return nil, err
 	}
 
-	buckets := makeBuckets(current, prev, m.disableCommentsParsing, m.l)
+	buckets := m.makeBuckets(current, prev)
 	startS := uint32(periodStart.Unix())
 	m.l.Debugf("Made %d buckets out of %d stat statements in %s+%d interval.",
 		len(buckets), len(current), periodStart.Format("15:04:05"), periodLengthSecs)
@@ -312,8 +312,9 @@ func (m *PGStatStatementsQAN) getNewBuckets(ctx context.Context, periodStart tim
 
 // makeBuckets uses current state of pg_stat_statements table and accumulated previous state
 // to make metrics buckets. It's a pure function for easier testing.
-func makeBuckets(current, prev statementsMap, disableCommentsParsing bool, l *logrus.Entry) []*agentv1.MetricsBucket {
+func (m *PGStatStatementsQAN) makeBuckets(current, prev statementsMap) []*agentv1.MetricsBucket {
 	res := make([]*agentv1.MetricsBucket, 0, len(current))
+	l := m.l
 
 	for queryID, currentPSS := range current {
 		prevPSS := prev[queryID]
@@ -339,10 +340,10 @@ func makeBuckets(current, prev statementsMap, disableCommentsParsing bool, l *lo
 		}
 
 		if len(currentPSS.Tables) == 0 {
-			currentPSS.Tables = extractTables(currentPSS.Query, l)
+			currentPSS.Tables = extractTables(currentPSS.Query, m.maxQueryLength, l)
 		}
 
-		if !disableCommentsParsing {
+		if !m.disableCommentsParsing {
 			comments, err := queryparser.PostgreSQLComments(currentPSS.Query)
 			if err != nil {
 				l.Errorf("failed to parse comments for query: %s", currentPSS.Query)

agent/agents/postgres/pgstatstatements/utils.go

@@ -72,9 +72,9 @@ func queryUsernames(q *reform.Querier) map[int64]string {
 	return res
 }
 
-func extractTables(query string, l *logrus.Entry) []string {
+func extractTables(query string, maxQueryLength int32, l *logrus.Entry) []string {
 	start := time.Now()
-	t, _ := truncate.Query(query, truncate.GetDefaultMaxQueryLength())
+	t, _ := truncate.Query(query, maxQueryLength)
 	tables, err := parser.ExtractTables(query)
 	if err != nil {
 		// log full query and error stack on debug level or more