Migrate to buf schema v2 and move api related tasks to dedicated Makefile for PMM v3

.github/workflows/main.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Update API compatibility descriptors
         run: |
           # log if descriptors changed, useful for "update descriptors" PRs
-          make descriptors
+          make -C api descriptors
           git diff --text
 
       - name: Run check-license

Makefile.include

@@ -1,5 +1,4 @@
-# vim: ts=8:sw=8:ft=make:noai:noet
-SWAGGER_UI_VERSION?=latest
+
 
 .PHONY: default init release gen test clean all
 default: help
@@ -25,8 +24,8 @@ release:              ## Build release versions of all components
 	make -C qan-api2 release
 
 gen: clean            ## Generate files
-	make gen-api
-	make clean_swagger
+	make -C api gen
+	make -C api clean-swagger
 
 	make -C agent gen
 	make -C admin gen
@@ -38,129 +37,13 @@ gen: clean            ## Generate files
 	make format ## TODO: One formatting run is not enough, figure out why.
 	go install -v ./...
 
+clean: 			  ## Remove generated files
+	make -C api clean
+
 gen-mocks:
 	find . -name mock_*.go -delete
 	./bin/mockery --config .mockery.yaml
 
-gen-api:            ## Generate PMM API
-	# generated by descriptors target
-	bin/buf breaking --against descriptor.bin api
-
-	bin/buf generate -v api
-
-	SPECS="\
-		api/agentlocal/v1 \
-		api/server/v1 \
-		api/user/v1 \
-		api/inventory/v1 \
-		api/management/v1 \
-		api/management/v1/service \
-		api/actions/v1 \
-		api/advisors/v1 \
-		api/alerting/v1 \
-		api/backup/v1 \
-		api/dump/v1beta1 \
-		api/accesscontrol/v1beta1 \
-		api/qan/v1 \
-		api/platform/v1"; \
-	for API in $$SPECS; do \
-		set -x ; \
-		bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
-		bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
-		bin/swagger validate $$API/json/$$(basename $$API).json ; \
-		bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
-			--additional-initialism=aws \
-			--additional-initialism=db \
-			--additional-initialism=ok \
-			--additional-initialism=pmm \
-			--additional-initialism=psmdb \
-			--additional-initialism=pxc \
-			--additional-initialism=pt \
-			--additional-initialism=qan \
-			--additional-initialism=rds \
-			--additional-initialism=sql \
-			--additional-initialism=ha ; \
-	done
-
-	# generate public API spec, omit agentlocal and inventory (always private),
-	# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
-	bin/swagger mixin --output=api/swagger/swagger.json \
-		api/swagger/header.json \
-		api/server/v1/json/v1.json \
-		api/user/v1/json/v1.json \
-		api/inventory/v1/json/v1.json \
-		api/management/v1/json/v1.json \
-		api/actions/v1/json/v1.json \
-		api/alerting/v1/json/v1.json \
-		api/advisors/v1/json/v1.json \
-		api/backup/v1/json/v1.json \
-		api/qan/v1/json/v1.json \
-		api/platform/v1/json/v1.json
-	bin/swagger validate api/swagger/swagger.json
-
-	bin/swagger-order --output=api/swagger/swagger.json api/swagger/swagger.json
-
-	# generate API spec with all PMM Server APIs (omit agentlocal)
-	bin/swagger mixin --output=api/swagger/swagger-dev.json \
-		api/swagger/header-dev.json \
-		api/server/v1/json/v1.json \
-		api/user/v1/json/v1.json \
-		api/inventory/v1/json/v1.json \
-		api/management/v1/json/v1.json \
-		api/actions/v1/json/v1.json \
-		api/alerting/v1/json/v1.json \
-		api/advisors/v1/json/v1.json \
-		api/backup/v1/json/v1.json \
-		api/dump/v1beta1/json/v1beta1.json \
-		api/accesscontrol/v1beta1/json/v1beta1.json \
-		api/qan/v1/json/v1.json \
-		api/platform/v1/json/v1.json
-
-	bin/swagger validate api/swagger/swagger-dev.json
-
-	bin/swagger-order --output=api/swagger/swagger-dev.json api/swagger/swagger-dev.json
-
-clean_swagger:
-	find api -name '*.swagger.json' -print -delete
-
-
-ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
-get_swagger_version:
-override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
-else
-get_swagger_version:
-endif
-
-update-swagger: get_swagger_version               ## Update Swagger UI in api/swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
-	curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
-	tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;api/swagger/;g'
-	rm -f api/swagger/.npmrc
-	sed -i 's/url: ".*"/url: "\/swagger.json",\n    validatorUrl: "none"/' api/swagger/swagger-initializer.js
-
-clean: clean_swagger  ## Remove generated files
-	find api -name '*.pb.go' -print -delete
-	find api -name '*.pb.gw.go' -print -delete
-	find api -name '*.validate.go' -print -delete
-
-	SPECS="\
-		api/agentlocal/v1 \
-		api/server/v1 \
-		api/user/v1 \
-		api/inventory/v1 \
-		api/management/v1 \
-		api/actions/v1 \
-		api/alerting/v1 \
-		api/advisors/v1 \
-		api/backup/v1 \
-		api/dump/v1beta1 \
-		api/accesscontrol/v1beta1 \
-		api/qan/v1 \
-		api/platform/v1"; \
-	for API in $$SPECS; do \
-		rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
-	done
-	rm -f api/swagger/swagger.json api/swagger/swagger-dev.json
-
 test-common:          ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed)
 	go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update)
 
@@ -182,14 +65,10 @@ check-all: check-license check    ## Run golangci linter to check for changes ag
 FILES = $(shell find . -type f -name '*.go')
 
 format:               ## Format source code
+	make -C api format
 	bin/gofumpt -l -w $(FILES)
 	bin/goimports -local github.com/percona/pmm -l -w $(FILES)
 	bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
-	bin/buf format api -w
 
 serve:                ## Serve API documentation with nginx
 	nginx -p . -c api/nginx/nginx.conf
-
-descriptors:          ## Update API compatibility descriptors
-	#./prototool break descriptor-set . -o api/api.descriptor
-	bin/buf build -o descriptor.bin --as-file-descriptor-set api

api/Makefile

@@ -0,0 +1,127 @@
+# vim: ts=8:sw=8:ft=make:noai:noet
+SWAGGER_UI_VERSION?=latest
+
+gen:            ## Generate PMM API
+	# generated by descriptors target
+	../bin/buf breaking --against descriptor.bin .
+
+	../bin/buf generate -v .
+
+	SPECS="\
+		agentlocal/v1 \
+		server/v1 \
+		user/v1 \
+		inventory/v1 \
+		management/v1 \
+		management/v1/service \
+		actions/v1 \
+		advisors/v1 \
+		alerting/v1 \
+		backup/v1 \
+		dump/v1beta1 \
+		accesscontrol/v1beta1 \
+		qan/v1 \
+		platform/v1"; \
+	for API in $$SPECS; do \
+		set -x ; \
+		../bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
+		../bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
+		../bin/swagger validate $$API/json/$$(basename $$API).json ; \
+		../bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
+			--additional-initialism=aws \
+			--additional-initialism=db \
+			--additional-initialism=ok \
+			--additional-initialism=pmm \
+			--additional-initialism=psmdb \
+			--additional-initialism=pxc \
+			--additional-initialism=pt \
+			--additional-initialism=qan \
+			--additional-initialism=rds \
+			--additional-initialism=sql \
+			--additional-initialism=ha ; \
+	done
+
+	# generate public API spec, omit agentlocal and inventory (always private),
+	# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
+	../bin/swagger mixin --output=swagger/swagger.json \
+		swagger/header.json \
+		server/v1/json/v1.json \
+		user/v1/json/v1.json \
+		inventory/v1/json/v1.json \
+		management/v1/json/v1.json \
+		actions/v1/json/v1.json \
+		alerting/v1/json/v1.json \
+		advisors/v1/json/v1.json \
+		backup/v1/json/v1.json \
+		qan/v1/json/v1.json \
+		platform/v1/json/v1.json
+	../bin/swagger validate swagger/swagger.json
+
+	../bin/swagger-order --output=swagger/swagger.json swagger/swagger.json
+
+	# generate API spec with all PMM Server APIs (omit agentlocal)
+	../bin/swagger mixin --output=swagger/swagger-dev.json \
+		swagger/header-dev.json \
+		server/v1/json/v1.json \
+		user/v1/json/v1.json \
+		inventory/v1/json/v1.json \
+		management/v1/json/v1.json \
+		actions/v1/json/v1.json \
+		alerting/v1/json/v1.json \
+		advisors/v1/json/v1.json \
+		backup/v1/json/v1.json \
+		dump/v1beta1/json/v1beta1.json \
+		accesscontrol/v1beta1/json/v1beta1.json \
+		qan/v1/json/v1.json \
+		platform/v1/json/v1.json
+
+	../bin/swagger validate swagger/swagger-dev.json
+
+	../bin/swagger-order --output=swagger/swagger-dev.json swagger/swagger-dev.json
+
+format:               ## Format API definitions
+	../bin/buf format . -w
+
+clean-swagger:
+	find . -name '*.swagger.json' -print -delete
+
+
+ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
+get_swagger_version:
+override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
+else
+get_swagger_version:
+endif
+
+update-swagger: get_swagger_version               ## Update Swagger UI in swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
+	curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
+	tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;swagger/;g'
+	rm -f swagger/.npmrc
+	sed -i 's/url: ".*"/url: "\/swagger.json",\n    validatorUrl: "none"/' swagger/swagger-initializer.js
+
+clean: clean-swagger  ## Remove generated files
+	find . -name '*.pb.go' -print -delete
+	find . -name '*.pb.gw.go' -print -delete
+	find . -name '*.validate.go' -print -delete
+
+	SPECS="\
+		agentlocal/v1 \
+		server/v1 \
+		user/v1 \
+		inventory/v1 \
+		management/v1 \
+		actions/v1 \
+		alerting/v1 \
+		advisors/v1 \
+		backup/v1 \
+		dump/v1beta1 \
+		accesscontrol/v1beta1 \
+		qan/v1 \
+		platform/v1"; \
+	for API in $$SPECS; do \
+		rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
+	done
+	rm -f swagger/swagger.json swagger/swagger-dev.json
+
+descriptors:          ## Update API compatibility descriptors
+	../bin/buf build -o descriptor.bin --as-file-descriptor-set .

api/buf.gen.yaml

@@ -0,0 +1,34 @@
+version: v2
+managed:
+  enabled: true
+  disable:
+    - file_option: go_package
+      module: buf.build/googleapis/googleapis
+    - file_option: go_package
+      module: buf.build/grpc-ecosystem/grpc-gateway
+    - file_option: go_package
+      module: buf.build/envoyproxy/protoc-gen-validate
+  override:
+    - file_option: go_package_prefix
+      value: github.com/percona/pmm/api
+plugins:
+  - local: ../bin/protoc-gen-go
+    out: .
+    opt: paths=source_relative
+  - local: ../bin/protoc-gen-go-grpc
+    out: .
+    opt: paths=source_relative
+  - local: ../bin/protoc-gen-grpc-gateway
+    out: .
+    opt: paths=source_relative
+  - local: ../bin/protoc-gen-validate
+    out: .
+    opt:
+      - paths=source_relative
+      - lang=go
+  - local: ../bin/protoc-gen-openapiv2
+    out: .
+    opt:
+      - simple_operation_ids=true
+      - json_names_for_fields=false
+      - proto3_optional_nullable=true

api/buf.lock

@@ -1,18 +1,12 @@
 # Generated by buf. DO NOT EDIT.
-version: v1
+version: v2
 deps:
-  - remote: buf.build
-    owner: envoyproxy
-    repository: protoc-gen-validate
+  - name: buf.build/envoyproxy/protoc-gen-validate
     commit: eac44469a7af47e7839a7f1f3d7ac004
-    digest: shake256:0feabcde01b6b11e3c75a5e3f807968d5995626546f39c37e5d4205892b3a59cced0ed83b35a2eb9e6dddd3309660ad46b737c9dcd224b425de0a6654ce04417
-  - remote: buf.build
-    owner: googleapis
-    repository: googleapis
+    digest: b5:2ef7b25d9671fde82d58278e94d209788fb6a42936cc2d60e42303b67dcdb72b2d0e121238ba44caf1870a34edab55518df5d3f1ee1e62b8d9b6cf98817eae6e
+  - name: buf.build/googleapis/googleapis
     commit: 28151c0d0a1641bf938a7672c500e01d
-    digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de
-  - remote: buf.build
-    owner: grpc-ecosystem
-    repository: grpc-gateway
+    digest: b5:93b70089baa4fc05a92d3e52db91a4b7812db3b57b9664f6cb301733938cb630e377a938e8a56779388171c749c1d42a2e9a6c6230f2ff45f127a8102a6a27d0
+  - name: buf.build/grpc-ecosystem/grpc-gateway
     commit: 3f42134f4c564983838425bc43c7a65f
-    digest: shake256:3d11d4c0fe5e05fda0131afefbce233940e27f0c31c5d4e385686aea58ccd30f72053f61af432fa83f1fc11cda57f5f18ca3da26a29064f73c5a0d076bba8d92
+    digest: b5:291b947d8ac09492517557e4e72e294788cb8201afc7d0df7bda80fa10931adb60d4d669208a7696bf24f1ecb2a33a16d4c1e766e6f31809248b00343119569b