From 74ea8541bb2300b49640e129d3202279a2025ed4 Mon Sep 17 00:00:00 2001 From: Joseph Anttila Hall Date: Fri, 8 Mar 2024 15:10:49 -0800 Subject: [PATCH] Change Dockerfiles (and make targets) to use cross-compile. Continue supporting ARCH env for backward compat, but introduce {BUILDARCH, TARGETARCH}. --- .github/workflows/e2e.yaml | 12 +++--- Makefile | 41 +++++++++++-------- artifacts/images/agent-build.Dockerfile | 7 ++-- artifacts/images/server-build.Dockerfile | 7 ++-- artifacts/images/test-client-build.Dockerfile | 7 ++-- artifacts/images/test-server-build.Dockerfile | 7 ++-- examples/kind/konnectivity-agent-ds.yaml | 3 +- examples/kind/konnectivity-server.yaml | 3 +- 8 files changed, 50 insertions(+), 37 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 54cee63e3..dbf1672ad 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -36,10 +36,10 @@ jobs: run: | mkdir _output - docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master -f artifacts/images/agent-build.Dockerfile --build-arg GO_TOOLCHAIN=${{ env.GO_TOOLCHAIN }} --build-arg GO_VERSION=${{ env.GO_VERSION }} --build-arg BASEIMAGE=${{ env.BASEIMAGE }} . - docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar - docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-server:master -f artifacts/images/server-build.Dockerfile --build-arg GO_TOOLCHAIN=${{ env.GO_TOOLCHAIN }} --build-arg GO_VERSION=${{ env.GO_VERSION }} --build-arg BASEIMAGE=${{ env.BASEIMAGE }} . - docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > _output/konnectivity-server.tar + make docker-build/proxy-agent-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }} + docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local > _output/konnectivity-agent.tar + make docker-build/proxy-server-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }} + docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local > _output/konnectivity-server.tar - uses: actions/upload-artifact@v2 with: @@ -150,8 +150,8 @@ jobs: # preload konnectivity images docker load --input konnectivity-server.tar docker load --input konnectivity-agent.tar - /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server:master --name ${{ env.KIND_CLUSTER_NAME}} - /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master --name ${{ env.KIND_CLUSTER_NAME}} + /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local --name ${{ env.KIND_CLUSTER_NAME}} + /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local --name ${{ env.KIND_CLUSTER_NAME}} kubectl apply -f examples/kind/konnectivity-server.yaml kubectl apply -f examples/kind/konnectivity-agent-ds.yaml diff --git a/Makefile b/Makefile index 868de2ae5..e2781a378 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +# ARCH specifies the golang target platform, for docker build +# targetes. It is supported for backward compatibility, but prefer +# BUILDARCH and TARGETARCH. ARCH ?= amd64 +# BUILDARCH specifies the golang build platform, for docker build targets. +BUILDARCH ?= amd64 +# TARGETARCH specifies the golang target platform, for docker build targets. +TARGETARCH ?= $(ARCH) ARCH_LIST ?= amd64 arm arm64 ppc64le s390x RELEASE_ARCH_LIST = amd64 arm64 # The output type could either be docker (local), or registry. @@ -201,53 +208,53 @@ docker-push: docker-push/proxy-agent docker-push/proxy-server .PHONY: docker-build/proxy-agent docker-build/proxy-agent: cmd/agent/main.go proto/agent/agent.pb.go buildx-setup @[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 ) - echo "Building proxy-agent for ${ARCH}" - ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/agent-build.Dockerfile -t ${AGENT_FULL_IMAGE}-$(ARCH):${TAG} + echo "Building proxy-agent with ${BUILDARCH} for ${TARGETARCH}" + ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/agent-build.Dockerfile -t ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG} .PHONY: docker-push/proxy-agent docker-push/proxy-agent: docker-build/proxy-agent @[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 ) - ${DOCKER_CMD} push ${AGENT_FULL_IMAGE}-$(ARCH):${TAG} + ${DOCKER_CMD} push ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG} .PHONY: docker-build/proxy-server docker-build/proxy-server: cmd/server/main.go proto/agent/agent.pb.go buildx-setup @[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 ) - echo "Building proxy-server for ${ARCH}" - ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/server-build.Dockerfile -t ${SERVER_FULL_IMAGE}-$(ARCH):${TAG} + echo "Building proxy-server with ${BUILDARCH} for ${TARGETARCH}" + ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/server-build.Dockerfile -t ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG} .PHONY: docker-push/proxy-server docker-push/proxy-server: docker-build/proxy-server @[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 ) - ${DOCKER_CMD} push ${SERVER_FULL_IMAGE}-$(ARCH):${TAG} + ${DOCKER_CMD} push ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG} .PHONY: docker-build/proxy-test-client docker-build/proxy-test-client: cmd/test-client/main.go proto/agent/agent.pb.go buildx-setup @[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 ) - echo "Building proxy-test-client for ${ARCH}" - ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-client-build.Dockerfile -t ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG} + echo "Building proxy-test-client with ${BUILDARCH} for ${TARGETARCH}" + ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-client-build.Dockerfile -t ${TEST_CLIENT_FULL_IMAGE}-$(TARGETARCH):${TAG} .PHONY: docker-push/proxy-test-client docker-push/proxy-test-client: docker-build/proxy-test-client @[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 ) - ${DOCKER_CMD} push ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG} + ${DOCKER_CMD} push ${TEST_CLIENT_FULL_IMAGE}-$(TARGETARCH):${TAG} .PHONY: docker-build/http-test-server docker-build/http-test-server: cmd/test-server/main.go buildx-setup @[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 ) - echo "Building http-test-server for ${ARCH}" - ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-server-build.Dockerfile -t ${TEST_SERVER_FULL_IMAGE}-$(ARCH):${TAG} + echo "Building http-test-server with ${BUILDARCH} for ${TARGETARCH}" + ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-server-build.Dockerfile -t ${TEST_SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG} .PHONY: docker-push/http-test-server docker-push/http-test-server: docker-build/http-test-server @[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 ) - ${DOCKER_CMD} push ${TEST_SERVER_FULL_IMAGE}-$(ARCH):${TAG} + ${DOCKER_CMD} push ${TEST_SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG} ## -------------------------------------- ## Docker — All ARCH ## -------------------------------------- # As `docker buildx` is time and resource consuming, if not necessary, building specific arch images, -# like `make docker-build-arch-aamd64`, is recommended. +# like `make docker-build-arch-amd64`, is recommended. .PHONY: docker-build-all docker-build-all: $(addprefix docker-build-arch-,$(ARCH_LIST)) @@ -261,16 +268,16 @@ docker-build-arch-%: $(MAKE) docker-build/proxy-server-$* docker-build/proxy-agent-%: - $(MAKE) ARCH=$* docker-build/proxy-agent + $(MAKE) TARGETARCH=$* docker-build/proxy-agent docker-push/proxy-agent-%: - $(MAKE) ARCH=$* docker-push/proxy-agent + $(MAKE) TARGETARCH=$* docker-push/proxy-agent docker-build/proxy-server-%: - $(MAKE) ARCH=$* docker-build/proxy-server + $(MAKE) TARGETARCH=$* docker-build/proxy-server docker-push/proxy-server-%: - $(MAKE) ARCH=$* docker-push/proxy-server + $(MAKE) TARGETARCH=$* docker-push/proxy-server .PHONY: docker-push-manifest/proxy-agent docker-push-manifest/proxy-agent: ## Push the fat manifest docker image. diff --git a/artifacts/images/agent-build.Dockerfile b/artifacts/images/agent-build.Dockerfile index 7a1ace9bb..08fd3dbcd 100644 --- a/artifacts/images/agent-build.Dockerfile +++ b/artifacts/images/agent-build.Dockerfile @@ -1,10 +1,11 @@ # Build the proxy-agent binary +ARG BUILDARCH ARG GO_TOOLCHAIN ARG GO_VERSION ARG BASEIMAGE -FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder +FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder # Copy in the go src WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy @@ -26,8 +27,8 @@ COPY proto/ proto/ # Build -ARG ARCH -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-agent sigs.k8s.io/apiserver-network-proxy/cmd/agent +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-agent sigs.k8s.io/apiserver-network-proxy/cmd/agent FROM ${BASEIMAGE} diff --git a/artifacts/images/server-build.Dockerfile b/artifacts/images/server-build.Dockerfile index 08cf88699..d20657863 100644 --- a/artifacts/images/server-build.Dockerfile +++ b/artifacts/images/server-build.Dockerfile @@ -1,10 +1,11 @@ # Build the proxy-server binary +ARG BUILDARCH ARG GO_TOOLCHAIN ARG GO_VERSION ARG BASEIMAGE -FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder +FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder # Copy in the go src WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy @@ -25,8 +26,8 @@ COPY cmd/ cmd/ COPY proto/ proto/ # Build -ARG ARCH -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-server sigs.k8s.io/apiserver-network-proxy/cmd/server +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-server sigs.k8s.io/apiserver-network-proxy/cmd/server FROM ${BASEIMAGE} diff --git a/artifacts/images/test-client-build.Dockerfile b/artifacts/images/test-client-build.Dockerfile index bd2431426..3d936bd30 100644 --- a/artifacts/images/test-client-build.Dockerfile +++ b/artifacts/images/test-client-build.Dockerfile @@ -1,10 +1,11 @@ # Build the client binary +ARG BUILDARCH ARG GO_TOOLCHAIN ARG GO_VERSION ARG BASEIMAGE -FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder +FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder # Copy in the go src WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy @@ -25,8 +26,8 @@ COPY cmd/ cmd/ COPY proto/ proto/ # Build -ARG ARCH -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-test-client sigs.k8s.io/apiserver-network-proxy/cmd/test-client +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-test-client sigs.k8s.io/apiserver-network-proxy/cmd/test-client FROM ${BASEIMAGE} diff --git a/artifacts/images/test-server-build.Dockerfile b/artifacts/images/test-server-build.Dockerfile index 4cadda109..139eaa586 100644 --- a/artifacts/images/test-server-build.Dockerfile +++ b/artifacts/images/test-server-build.Dockerfile @@ -1,10 +1,11 @@ # Build the http test server binary +ARG BUILDARCH ARG GO_TOOLCHAIN ARG GO_VERSION ARG BASEIMAGE -FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder +FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder # Copy in the go src WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy @@ -24,8 +25,8 @@ COPY pkg/ pkg/ COPY cmd/ cmd/ # Build -ARG ARCH -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o http-test-server sigs.k8s.io/apiserver-network-proxy/cmd/test-server +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o http-test-server sigs.k8s.io/apiserver-network-proxy/cmd/test-server FROM ${BASEIMAGE} diff --git a/examples/kind/konnectivity-agent-ds.yaml b/examples/kind/konnectivity-agent-ds.yaml index 98f7cbd29..3ef45d555 100644 --- a/examples/kind/konnectivity-agent-ds.yaml +++ b/examples/kind/konnectivity-agent-ds.yaml @@ -36,7 +36,8 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: konnectivity-agent-container - image: gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master + image: gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local + imagePullPolicy: Never resources: requests: cpu: 50m diff --git a/examples/kind/konnectivity-server.yaml b/examples/kind/konnectivity-server.yaml index ffb2d198e..cfb7e3738 100644 --- a/examples/kind/konnectivity-server.yaml +++ b/examples/kind/konnectivity-server.yaml @@ -55,7 +55,8 @@ spec: hostNetwork: true containers: - name: konnectivity-server-container - image: gcr.io/k8s-staging-kas-network-proxy/proxy-server:master + image: gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local + imagePullPolicy: Never resources: requests: cpu: 1m