Skip to content

Commit

Permalink
✨ test/e2e: use vSphere projects from Boskos (#3034)
Browse files Browse the repository at this point in the history
* test/e2e: use vSphere projects from Boskos

* review fixes
  • Loading branch information
sbueringer committed Jun 11, 2024
1 parent fb6f462 commit 996d756
Show file tree
Hide file tree
Showing 23 changed files with 1,794 additions and 142 deletions.
1 change: 0 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
.golangci.yml
bin/
**/*.yaml
hack/
out/
docs/
packaging/
Expand Down
43 changes: 41 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ GINKGO_NODES ?= 1
GINKGO_TIMEOUT ?= 3h
E2E_CONF_FILE ?= $(abspath test/e2e/config/vsphere.yaml)
E2E_CONF_OVERRIDE_FILE ?= $(abspath test/e2e/config/config-overrides.yaml)
E2E_IPAM_KUBECONFIG ?=
E2E_VSPHERE_IP_POOL ?=
E2E_TEMPLATE_DIR := $(abspath test/e2e/data/)
E2E_GOVMOMI_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-govmomi
E2E_SUPERVISOR_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-supervisor
Expand Down Expand Up @@ -186,6 +186,9 @@ IMPORT_BOSS_PKG := k8s.io/code-generator/cmd/import-boss

CAPI_HACK_TOOLS_VER := ef04465b2ba76214eea570e27e8146c96412e32a # Note: this is the commit ID of CAPI v1.7.1

BOSKOSCTL_BIN := boskosctl
BOSKOSCTL := $(abspath $(TOOLS_BIN_DIR)/$(BOSKOSCTL_BIN))

CONVERSION_VERIFIER_VER := $(CAPI_HACK_TOOLS_VER)
CONVERSION_VERIFIER_BIN := conversion-verifier
CONVERSION_VERIFIER := $(abspath $(TOOLS_BIN_DIR)/$(CONVERSION_VERIFIER_BIN)-$(CONVERSION_VERIFIER_VER))
Expand Down Expand Up @@ -230,6 +233,14 @@ VM_OPERATOR_ALL_ARCH = amd64 arm64
NET_OPERATOR_IMAGE_NAME ?= cluster-api-net-operator
NET_OPERATOR_IMG ?= $(STAGING_REGISTRY)/$(NET_OPERATOR_IMAGE_NAME)

# boskosctl
BOSKOSCTL_IMG ?= gcr.io/k8s-staging-capi-vsphere/extra/boskosctl
BOSKOSCTL_IMG_TAG ?= $(shell git describe --always --dirty)

# openvpn
OPENVPN_IMG ?= gcr.io/k8s-staging-capi-vsphere/extra/openvpn
OPENVPN_IMG_TAG ?= $(shell git describe --always --dirty)

# It is set by Prow GIT_TAG, a git-based tag of the form vYYYYMMDD-hash, e.g., v20210120-v0.3.10-308-gc61521971

TAG ?= dev
Expand Down Expand Up @@ -560,6 +571,28 @@ docker-build-net-operator: docker-pull-prerequisites ## Build the docker image f
$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(NETOP_DIR)/config/default/manager_pull_policy.yaml"; \
fi

.PHONY: docker-build-boskosctl
docker-build-boskosctl:
cat hack/tools/boskosctl/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) . -t $(BOSKOSCTL_IMG):$(BOSKOSCTL_IMG_TAG) --file -
docker tag $(BOSKOSCTL_IMG):$(BOSKOSCTL_IMG_TAG) $(BOSKOSCTL_IMG):latest
.PHONY: build

.PHONY: docker-push-boskosctl
docker-push-boskosctl:
docker push $(BOSKOSCTL_IMG):$(BOSKOSCTL_IMG_TAG)
docker push $(BOSKOSCTL_IMG):latest

.PHONY: docker-build-openvpn
docker-build-openvpn:
cat hack/tools/openvpn/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) . -t $(OPENVPN_IMG):$(OPENVPN_IMG_TAG) --file -
docker tag $(OPENVPN_IMG):$(OPENVPN_IMG_TAG) $(OPENVPN_IMG):latest
.PHONY: build

.PHONY: docker-push-openvpn
docker-push-openvpn:
docker push $(OPENVPN_IMG):$(OPENVPN_IMG_TAG)
docker push $(OPENVPN_IMG):latest

## --------------------------------------
## Testing
## --------------------------------------
Expand Down Expand Up @@ -621,7 +654,7 @@ e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
--e2e.artifacts-folder="$(ARTIFACTS)" \
--e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
--e2e.use-existing-cluster="$(USE_EXISTING_CLUSTER)" \
--e2e.ipam-kubeconfig="$(E2E_IPAM_KUBECONFIG)"
--e2e.ip-pool='$(E2E_VSPHERE_IP_POOL)'

## --------------------------------------
## Release
Expand Down Expand Up @@ -952,6 +985,9 @@ $(CONVERSION_GEN_BIN): $(CONVERSION_GEN) ## Build a local copy of conversion-gen
.PHONY: $(PROWJOB_GEN_BIN)
$(PROWJOB_GEN_BIN): $(PROWJOB_GEN) ## Build a local copy of prowjob-gen.

.PHONY: $(BOSKOSCTL_BIN)
$(BOSKOSCTL_BIN): $(BOSKOSCTL) ## Build a local copy of boskosctl.

.PHONY: $(CONVERSION_VERIFIER_BIN)
$(CONVERSION_VERIFIER_BIN): $(CONVERSION_VERIFIER) ## Build a local copy of conversion-verifier.

Expand Down Expand Up @@ -1006,6 +1042,9 @@ $(CONTROLLER_GEN): # Build controller-gen.
$(CONVERSION_GEN): # Build conversion-gen.
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(CONVERSION_GEN_PKG) $(CONVERSION_GEN_BIN) $(CONVERSION_GEN_VER)

$(BOSKOSCTL): # Build boskosctl from tools folder.
go build -o $(TOOLS_BIN_DIR)/$(BOSKOSCTL_BIN) ./hack/tools/boskosctl

$(CONVERSION_VERIFIER): # Build conversion-verifier.
GOBIN=$(TOOLS_BIN_DIR) $(GO_TOOLS_BUILD) $(CONVERSION_VERIFIER_PKG) $(CONVERSION_VERIFIER_BIN) $(CONVERSION_VERIFIER_VER)

Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ require (
github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505
github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20240404200847-de75746a9505
github.com/vmware/govmomi v0.37.3
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/mod v0.18.0
golang.org/x/tools v0.22.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
Expand Down
65 changes: 52 additions & 13 deletions hack/e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ REPO_ROOT=$(git rev-parse --show-toplevel)
RE_VCSIM='\[vcsim\\]'

# In CI, ARTIFACTS is set to a different directory. This stores the value of
# ARTIFACTS i1n ORIGINAL_ARTIFACTS and replaces ARTIFACTS by a temporary directory
# ARTIFACTS in ORIGINAL_ARTIFACTS and replaces ARTIFACTS by a temporary directory
# which gets cleaned up from credentials at the end of the test.
export ORIGINAL_ARTIFACTS=""
export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
Expand All @@ -36,10 +36,22 @@ fi
# shellcheck source=./hack/ensure-go.sh
source "${REPO_ROOT}/hack/ensure-go.sh"

# shellcheck source=./hack/ensure-kubectl.sh
source "${REPO_ROOT}/hack/ensure-kubectl.sh"
export BOSKOS_RESOURCE_OWNER=cluster-api-provider-vsphere
if [[ "${JOB_NAME}" != "" ]]; then
export BOSKOS_RESOURCE_OWNER="${JOB_NAME}/${BUILD_ID}"
fi
export BOSKOS_RESOURCE_TYPE=vsphere-project-cluster-api-provider

on_exit() {
# Only handle Boskos when we have to (not for vcsim)
if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
# Stop boskos heartbeat
[[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"

# If Boskos is being used then release the vsphere project.
[ -z "${BOSKOS_HOST:-}" ] || docker run -e VSPHERE_USERNAME -e VSPHERE_PASSWORD gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest release --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" --vsphere-server="${VSPHERE_SERVER}" --vsphere-tls-thumbprint="${VSPHERE_TLS_THUMBPRINT}" --vsphere-folder="${BOSKOS_RESOURCE_FOLDER}" --vsphere-resource-pool="${BOSKOS_RESOURCE_POOL}"
fi

# kill the VPN only when we started it (not vcsim)
if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
docker kill vpn
Expand Down Expand Up @@ -86,18 +98,13 @@ export VSPHERE_SSH_PRIVATE_KEY="/root/ssh/.private-key/private-key"
export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere.yaml"
export E2E_CONF_OVERRIDE_FILE=""
export E2E_VM_OPERATOR_VERSION="${VM_OPERATOR_VERSION:-v1.8.6-0-gde75746a}"
export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
export DOCKER_IMAGE_TAR="/tmp/images/image.tar"
export GC_KIND="false"

# Make tests run in-parallel
export GINKGO_NODES=5

# Set the kubeconfig to the IPAM cluster so the e2e tests can claim ip addresses
# for kube-vip.
export E2E_IPAM_KUBECONFIG="/root/ipam-conf/capv-services.conf"

# Only run the vpn/check for IPAM when we need them (not vcsim)
# Only run the vpn/check for IPAM when we need them (not for vcsim)
if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
# Run the vpn client in container
docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
Expand All @@ -107,11 +114,11 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
# Tail the vpn logs
docker logs vpn

# Wait until the VPN connection is active and we are able to reach the ipam cluster
function wait_for_ipam_reachable() {
# Wait until the VPN connection is active.
function wait_for_vpn_up() {
local n=0
until [ $n -ge 30 ]; do
kubectl --kubeconfig="${E2E_IPAM_KUBECONFIG}" --request-timeout=2s get inclusterippools.ipam.cluster.x-k8s.io && RET=$? || RET=$?
curl "https://${VSPHERE_SERVER}" --connect-timeout 2 -k && RET=$? || RET=$?
if [[ "$RET" -eq 0 ]]; then
break
fi
Expand All @@ -120,7 +127,39 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
done
return "$RET"
}
wait_for_ipam_reachable
wait_for_vpn_up

# If BOSKOS_HOST is set then acquire a vsphere-project from Boskos.
if [ -n "${BOSKOS_HOST:-}" ]; then
# Check out the account from Boskos and store the produced environment
# variables in a temporary file.
account_env_var_file="$(mktemp)"
docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest acquire --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-type="${BOSKOS_RESOURCE_TYPE}" 1>"${account_env_var_file}"
checkout_account_status="${?}"

# If the checkout process was a success then load the account's
# environment variables into this process.
# shellcheck disable=SC1090
[ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
export BOSKOS_RESOURCE_NAME=${BOSKOS_RESOURCE_NAME}
export VSPHERE_FOLDER=${BOSKOS_RESOURCE_FOLDER}
export VSPHERE_RESOURCE_POOL=${BOSKOS_RESOURCE_POOL}
export E2E_VSPHERE_IP_POOL="${BOSKOS_RESOURCE_IP_POOL}"

# Always remove the account environment variable file. It contains
# sensitive information.
rm -f "${account_env_var_file}"

if [ ! "${checkout_account_status}" = "0" ]; then
echo "error getting vsphere project from Boskos" 1>&2
exit "${checkout_account_status}"
fi

# Run the heartbeat to tell boskos periodically that we are still
# using the checked out account.
docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest heartbeat --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" >>"${ARTIFACTS}/boskos-heartbeat.log" 2>&1 &
HEART_BEAT_PID=$!
fi
fi

make envsubst
Expand Down
71 changes: 71 additions & 0 deletions hack/tools/boskosctl/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# syntax=docker/dockerfile:1.4

# Copyright 2024 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Build the manager binary
# Run this with docker build --build-arg builder_image=<golang:x.y.z>
ARG builder_image

# Build architecture
ARG ARCH

# Ignore Hadolint rule "Always tag the version of an image explicitly."
# It's an invalid finding since the image is explicitly set in the Makefile.
# https://github.com/hadolint/hadolint/wiki/DL3006
# hadolint ignore=DL3006
FROM ${builder_image} as builder
WORKDIR /workspace

# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
ARG goproxy=https://proxy.golang.org
# Run this with docker build --build-arg package=./controlplane/kubeadm or --build-arg package=./bootstrap/kubeadm
ENV GOPROXY=$goproxy

# Copy the Go Modules manifests
COPY go.mod go.mod
COPY go.sum go.sum

# Cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer
RUN --mount=type=cache,target=/go/pkg/mod \
go mod download

# Copy the sources
COPY ./ ./

# Cache the go build into the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls
RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
go build .

# Build
ARG ldflags

# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
RUN ls
RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \
-o boskosctl ./hack/tools/boskosctl

# Production image
FROM gcr.io/distroless/static:nonroot-amd64

WORKDIR /
COPY --from=builder /workspace/boskosctl .
# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
USER 65532
ENTRYPOINT ["/boskosctl"]
Loading

0 comments on commit 996d756

Please sign in to comment.