Skip to content

Commit

Permalink
Fix github workflow of e2e.
Browse files Browse the repository at this point in the history 
Signed-off-by: Lan Liang <gcslyp@gmail.com>
  • Loading branch information
@liangyuanpeng
liangyuanpeng committed Jan 25, 2024
1 parent 033b320 commit 1d12f9f
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 77 deletions.
167 changes: 90 additions & 77 deletions .github/workflows/e2e.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ jobs:
# kubectl
curl -L https://dl.k8s.io/${{ env.K8S_VERSION }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
# kind
curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/v0.17.0/kind-linux-amd64
curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
# Install
sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo
sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test
Expand All @@ -103,6 +103,12 @@ jobs:
ipFamily: ${IP_FAMILY}
nodes:
- role: control-plane
kubeadmConfigPatchesJSON6902:
- kind: ClusterConfiguration
patch: |
- op: add
path: /apiServer/certSANs/-
value: konnectivity-server.kube-system.svc.cluster.local
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
Expand All @@ -121,7 +127,10 @@ jobs:
extraMounts:
- hostPath: ./examples/kind/egress_selector_configuration.yaml
containerPath: /etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml
- role: worker
- role: worker
EOF
# dump the kubeconfig for later
/usr/local/bin/kind get kubeconfig --name ${{ env.KIND_CLUSTER_NAME}} > _artifacts/kubeconfig.conf
Expand All @@ -143,79 +152,83 @@ jobs:
kubectl apply -f examples/kind/konnectivity-server.yaml
kubectl apply -f examples/kind/konnectivity-agent-ds.yaml
# - name: Get Cluster status
# run: |
# # wait network is ready
# sleep 5
# /usr/local/bin/kubectl get nodes -o wide
# /usr/local/bin/kubectl get pods -A
# /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=konnectivity-agent
# /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=konnectivity-server
# # smoke test
# /usr/local/bin/kubectl run test --image httpd:2
# /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods test
# /usr/local/bin/kubectl logs test

# - name: Workaround CoreDNS for IPv6 airgapped
# if: ${{ matrix.ipFamily == 'ipv6' }}
# run: |
# # Patch CoreDNS to work in Github CI
# # 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
# # to work in an offline environment:
# # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
# # 2. Github CI adds following domains to resolv.conf search field:
# # .net.
# # CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
# # otherwise pods stops trying to resolve the domain.
# # Get the current config
# original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns)
# echo "Original CoreDNS config:"
# echo "${original_coredns}"
# # Patch it
# fixed_coredns=$(
# printf '%s' "${original_coredns}" | sed \
# -e 's/^.*kubernetes cluster\.local/& net/' \
# -e '/^.*upstream$/d' \
# -e '/^.*fallthrough.*$/d' \
# -e '/^.*forward . \/etc\/resolv.conf$/d' \
# -e '/^.*loop$/d' \
# )
# echo "Patched CoreDNS config:"
# echo "${fixed_coredns}"
# printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f -

# - name: Run tests
# run: |
# export KUBERNETES_CONFORMANCE_TEST='y'
# export E2E_REPORT_DIR=${PWD}/_artifacts

# # Run tests
# /usr/local/bin/ginkgo --nodes=25 \
# --focus="\[Conformance\]" \
# --skip="Feature|Federation|machinery|PerformanceDNS|DualStack|Disruptive|Serial|Slow|KubeProxy|LoadBalancer|GCE|Netpol|NetworkPolicy|NodeConformance" \
# /usr/local/bin/e2e.test \
# -- \
# --kubeconfig=${PWD}/_artifacts/kubeconfig.conf \
# --provider=local \
# --dump-logs-on-failure=false \
# --report-dir=${E2E_REPORT_DIR} \
# --disable-log-dump=true

# - name: Upload Junit Reports
# if: always()
# uses: actions/upload-artifact@v2
# with:
# name: kind-junit-${{ env.JOB_NAME }}-${{ github.run_id }}
# path: './_artifacts/*.xml'

# - name: Export logs
# if: always()
# run: |
# /usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug ./_artifacts/logs

# - name: Upload logs
# if: always()
# uses: actions/upload-artifact@v2
# with:
# name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
# path: ./_artifacts/logs
- name: Get Cluster status
run: |
# wait network is ready
sleep 5
/usr/local/bin/kubectl get nodes -o wide
/usr/local/bin/kubectl get pods -A
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns
# smoke test
/usr/local/bin/kubectl run test --image httpd:2
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods test
/usr/local/bin/kubectl get pods -A -owide
if [ ${{ matrix.ipFamily }} = "ipv4" ];then
sleep 50s
fi
/usr/local/bin/kubectl logs test
- name: Workaround CoreDNS for IPv6 airgapped
if: ${{ matrix.ipFamily == 'ipv6' }}
run: |
# Patch CoreDNS to work in Github CI
# 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
# to work in an offline environment:
# https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
# 2. Github CI adds following domains to resolv.conf search field:
# .net.
# CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
# otherwise pods stops trying to resolve the domain.
# Get the current config
original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns)
echo "Original CoreDNS config:"
echo "${original_coredns}"
# Patch it
fixed_coredns=$(
printf '%s' "${original_coredns}" | sed \
-e 's/^.*kubernetes cluster\.local/& net/' \
-e '/^.*upstream$/d' \
-e '/^.*fallthrough.*$/d' \
-e '/^.*forward . \/etc\/resolv.conf$/d' \
-e '/^.*loop$/d' \
)
echo "Patched CoreDNS config:"
echo "${fixed_coredns}"
printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f -
- name: Run tests
run: |
export KUBERNETES_CONFORMANCE_TEST='y'
export E2E_REPORT_DIR=${PWD}/_artifacts
# Run tests
/usr/local/bin/ginkgo --nodes=25 \
--focus="\[Conformance\]" \
--skip="Feature|Federation|machinery|PerformanceDNS|DualStack|Disruptive|Serial|Slow|KubeProxy|LoadBalancer|GCE|Netpol|NetworkPolicy|NodeConformance" \
/usr/local/bin/e2e.test \
-- \
--kubeconfig=${PWD}/_artifacts/kubeconfig.conf \
--provider=local \
--dump-logs-on-failure=false \
--report-dir=${E2E_REPORT_DIR} \
--disable-log-dump=true
- name: Upload Junit Reports
if: always()
uses: actions/upload-artifact@v2
with:
name: kind-junit-${{ env.JOB_NAME }}-${{ github.run_id }}
path: './_artifacts/*.xml'

- name: Export logs
if: always()
run: |
/usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug ./_artifacts/logs
- name: Upload logs
if: always()
uses: actions/upload-artifact@v2
with:
name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
path: ./_artifacts/logs
7 changes: 7 additions & 0 deletions examples/kind/konnectivity-agent-ds.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,13 @@ spec:
path: /healthz
initialDelaySeconds: 15
timeoutSeconds: 15
readinessProbe:
httpGet:
scheme: HTTP
port: 8093
path: /readyz
initialDelaySeconds: 15
timeoutSeconds: 15
volumeMounts:
- mountPath: /var/run/secrets/tokens
name: konnectivity-agent-token
Expand Down

0 comments on commit 1d12f9f

Please sign in to comment.